@delali/narsil 0.3.1-next.131 → 0.3.1-next.141
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/analysis-change-HQUT2Z3I.mjs +43 -0
- package/dist/analysis-change-entry-TZTG4AZB.mjs +1 -0
- package/dist/browser/analysis-change-JPV775LR.mjs +900 -0
- package/dist/browser/analysis-change-entry-7HR4LWL3.mjs +1 -0
- package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
- package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
- package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-5DB2VDGI.mjs} +1149 -1853
- package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
- package/dist/browser/chunk-CCOAOO73.mjs +86 -0
- package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-D33A5VMU.mjs} +16 -16
- package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-HKX2G6WH.mjs} +1520 -2159
- package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-KTOK275N.mjs} +606 -830
- package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-LVY2LI35.mjs} +2 -2
- package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
- package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
- package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
- package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
- package/dist/browser/chunk-R4WUDZQV.mjs +176 -0
- package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-TCLEOZ4P.mjs} +3 -3
- package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
- package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
- package/dist/browser/chunk-XNWPP5NS.mjs +21 -0
- package/dist/browser/chunk-ZEV26AQ6.mjs +270 -0
- package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
- package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-SNOWIT6Z.mjs} +370 -95
- package/dist/browser/import-TLB2A5LJ.mjs +9 -0
- package/dist/browser/module-BUFNZMEU.mjs +12 -0
- package/dist/browser/{module-4HIMOQM5.mjs → module-BVP4TKZU.mjs} +397 -109
- package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
- package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-ONYNYUDE.mjs} +8 -2
- package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
- package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-QXBKHJAG.mjs} +135 -183
- package/dist/browser/{write-YIC2MKRY.mjs → write-WHFBDWQS.mjs} +64 -90
- package/dist/{capabilities-B6WxrWxt.d.ts → capabilities-BCdUwK_L.d.ts} +10 -3
- package/dist/chunk-46NZOPHS.mjs +21 -0
- package/dist/{chunk-JT3YZGCE.mjs → chunk-5WYVUPGD.mjs} +3 -188
- package/dist/chunk-6SNBQESJ.mjs +214 -0
- package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
- package/dist/chunk-7AUASRE7.mjs +653 -0
- package/dist/{chunk-NMXRXFGZ.mjs → chunk-AST3SHFA.mjs} +27 -25
- package/dist/chunk-BOJZGUS6.mjs +177 -0
- package/dist/{chunk-T4OFG4IQ.mjs → chunk-BRUUBXCB.mjs} +92 -17
- package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
- package/dist/chunk-CZQ33DLW.mjs +78 -0
- package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
- package/dist/{chunk-EUCX5K74.mjs → chunk-DQNKVOJ2.mjs} +65 -65
- package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
- package/dist/chunk-FOCV4CBZ.mjs +87 -0
- package/dist/{chunk-HCGJKDZV.mjs → chunk-GLZHNSGB.mjs} +1 -1
- package/dist/chunk-GWOS5DYC.mjs +2637 -0
- package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
- package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
- package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
- package/dist/{chunk-TGHNXN7W.mjs → chunk-IO5F7ZBE.mjs} +261 -948
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/{chunk-7OIQ2SIO.mjs → chunk-KEIYVZUX.mjs} +5 -5
- package/dist/{chunk-PU6SKH7O.mjs → chunk-KGOXHLLZ.mjs} +7 -3
- package/dist/{chunk-WVRDRLIC.mjs → chunk-KZL72VTC.mjs} +70 -18
- package/dist/chunk-L76W3IZJ.mjs +193 -0
- package/dist/{chunk-2JWNJSNE.mjs → chunk-LA4WNDHL.mjs} +88 -801
- package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
- package/dist/{chunk-JJ6EUMXB.mjs → chunk-NJJCPFXN.mjs} +2 -2
- package/dist/chunk-NN33CZRE.mjs +86 -0
- package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
- package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
- package/dist/chunk-NRUXBYPH.mjs +121 -0
- package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
- package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
- package/dist/{chunk-F6AAXI7T.mjs → chunk-RHKRTYGO.mjs} +13 -4
- package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
- package/dist/{chunk-RCMNL7RI.mjs → chunk-SBMAWTRG.mjs} +41 -4
- package/dist/chunk-SRJBFG2T.mjs +881 -0
- package/dist/{chunk-X5CNUKD5.mjs → chunk-TNBIK2LQ.mjs} +1663 -1652
- package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
- package/dist/chunk-UUZCQQG2.mjs +209 -0
- package/dist/{chunk-6M6SDH3S.mjs → chunk-VEFN6X6Q.mjs} +8 -8
- package/dist/chunk-VJEGOELZ.mjs +177 -0
- package/dist/chunk-W2COJ5AH.mjs +148 -0
- package/dist/{chunk-RWESFSJI.mjs → chunk-WJGO5PGV.mjs} +317 -174
- package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
- package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
- package/dist/{chunk-Y4BVTQ53.mjs → chunk-XZ3SRNDJ.mjs} +25 -64
- package/dist/{chunk-K77FJXKS.mjs → chunk-YHUTF6EA.mjs} +474 -93
- package/dist/{chunk-OGLV4I3L.mjs → chunk-YLNSJQLX.mjs} +12 -7
- package/dist/{chunk-SOHMMNCH.mjs → chunk-Z4HFMYNX.mjs} +32 -27
- package/dist/{chunk-5W7AEVTH.mjs → chunk-ZMBFWQXN.mjs} +2 -2
- package/dist/{chunk-2MKX2SCO.mjs → chunk-ZPWS2SKJ.mjs} +297 -274
- package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
- package/dist/{client-gNAfaHcE.d.ts → client-B12YFALP.d.ts} +18 -3
- package/dist/client.d.ts +7 -7
- package/dist/client.mjs +11 -1
- package/dist/cold-module-4X7WJCFM.mjs +783 -0
- package/dist/{config-D1cDv3sE.d.ts → config-BhZxIeb1.d.ts} +10 -1
- package/dist/{config-CmYJO34X.d.ts → config-CIG5yDyg.d.ts} +2 -2
- package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-7XOQ7VQP.mjs} +14 -14
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +24 -8
- package/dist/distribution.mjs +982 -211
- package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-324D2DDD.mjs} +362 -94
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-m_Ddr1bz.d.ts → engine-CtzbHOmG.d.ts} +64 -7
- package/dist/import-OOBUW3RR.mjs +44 -0
- package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
- package/dist/index.browser.mjs +5 -5
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +22 -22
- package/dist/{long-query-thread-BSL25PD6.mjs → long-query-thread-4SW7RWDE.mjs} +15 -15
- package/dist/{module-2E4XYSGT.mjs → module-DXSJ5LT2.mjs} +378 -87
- package/dist/module-IYPLWHGC.mjs +48 -0
- package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +28 -26
- package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
- package/dist/react.d.ts +9 -9
- package/dist/react.mjs +4 -0
- package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-FEYAC4SB.mjs} +8 -2
- package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
- package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
- package/dist/server.d.ts +12 -10
- package/dist/server.mjs +54 -21
- package/dist/{snapshot-DOQLGGEX.mjs → snapshot-CGM65VCD.mjs} +101 -197
- package/dist/synonyms.mjs +6 -4
- package/dist/{tasks-Co8HfRD7.d.ts → tasks-CjRJRGTb.d.ts} +26 -8
- package/dist/{thread-SKO7CN3R.mjs → thread-KGNAIKPD.mjs} +14 -14
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +6 -5
- package/dist/workers/entry.mjs +8 -8
- package/dist/{write-ZKR7SNRT.mjs → write-2DZPEQL2.mjs} +82 -107
- package/openapi.json +114 -6
- package/package.json +12 -8
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
import { spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
|
|
2
|
-
import { VECTOR_WORKER_COPIES_ALLOWED,
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
import {
|
|
6
|
-
import './chunk-
|
|
7
|
-
import
|
|
2
|
+
import { VECTOR_WORKER_COPIES_ALLOWED, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, noBuildHistory, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, handOverBuildTurns, filterForOptions, OSQ4_MIN_DIMENSION, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, allLiveDocIds, threadsHoldCurrentLayout, WORKER_COPY_MIN_VECTORS, SHUFFLED_REBUILD_MEMORY_RETRY_MS, recalibrateFromStore, signatureOf, adoptGraph, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-6RHTAVZX.mjs';
|
|
3
|
+
import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, osqBitsOf, createOsqQuantizer, ordinalFilterValues, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, BLOCK_MAX_BYTES, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, growBufferTo, nativeStoreScratchBytes, noteFallback, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, QUERY_STAGED, grownByteLength, createVectorBlock, ensureBlockSlots, createHNSWIndex, topLayerInParts, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy } from './chunk-HKX2G6WH.mjs';
|
|
4
|
+
export { releaseLocksHeldBy } from './chunk-HKX2G6WH.mjs';
|
|
5
|
+
import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-LVY2LI35.mjs';
|
|
6
|
+
import { createVectorFileReader } from './chunk-3ZBJYAS4.mjs';
|
|
7
|
+
import './chunk-OIZS5KIX.mjs';
|
|
8
|
+
import { MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, compareCodePoints, UNPLACED_FLAG_CHUNK_ORDINALS, MAX_VECTOR_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, resolveWorkerCount, resolveWorkerEntry, detectRuntime, noteThreadSlotLive, noteThreadSlotGone, WORKER_YOUNG_GENERATION_MB, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS } from './chunk-D33A5VMU.mjs';
|
|
9
|
+
import './chunk-MZI3TEOW.mjs';
|
|
10
|
+
import { NarsilError, ErrorCodes, isNarsilError } from './chunk-35B42UFM.mjs';
|
|
8
11
|
|
|
9
12
|
// src/core/heap.ts
|
|
10
13
|
function createMaxHeap(compare) {
|
|
@@ -351,21 +354,6 @@ function createSharedVectorSearcher(options) {
|
|
|
351
354
|
};
|
|
352
355
|
}
|
|
353
356
|
|
|
354
|
-
// src/vector/vector-store/vector-file.browser.ts
|
|
355
|
-
function filesystemUnavailable() {
|
|
356
|
-
throw new NarsilError(
|
|
357
|
-
ErrorCodes.CONFIG_INVALID,
|
|
358
|
-
"Reading vectors from a checkpoint file requires a Node.js runtime; keep the field in memory in the browser"
|
|
359
|
-
);
|
|
360
|
-
}
|
|
361
|
-
function createVectorFileReader() {
|
|
362
|
-
return {
|
|
363
|
-
readInto: () => filesystemUnavailable(),
|
|
364
|
-
retainOnly: () => void 0,
|
|
365
|
-
close: () => void 0
|
|
366
|
-
};
|
|
367
|
-
}
|
|
368
|
-
|
|
369
357
|
// src/vector/vector-store/hot-block-distance.ts
|
|
370
358
|
function jsDistance(a, b, magA, magB, metric) {
|
|
371
359
|
switch (metric) {
|
|
@@ -377,6 +365,11 @@ function jsDistance(a, b, magA, magB, metric) {
|
|
|
377
365
|
return euclideanDistance(a, b);
|
|
378
366
|
}
|
|
379
367
|
}
|
|
368
|
+
function distanceFromQueryOf(source, coldScratch, query, queryMagnitude, ordinal, metric) {
|
|
369
|
+
if (!source.holdsOrdinal(ordinal)) return Number.POSITIVE_INFINITY;
|
|
370
|
+
const vector = source.isCold(ordinal) && source.readColdInto(ordinal, coldScratch) ? coldScratch : source.vectorAt(ordinal);
|
|
371
|
+
return jsDistance(query, vector, queryMagnitude, source.magnitudeAt(ordinal), metric);
|
|
372
|
+
}
|
|
380
373
|
function inMemory(reads, ordinal) {
|
|
381
374
|
return ordinal >= 0 && ordinal < reads.limit && reads.diskFile[ordinal] === IN_MEMORY;
|
|
382
375
|
}
|
|
@@ -473,9 +466,7 @@ function openSharedVectorStore(initial, threadSlot) {
|
|
|
473
466
|
}
|
|
474
467
|
return diskFile[ordinal];
|
|
475
468
|
}
|
|
476
|
-
|
|
477
|
-
return fileOf(ordinal) !== IN_MEMORY;
|
|
478
|
-
}
|
|
469
|
+
const isCold = (ordinal) => fileOf(ordinal) !== IN_MEMORY;
|
|
479
470
|
function holdState(ordinal) {
|
|
480
471
|
if (ordinal < 0) return NOT_HELD;
|
|
481
472
|
if (ordinal >= present.length) {
|
|
@@ -487,9 +478,7 @@ function openSharedVectorStore(initial, threadSlot) {
|
|
|
487
478
|
const index = Math.floor(ordinal / capacity);
|
|
488
479
|
return index < handles.blocks.length && handles.blocks[index] !== null ? HELD_HOT : NOT_HELD;
|
|
489
480
|
}
|
|
490
|
-
|
|
491
|
-
return holdState(ordinal) !== NOT_HELD;
|
|
492
|
-
}
|
|
481
|
+
const holds = (ordinal) => holdState(ordinal) !== NOT_HELD;
|
|
493
482
|
function readFromFile(file, offset, target) {
|
|
494
483
|
const path = handles.vectorFiles[file];
|
|
495
484
|
if (path === void 0 || path.length === 0) return false;
|
|
@@ -697,6 +686,7 @@ function openSharedVectorStore(initial, threadSlot) {
|
|
|
697
686
|
}
|
|
698
687
|
return jsDistance(currentQuery, vectorIn(block, local), prepared.magnitude, magnitudes[ordinal], metric);
|
|
699
688
|
},
|
|
689
|
+
distanceFromQuery: (query, queryMagnitude, ordinal, metric) => distanceFromQueryOf(view, coldScratchB, query, queryMagnitude, ordinal, metric),
|
|
700
690
|
hotBlockReads,
|
|
701
691
|
queryDistance: (prepared, metric) => queryDistanceOf(view, prepared, metric),
|
|
702
692
|
ordinalDistance: (from, metric) => ordinalDistanceOf(view, from, metric),
|
|
@@ -840,6 +830,11 @@ function createOrdinalReads(openStore, readable) {
|
|
|
840
830
|
if (open === null || !readable(ordinal)) return Number.POSITIVE_INFINITY;
|
|
841
831
|
return open.view.distanceFromArena(prepared, ordinal, metric);
|
|
842
832
|
},
|
|
833
|
+
distanceFromQuery(query, queryMagnitude, ordinal, metric) {
|
|
834
|
+
const open = openStore();
|
|
835
|
+
if (open === null || !readable(ordinal)) return Number.POSITIVE_INFINITY;
|
|
836
|
+
return open.view.distanceFromQuery(query, queryMagnitude, ordinal, metric);
|
|
837
|
+
},
|
|
843
838
|
queryDistance(prepared, metric) {
|
|
844
839
|
const open = openStore();
|
|
845
840
|
if (open === null) return () => Number.POSITIVE_INFINITY;
|
|
@@ -1016,13 +1011,12 @@ function blockFullyCold(store, blockIndex, documents, replacedOrdinals) {
|
|
|
1016
1011
|
}
|
|
1017
1012
|
return true;
|
|
1018
1013
|
}
|
|
1019
|
-
function fileHoldsSameBytes(store, ordinal, location) {
|
|
1020
|
-
const fromFile = new Float32Array(store.handles.dimension);
|
|
1014
|
+
function fileHoldsSameBytes(store, ordinal, location, fromFile) {
|
|
1021
1015
|
if (!store.view.readFromFile(location.fileIndex, location.offset, fromFile)) return false;
|
|
1022
1016
|
const held = store.view.vectorAt(ordinal);
|
|
1023
|
-
const
|
|
1024
|
-
const
|
|
1025
|
-
for (let i = 0; i <
|
|
1017
|
+
const fileWords = new Uint32Array(fromFile.buffer, fromFile.byteOffset, fromFile.length);
|
|
1018
|
+
const heldWords = new Uint32Array(held.buffer, held.byteOffset, held.length);
|
|
1019
|
+
for (let i = 0; i < fileWords.length; i++) if (fileWords[i] !== heldWords[i]) return false;
|
|
1026
1020
|
return true;
|
|
1027
1021
|
}
|
|
1028
1022
|
function forgetUnreferencedFiles(store, documents, replacedOrdinals) {
|
|
@@ -1073,6 +1067,7 @@ function createVectorStore(options) {
|
|
|
1073
1067
|
const replacedOrdinals = /* @__PURE__ */ new Set();
|
|
1074
1068
|
let unknownPartitions = 0;
|
|
1075
1069
|
let liveCount = 0;
|
|
1070
|
+
let readBackVector = null;
|
|
1076
1071
|
let open = options?.dimension === void 0 ? null : openHandles(options.dimension, codeBits, blockBytes);
|
|
1077
1072
|
function forgetDocuments() {
|
|
1078
1073
|
docToOrd.clear();
|
|
@@ -1217,7 +1212,10 @@ function createVectorStore(options) {
|
|
|
1217
1212
|
releaseToFile(ordinal, location) {
|
|
1218
1213
|
const store = requireOpen();
|
|
1219
1214
|
if (ordToDoc[ordinal] === void 0) return false;
|
|
1220
|
-
if (store.diskFile[ordinal] === IN_MEMORY
|
|
1215
|
+
if (store.diskFile[ordinal] === IN_MEMORY) {
|
|
1216
|
+
readBackVector ??= new Float32Array(store.handles.dimension);
|
|
1217
|
+
if (!fileHoldsSameBytes(store, ordinal, location, readBackVector)) return false;
|
|
1218
|
+
}
|
|
1221
1219
|
Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
|
|
1222
1220
|
store.diskOffset[ordinal] = location.offset;
|
|
1223
1221
|
store.diskFile[ordinal] = location.fileIndex;
|
|
@@ -2599,19 +2597,25 @@ function beginCheckpointRestore(state, shape) {
|
|
|
2599
2597
|
const dead = isDead(file.dead, i);
|
|
2600
2598
|
if (dead && !graphHoldsNodeFor(firstNumber + i)) continue;
|
|
2601
2599
|
const docId = payload.docIds[i];
|
|
2602
|
-
const vector = vectors.subarray(i * dimension, (i + 1) * dimension);
|
|
2603
2600
|
const offset = onDisk ? location.vectorsOffset + i * dimension * 4 : 0;
|
|
2601
|
+
const vectorHere = () => file.byParts === void 0 ? vectors.subarray(i * dimension, (i + 1) * dimension) : file.byParts.vectorAt(i);
|
|
2604
2602
|
if (dead) {
|
|
2605
|
-
ordinals[i] = state.store.insertReplaced(
|
|
2603
|
+
ordinals[i] = state.store.insertReplaced(
|
|
2604
|
+
docId,
|
|
2605
|
+
vectorHere(),
|
|
2606
|
+
fileIndex === null ? void 0 : { fileIndex, offset }
|
|
2607
|
+
);
|
|
2606
2608
|
tombstoned.add(ordinals[i]);
|
|
2607
2609
|
continue;
|
|
2608
2610
|
}
|
|
2609
2611
|
const replaced = state.store.getOrdinal(docId);
|
|
2610
2612
|
if (replaced !== void 0) state.osq?.removeOrdinal(replaced);
|
|
2611
2613
|
if (fileIndex !== null) {
|
|
2612
|
-
|
|
2614
|
+
const known = file.byParts?.magnitudes[i];
|
|
2615
|
+
ordinals[i] = state.store.insertCold(docId, known ?? magnitude(vectorHere()), { fileIndex, offset });
|
|
2613
2616
|
continue;
|
|
2614
2617
|
}
|
|
2618
|
+
const vector = vectorHere();
|
|
2615
2619
|
ordinals[i] = state.store.insert(docId, vector);
|
|
2616
2620
|
if (onDisk) state.pendingLocations.set(docId, { path: location.path, offset });
|
|
2617
2621
|
}
|
|
@@ -2653,6 +2657,223 @@ function beginCheckpointRestore(state, shape) {
|
|
|
2653
2657
|
return { addFile, finish };
|
|
2654
2658
|
}
|
|
2655
2659
|
|
|
2660
|
+
// src/vector/vector-index/cold.ts
|
|
2661
|
+
function beginColdHold(state, field, loaded, searchAnswered) {
|
|
2662
|
+
const hold = {
|
|
2663
|
+
field,
|
|
2664
|
+
loaded,
|
|
2665
|
+
searchAnswered,
|
|
2666
|
+
written: /* @__PURE__ */ new Map(),
|
|
2667
|
+
removedNumbers: /* @__PURE__ */ new Set(),
|
|
2668
|
+
removedIds: /* @__PURE__ */ new Set(),
|
|
2669
|
+
writes: [],
|
|
2670
|
+
live: field.liveCount
|
|
2671
|
+
};
|
|
2672
|
+
state.cold = hold;
|
|
2673
|
+
return hold;
|
|
2674
|
+
}
|
|
2675
|
+
function endColdHold(state) {
|
|
2676
|
+
const hold = state.cold;
|
|
2677
|
+
if (hold === null) return;
|
|
2678
|
+
state.cold = null;
|
|
2679
|
+
for (const write of hold.writes) write();
|
|
2680
|
+
hold.field.close();
|
|
2681
|
+
}
|
|
2682
|
+
function cancelColdHold(state) {
|
|
2683
|
+
const hold = state.cold;
|
|
2684
|
+
if (hold === null) return;
|
|
2685
|
+
state.cold = null;
|
|
2686
|
+
hold.field.close();
|
|
2687
|
+
}
|
|
2688
|
+
function openColdField(state, field, load) {
|
|
2689
|
+
cancelColdHold(state);
|
|
2690
|
+
let finished = () => void 0;
|
|
2691
|
+
let failed = () => void 0;
|
|
2692
|
+
const loaded = new Promise((resolve, reject) => {
|
|
2693
|
+
finished = resolve;
|
|
2694
|
+
failed = reject;
|
|
2695
|
+
});
|
|
2696
|
+
let searchAnswered = () => void 0;
|
|
2697
|
+
const firstSearchAnswered = new Promise((resolve) => {
|
|
2698
|
+
searchAnswered = resolve;
|
|
2699
|
+
});
|
|
2700
|
+
const hold = beginColdHold(state, field, loaded, searchAnswered);
|
|
2701
|
+
void load(
|
|
2702
|
+
(shape) => beginCheckpointRestore(state, shape),
|
|
2703
|
+
() => state.cold === hold,
|
|
2704
|
+
firstSearchAnswered
|
|
2705
|
+
).then(
|
|
2706
|
+
() => {
|
|
2707
|
+
if (state.cold === hold) endColdHold(state);
|
|
2708
|
+
finished();
|
|
2709
|
+
},
|
|
2710
|
+
(error) => failed(error)
|
|
2711
|
+
);
|
|
2712
|
+
loaded.catch(() => void 0);
|
|
2713
|
+
}
|
|
2714
|
+
function retireOlderCopy(hold, docId) {
|
|
2715
|
+
if (hold.written.delete(docId)) return true;
|
|
2716
|
+
if (hold.removedIds.has(docId)) return false;
|
|
2717
|
+
const number = hold.field.numberOf(docId);
|
|
2718
|
+
if (number === void 0) return false;
|
|
2719
|
+
hold.removedIds.add(docId);
|
|
2720
|
+
hold.removedNumbers.add(number);
|
|
2721
|
+
return true;
|
|
2722
|
+
}
|
|
2723
|
+
function coldInsert(hold, docId, vector, partitionId) {
|
|
2724
|
+
const replaced = retireOlderCopy(hold, docId);
|
|
2725
|
+
if (!replaced) hold.live += 1;
|
|
2726
|
+
hold.written.set(docId, { vector: vector.slice(), partitionId });
|
|
2727
|
+
return replaced;
|
|
2728
|
+
}
|
|
2729
|
+
function coldRemove(hold, docId) {
|
|
2730
|
+
const removed = retireOlderCopy(hold, docId);
|
|
2731
|
+
if (removed) hold.live -= 1;
|
|
2732
|
+
return removed;
|
|
2733
|
+
}
|
|
2734
|
+
function coldHas(hold, docId) {
|
|
2735
|
+
if (hold.written.has(docId)) return true;
|
|
2736
|
+
if (hold.removedIds.has(docId)) return false;
|
|
2737
|
+
return hold.field.numberOf(docId) !== void 0;
|
|
2738
|
+
}
|
|
2739
|
+
function coldVectorOf(hold, docId) {
|
|
2740
|
+
const written = hold.written.get(docId);
|
|
2741
|
+
if (written !== void 0) return written.vector.slice();
|
|
2742
|
+
if (hold.removedIds.has(docId)) return null;
|
|
2743
|
+
const number = hold.field.numberOf(docId);
|
|
2744
|
+
return number === void 0 ? null : hold.field.vectorOfNumber(number);
|
|
2745
|
+
}
|
|
2746
|
+
function isAdmitted(subsets, docId) {
|
|
2747
|
+
for (const subset of subsets) {
|
|
2748
|
+
const { docStore } = subset.owner;
|
|
2749
|
+
if (docStore.numberingGeneration() !== subset.numberingGeneration) continue;
|
|
2750
|
+
const internalId = docStore.getInternalId(docId);
|
|
2751
|
+
if (internalId === void 0) continue;
|
|
2752
|
+
const word = subset.bits[internalId >>> 5];
|
|
2753
|
+
if (word !== void 0 && (word & 1 << (internalId & 31)) !== 0 && docStore.resolver().isLive(internalId)) {
|
|
2754
|
+
return true;
|
|
2755
|
+
}
|
|
2756
|
+
}
|
|
2757
|
+
return false;
|
|
2758
|
+
}
|
|
2759
|
+
function admissionOf(options, hold) {
|
|
2760
|
+
const { filterSubsets, filterPartitions } = options;
|
|
2761
|
+
if (filterSubsets !== void 0) return (docId) => isAdmitted(filterSubsets, docId);
|
|
2762
|
+
if (filterPartitions !== void 0) {
|
|
2763
|
+
return (docId) => {
|
|
2764
|
+
const partition = hold.field.partitionOf(docId);
|
|
2765
|
+
return partition !== void 0 && filterPartitions.has(partition);
|
|
2766
|
+
};
|
|
2767
|
+
}
|
|
2768
|
+
return null;
|
|
2769
|
+
}
|
|
2770
|
+
function bitCount(word) {
|
|
2771
|
+
let value = word - (word >>> 1 & 1431655765);
|
|
2772
|
+
value = (value & 858993459) + (value >>> 2 & 858993459);
|
|
2773
|
+
return (value + (value >>> 4) & 252645135) * 16843009 >>> 24;
|
|
2774
|
+
}
|
|
2775
|
+
function admittedBySubsets(subsets) {
|
|
2776
|
+
let admitted = 0;
|
|
2777
|
+
for (const subset of subsets) {
|
|
2778
|
+
for (let at = 0; at < subset.bits.length; at++) admitted += bitCount(subset.bits[at]);
|
|
2779
|
+
}
|
|
2780
|
+
return admitted;
|
|
2781
|
+
}
|
|
2782
|
+
function* idsOfSubsets(subsets) {
|
|
2783
|
+
for (const subset of subsets) {
|
|
2784
|
+
const { bits } = subset;
|
|
2785
|
+
const { docStore } = subset.owner;
|
|
2786
|
+
if (docStore.numberingGeneration() !== subset.numberingGeneration) continue;
|
|
2787
|
+
for (let wordIndex = 0; wordIndex < bits.length; wordIndex++) {
|
|
2788
|
+
let word = bits[wordIndex];
|
|
2789
|
+
while (word !== 0) {
|
|
2790
|
+
const internalId = (wordIndex << 5) + (Math.clz32(word & -word) ^ 31);
|
|
2791
|
+
word &= word - 1;
|
|
2792
|
+
if (!docStore.resolver().isLive(internalId)) continue;
|
|
2793
|
+
const docId = docStore.getExternalId(internalId);
|
|
2794
|
+
if (docId !== void 0) yield docId;
|
|
2795
|
+
}
|
|
2796
|
+
}
|
|
2797
|
+
}
|
|
2798
|
+
}
|
|
2799
|
+
function scoreOf(query, qMag, vector, metric) {
|
|
2800
|
+
const vectorMagnitude = metric === "cosine" ? magnitude(vector) : 0;
|
|
2801
|
+
return toScore(toDistance(query, vector, qMag, vectorMagnitude, metric), metric);
|
|
2802
|
+
}
|
|
2803
|
+
function scanWritten(hold, query, k, options, admits) {
|
|
2804
|
+
if (hold.written.size === 0) return [];
|
|
2805
|
+
const qMag = magnitude(query);
|
|
2806
|
+
const heap = createBoundedMaxHeap(highScoreFirst, Math.max(k, 1));
|
|
2807
|
+
for (const [docId, { vector }] of hold.written) {
|
|
2808
|
+
if (admits !== null && !admits(docId)) continue;
|
|
2809
|
+
const score = scoreOf(query, qMag, vector, options.metric);
|
|
2810
|
+
if (score >= options.minSimilarity) heap.push({ docId, score });
|
|
2811
|
+
}
|
|
2812
|
+
return heap.toSortedArray().reverse();
|
|
2813
|
+
}
|
|
2814
|
+
function scanSubsets(hold, query, k, options, subsets) {
|
|
2815
|
+
const qMag = magnitude(query);
|
|
2816
|
+
const heap = createBoundedMaxHeap(highScoreFirst, Math.max(k, 1));
|
|
2817
|
+
let matched = 0;
|
|
2818
|
+
for (const docId of idsOfSubsets(subsets)) {
|
|
2819
|
+
const written = hold.written.get(docId);
|
|
2820
|
+
let score;
|
|
2821
|
+
if (written !== void 0) {
|
|
2822
|
+
score = scoreOf(query, qMag, written.vector, options.metric);
|
|
2823
|
+
} else {
|
|
2824
|
+
if (hold.removedIds.has(docId)) continue;
|
|
2825
|
+
const number = hold.field.numberOf(docId);
|
|
2826
|
+
if (number === void 0) continue;
|
|
2827
|
+
score = scoreOf(query, qMag, hold.field.reusedVectorOfNumber(number), options.metric);
|
|
2828
|
+
}
|
|
2829
|
+
if (score < options.minSimilarity) continue;
|
|
2830
|
+
matched += 1;
|
|
2831
|
+
heap.push({ docId, score });
|
|
2832
|
+
}
|
|
2833
|
+
return { results: heap.toSortedArray().reverse(), matched };
|
|
2834
|
+
}
|
|
2835
|
+
function outcomeWithoutGraphWalk(state, hold, query, k, options) {
|
|
2836
|
+
if (hold.live === 0) return { results: [], matched: 0, matchedExact: true };
|
|
2837
|
+
const subsets = options.filterSubsets;
|
|
2838
|
+
let admitted = hold.live;
|
|
2839
|
+
if (subsets !== void 0) {
|
|
2840
|
+
admitted = admittedBySubsets(subsets);
|
|
2841
|
+
if (admitted === 0) return { results: [], matched: 0, matchedExact: true };
|
|
2842
|
+
if (admitted / hold.live < state.filterThreshold) {
|
|
2843
|
+
const { results, matched } = scanSubsets(hold, query, k, options, subsets);
|
|
2844
|
+
return { results: results.slice(0, Math.max(k, 0)), matched, matchedExact: true };
|
|
2845
|
+
}
|
|
2846
|
+
}
|
|
2847
|
+
if (k <= 0) {
|
|
2848
|
+
return options.minSimilarity === Number.NEGATIVE_INFINITY ? { results: [], matched: admitted, matchedExact: true } : { results: [], matched: 0, matchedExact: false };
|
|
2849
|
+
}
|
|
2850
|
+
return { admits: admissionOf(options, hold), admitted };
|
|
2851
|
+
}
|
|
2852
|
+
function outcomeOfGraphWalk(hold, walk, fromGraph, query, k, options) {
|
|
2853
|
+
const results = mergeResults(fromGraph, scanWritten(hold, query, k, options, walk.admits), k);
|
|
2854
|
+
return options.minSimilarity === Number.NEGATIVE_INFINITY ? { results, matched: walk.admitted, matchedExact: true } : { results, matched: results.length, matchedExact: false };
|
|
2855
|
+
}
|
|
2856
|
+
function coldSearch(state, hold, query, k, options) {
|
|
2857
|
+
const walk = outcomeWithoutGraphWalk(state, hold, query, k, options);
|
|
2858
|
+
if ("results" in walk) {
|
|
2859
|
+
hold.searchAnswered();
|
|
2860
|
+
return walk;
|
|
2861
|
+
}
|
|
2862
|
+
const fromGraph = hold.field.search(query, k, options, walk.admits, hold.removedNumbers);
|
|
2863
|
+
hold.searchAnswered();
|
|
2864
|
+
return outcomeOfGraphWalk(hold, walk, fromGraph, query, k, options);
|
|
2865
|
+
}
|
|
2866
|
+
async function coldSearchConcurrently(state, hold, query, k, options) {
|
|
2867
|
+
const walk = outcomeWithoutGraphWalk(state, hold, query, k, options);
|
|
2868
|
+
if ("results" in walk) {
|
|
2869
|
+
hold.searchAnswered();
|
|
2870
|
+
return walk;
|
|
2871
|
+
}
|
|
2872
|
+
const fromGraph = await hold.field.searchConcurrently(query, k, options, walk.admits, hold.removedNumbers);
|
|
2873
|
+
hold.searchAnswered();
|
|
2874
|
+
return outcomeOfGraphWalk(hold, walk, fromGraph, query, k, options);
|
|
2875
|
+
}
|
|
2876
|
+
|
|
2656
2877
|
// src/vector/vector-index/flagged-documents.ts
|
|
2657
2878
|
var FlaggedDocuments = class extends Set {
|
|
2658
2879
|
constructor(store, layout) {
|
|
@@ -2804,7 +3025,7 @@ function completeGraph(state) {
|
|
|
2804
3025
|
}
|
|
2805
3026
|
function maintenanceStatus(state) {
|
|
2806
3027
|
const storeSize = state.store.size;
|
|
2807
|
-
const graphCount = state.hnsw ? 1 : 0;
|
|
3028
|
+
const graphCount = state.hnsw || state.cold ? 1 : 0;
|
|
2808
3029
|
const graphNodes = state.hnsw === null ? storeSize : state.hnsw.size + state.hnsw.tombstoneCount;
|
|
2809
3030
|
const tombstoned = state.tombstones.size + state.store.replacedVectorCount;
|
|
2810
3031
|
const tombstoneRatio = graphNodes > 0 ? tombstoned / graphNodes : 0;
|
|
@@ -2967,6 +3188,91 @@ function withBufferedVectors(state, query, k, options, filter, graphResults, buf
|
|
|
2967
3188
|
return mergeResults(graphResults, scan.results, k);
|
|
2968
3189
|
}
|
|
2969
3190
|
|
|
3191
|
+
// src/vector/vector-index/writes.ts
|
|
3192
|
+
function createVectorWrites(state) {
|
|
3193
|
+
function validateDimension(vector) {
|
|
3194
|
+
if (vector.length !== state.dimension) {
|
|
3195
|
+
throw new NarsilError(
|
|
3196
|
+
ErrorCodes.VECTOR_DIMENSION_MISMATCH,
|
|
3197
|
+
`Vector dimension mismatch: expected ${state.dimension}, got ${vector.length}`,
|
|
3198
|
+
{ expected: state.dimension, received: vector.length }
|
|
3199
|
+
);
|
|
3200
|
+
}
|
|
3201
|
+
}
|
|
3202
|
+
function heldVectorOrUndefinedWhereItsFileFailsToRead(ordinal) {
|
|
3203
|
+
try {
|
|
3204
|
+
return state.store.entryForOrdinal(ordinal)?.vector;
|
|
3205
|
+
} catch {
|
|
3206
|
+
return void 0;
|
|
3207
|
+
}
|
|
3208
|
+
}
|
|
3209
|
+
function holdsThisVectorAlready(docId, ordinal, vector) {
|
|
3210
|
+
if (state.tombstones.has(docId)) return false;
|
|
3211
|
+
const held = heldVectorOrUndefinedWhereItsFileFailsToRead(ordinal);
|
|
3212
|
+
if (held === void 0) return false;
|
|
3213
|
+
for (let i = 0; i < vector.length; i++) if (held[i] !== vector[i]) return false;
|
|
3214
|
+
return true;
|
|
3215
|
+
}
|
|
3216
|
+
function applyInsert(docId, vector, partitionId) {
|
|
3217
|
+
handOverBuildTurns(state);
|
|
3218
|
+
const previous = state.store.getOrdinal(docId);
|
|
3219
|
+
if (previous !== void 0 && holdsThisVectorAlready(docId, previous, vector)) {
|
|
3220
|
+
if (partitionId !== void 0 && state.store.partitionOfOrdinal(previous) !== partitionId) {
|
|
3221
|
+
noteWrite(state);
|
|
3222
|
+
state.store.setPartition(docId, partitionId);
|
|
3223
|
+
}
|
|
3224
|
+
return;
|
|
3225
|
+
}
|
|
3226
|
+
noteWrite(state);
|
|
3227
|
+
state.tombstones.delete(docId);
|
|
3228
|
+
const graphHoldsPrevious = previous !== void 0 && ((state.hnsw?.holdsNode(previous) ?? false) || (state.freshGraph?.holdsNode(previous) ?? false));
|
|
3229
|
+
state.store.insert(docId, vector, partitionId, graphHoldsPrevious);
|
|
3230
|
+
if (previous !== void 0) {
|
|
3231
|
+
state.hnsw?.markTombstoneOrdinal(previous);
|
|
3232
|
+
state.freshGraph?.markTombstoneOrdinal(previous);
|
|
3233
|
+
if (!graphHoldsPrevious) state.osq?.removeOrdinal(previous);
|
|
3234
|
+
}
|
|
3235
|
+
state.buffer.add(docId);
|
|
3236
|
+
state.pendingLocations.delete(docId);
|
|
3237
|
+
if (!threadsHoldCurrentLayout(state)) scheduleWorkerCopyLoad(state);
|
|
3238
|
+
}
|
|
3239
|
+
function applyRemove(docId) {
|
|
3240
|
+
handOverBuildTurns(state);
|
|
3241
|
+
if (!state.store.has(docId)) return;
|
|
3242
|
+
noteWrite(state);
|
|
3243
|
+
state.tombstones.add(docId);
|
|
3244
|
+
state.buffer.delete(docId);
|
|
3245
|
+
state.pendingLocations.delete(docId);
|
|
3246
|
+
if (state.hnsw) {
|
|
3247
|
+
state.hnsw.markTombstone(docId);
|
|
3248
|
+
}
|
|
3249
|
+
}
|
|
3250
|
+
return {
|
|
3251
|
+
insert(docId, vector, partitionId) {
|
|
3252
|
+
validateDimension(vector);
|
|
3253
|
+
const hold = state.cold;
|
|
3254
|
+
if (hold === null) {
|
|
3255
|
+
applyInsert(docId, vector, partitionId);
|
|
3256
|
+
return;
|
|
3257
|
+
}
|
|
3258
|
+
noteWrite(state);
|
|
3259
|
+
coldInsert(hold, docId, vector, partitionId);
|
|
3260
|
+
const held = vector.slice();
|
|
3261
|
+
hold.writes.push(() => applyInsert(docId, held, partitionId));
|
|
3262
|
+
},
|
|
3263
|
+
remove(docId) {
|
|
3264
|
+
const hold = state.cold;
|
|
3265
|
+
if (hold === null) {
|
|
3266
|
+
applyRemove(docId);
|
|
3267
|
+
return;
|
|
3268
|
+
}
|
|
3269
|
+
if (!coldRemove(hold, docId)) return;
|
|
3270
|
+
noteWrite(state);
|
|
3271
|
+
hold.writes.push(() => applyRemove(docId));
|
|
3272
|
+
}
|
|
3273
|
+
};
|
|
3274
|
+
}
|
|
3275
|
+
|
|
2970
3276
|
// src/vector/vector-index/index.ts
|
|
2971
3277
|
function defaultQuantizationFor(dimension) {
|
|
2972
3278
|
if (dimension >= OSQ4_MIN_DIMENSION) return "osq4";
|
|
@@ -3037,76 +3343,26 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
3037
3343
|
sharedHandles: /* @__PURE__ */ new Map(),
|
|
3038
3344
|
sharedLayoutRevision: 0,
|
|
3039
3345
|
sharing: Promise.resolve(),
|
|
3040
|
-
releaseToFilesInFlight: Promise.resolve()
|
|
3346
|
+
releaseToFilesInFlight: Promise.resolve(),
|
|
3347
|
+
cold: null
|
|
3041
3348
|
};
|
|
3042
|
-
|
|
3043
|
-
if (vector.length !== state.dimension) {
|
|
3044
|
-
throw new NarsilError(
|
|
3045
|
-
ErrorCodes.VECTOR_DIMENSION_MISMATCH,
|
|
3046
|
-
`Vector dimension mismatch: expected ${state.dimension}, got ${vector.length}`,
|
|
3047
|
-
{ expected: state.dimension, received: vector.length }
|
|
3048
|
-
);
|
|
3049
|
-
}
|
|
3050
|
-
}
|
|
3051
|
-
function heldVectorOrUndefinedWhereItsFileFailsToRead(ordinal) {
|
|
3052
|
-
try {
|
|
3053
|
-
return state.store.entryForOrdinal(ordinal)?.vector;
|
|
3054
|
-
} catch {
|
|
3055
|
-
return void 0;
|
|
3056
|
-
}
|
|
3057
|
-
}
|
|
3058
|
-
function holdsThisVectorAlready(docId, ordinal, vector) {
|
|
3059
|
-
if (state.tombstones.has(docId)) return false;
|
|
3060
|
-
const held = heldVectorOrUndefinedWhereItsFileFailsToRead(ordinal);
|
|
3061
|
-
if (held === void 0) return false;
|
|
3062
|
-
for (let i = 0; i < vector.length; i++) if (held[i] !== vector[i]) return false;
|
|
3063
|
-
return true;
|
|
3064
|
-
}
|
|
3065
|
-
function insert(docId, vector, partitionId) {
|
|
3066
|
-
validateDimension(vector);
|
|
3067
|
-
handOverBuildTurns(state);
|
|
3068
|
-
const previous = state.store.getOrdinal(docId);
|
|
3069
|
-
if (previous !== void 0 && holdsThisVectorAlready(docId, previous, vector)) {
|
|
3070
|
-
if (partitionId !== void 0 && state.store.partitionOfOrdinal(previous) !== partitionId) {
|
|
3071
|
-
noteWrite(state);
|
|
3072
|
-
state.store.setPartition(docId, partitionId);
|
|
3073
|
-
}
|
|
3074
|
-
return;
|
|
3075
|
-
}
|
|
3076
|
-
noteWrite(state);
|
|
3077
|
-
state.tombstones.delete(docId);
|
|
3078
|
-
const graphHoldsPrevious = previous !== void 0 && ((state.hnsw?.holdsNode(previous) ?? false) || (state.freshGraph?.holdsNode(previous) ?? false));
|
|
3079
|
-
state.store.insert(docId, vector, partitionId, graphHoldsPrevious);
|
|
3080
|
-
if (previous !== void 0) {
|
|
3081
|
-
state.hnsw?.markTombstoneOrdinal(previous);
|
|
3082
|
-
state.freshGraph?.markTombstoneOrdinal(previous);
|
|
3083
|
-
if (!graphHoldsPrevious) state.osq?.removeOrdinal(previous);
|
|
3084
|
-
}
|
|
3085
|
-
state.buffer.add(docId);
|
|
3086
|
-
state.pendingLocations.delete(docId);
|
|
3087
|
-
if (!threadsHoldCurrentLayout(state)) scheduleWorkerCopyLoad(state);
|
|
3088
|
-
}
|
|
3089
|
-
function remove(docId) {
|
|
3090
|
-
handOverBuildTurns(state);
|
|
3091
|
-
if (!state.store.has(docId)) return;
|
|
3092
|
-
noteWrite(state);
|
|
3093
|
-
state.tombstones.add(docId);
|
|
3094
|
-
state.buffer.delete(docId);
|
|
3095
|
-
state.pendingLocations.delete(docId);
|
|
3096
|
-
if (state.hnsw) {
|
|
3097
|
-
state.hnsw.markTombstone(docId);
|
|
3098
|
-
}
|
|
3099
|
-
}
|
|
3349
|
+
const { insert, remove } = createVectorWrites(state);
|
|
3100
3350
|
function getVector(docId) {
|
|
3351
|
+
if (state.cold !== null) return coldVectorOf(state.cold, docId);
|
|
3101
3352
|
if (state.tombstones.has(docId)) return null;
|
|
3102
3353
|
const entry = state.store.get(docId);
|
|
3103
3354
|
if (!entry) return null;
|
|
3104
3355
|
return new Float32Array(entry.vector);
|
|
3105
3356
|
}
|
|
3106
3357
|
function has(docId) {
|
|
3358
|
+
if (state.cold !== null) return coldHas(state.cold, docId);
|
|
3107
3359
|
return state.store.has(docId) && !state.tombstones.has(docId);
|
|
3108
3360
|
}
|
|
3361
|
+
async function whenLoaded() {
|
|
3362
|
+
while (state.cold !== null) await state.cold.loaded;
|
|
3363
|
+
}
|
|
3109
3364
|
async function searchParallel(query, k, options) {
|
|
3365
|
+
if (state.cold !== null) return coldSearchConcurrently(state, state.cold, query, k, options);
|
|
3110
3366
|
handOverBuildTurns(state);
|
|
3111
3367
|
const confined = options.filterSubsets !== void 0 || options.filterPartitions !== void 0;
|
|
3112
3368
|
let filter = filterForOptions(state, options);
|
|
@@ -3153,44 +3409,73 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
3153
3409
|
get storage() {
|
|
3154
3410
|
return state.storage;
|
|
3155
3411
|
},
|
|
3412
|
+
get metric() {
|
|
3413
|
+
return state.metric;
|
|
3414
|
+
},
|
|
3156
3415
|
insert,
|
|
3157
3416
|
remove,
|
|
3158
|
-
partitionsKnown: () => state.store.partitionsKnown,
|
|
3159
|
-
assignPartitions: (resolve) =>
|
|
3160
|
-
|
|
3161
|
-
|
|
3162
|
-
|
|
3163
|
-
|
|
3164
|
-
|
|
3417
|
+
partitionsKnown: () => state.cold !== null || state.store.partitionsKnown,
|
|
3418
|
+
assignPartitions: (resolve) => {
|
|
3419
|
+
if (state.cold === null) assignStorePartitions(state, resolve);
|
|
3420
|
+
},
|
|
3421
|
+
scheduleBuild: () => {
|
|
3422
|
+
if (state.cold === null) scheduleBuild(state);
|
|
3423
|
+
},
|
|
3424
|
+
awaitPendingBuild: async () => {
|
|
3425
|
+
await whenLoaded();
|
|
3426
|
+
await awaitBuilds(state);
|
|
3427
|
+
},
|
|
3428
|
+
completeGraph: async () => {
|
|
3429
|
+
await whenLoaded();
|
|
3430
|
+
await completeGraph(state);
|
|
3431
|
+
},
|
|
3432
|
+
dispose: () => {
|
|
3433
|
+
cancelColdHold(state);
|
|
3434
|
+
disposeIndex(state);
|
|
3435
|
+
},
|
|
3436
|
+
search: (query, k, options) => state.cold === null ? search(state, query, k, options) : coldSearch(state, state.cold, query, k, options),
|
|
3165
3437
|
searchParallel,
|
|
3166
|
-
refreshWorkerCopies: () =>
|
|
3438
|
+
refreshWorkerCopies: () => {
|
|
3439
|
+
if (state.cold === null) refreshWorkerCopies(state);
|
|
3440
|
+
},
|
|
3167
3441
|
getVector,
|
|
3168
3442
|
has,
|
|
3169
3443
|
compact: () => {
|
|
3444
|
+
if (state.cold !== null) return;
|
|
3170
3445
|
noteWrite(state);
|
|
3171
3446
|
compact(state);
|
|
3172
3447
|
},
|
|
3173
3448
|
optimize: async () => {
|
|
3449
|
+
await whenLoaded();
|
|
3174
3450
|
noteWrite(state);
|
|
3175
3451
|
await optimize(state);
|
|
3176
3452
|
},
|
|
3177
|
-
maintenanceStatus: () => maintenanceStatus(state),
|
|
3178
|
-
estimateMemoryBytes: () => estimateMemoryBytes(state),
|
|
3453
|
+
maintenanceStatus: () => ({ ...maintenanceStatus(state), loadingFromFiles: state.cold !== null }),
|
|
3454
|
+
estimateMemoryBytes: () => estimateMemoryBytes(state) + (state.cold?.field.residentBytes ?? 0),
|
|
3179
3455
|
growthBytesFor: (vectors) => state.store.growthBytesFor(vectors),
|
|
3180
3456
|
deserialize: (parts) => {
|
|
3457
|
+
cancelColdHold(state);
|
|
3181
3458
|
invalidateWorkerCopies(state);
|
|
3182
3459
|
deserialize(state, parts);
|
|
3183
3460
|
},
|
|
3184
3461
|
planCheckpoint: (listedKeys, rewritesGraph) => planCheckpoint(state, listedKeys, rewritesGraph),
|
|
3185
3462
|
recordCheckpoint: (plan, written) => recordCheckpoint(state, plan, written),
|
|
3186
3463
|
restoreCheckpoint: (shape) => {
|
|
3464
|
+
cancelColdHold(state);
|
|
3187
3465
|
invalidateWorkerCopies(state);
|
|
3188
3466
|
return beginCheckpointRestore(state, shape);
|
|
3189
3467
|
},
|
|
3190
|
-
|
|
3191
|
-
|
|
3468
|
+
openColdField: (field, load) => openColdField(state, field, load),
|
|
3469
|
+
startShuffledRebuildWhenDue: () => {
|
|
3470
|
+
if (state.cold === null) startShuffledRebuild(state);
|
|
3471
|
+
},
|
|
3472
|
+
vectorFilesInUse: () => {
|
|
3473
|
+
const stored = state.store.dimension === 0 ? [] : state.store.handles.vectorFiles.filter((path) => path !== "");
|
|
3474
|
+
return state.cold === null ? stored : [...stored, ...state.cold.field.filePaths()];
|
|
3475
|
+
},
|
|
3192
3476
|
adoptDiskLayout: (layout) => adoptDiskLayout(state, layout),
|
|
3193
3477
|
releaseVectorFiles: async () => {
|
|
3478
|
+
await whenLoaded();
|
|
3194
3479
|
await withdrawWorkerCopies(state);
|
|
3195
3480
|
state.store.closeFiles();
|
|
3196
3481
|
},
|
|
@@ -3200,7 +3485,10 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
3200
3485
|
state.pendingLocations.set(docId, { ...location, path: rewrite(location.path) });
|
|
3201
3486
|
}
|
|
3202
3487
|
},
|
|
3203
|
-
holdWorkerCopies: () =>
|
|
3488
|
+
holdWorkerCopies: async () => {
|
|
3489
|
+
await whenLoaded();
|
|
3490
|
+
await holdWorkerCopies(state);
|
|
3491
|
+
},
|
|
3204
3492
|
resumeWorkerCopies: () => {
|
|
3205
3493
|
state.workerCopiesHeld = false;
|
|
3206
3494
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createBitSet, bitsetIsEmpty, GrowableUint32, PHRASE_ROWS_START, bitsetHas, bitsetSet, PHRASE_SORT_RADIX_BITS, PHRASE_SORT_CLOCK_ROWS } from './chunk-26A5JFT2.mjs';
|
|
2
|
-
import { NarsilError, ErrorCodes, MAX_QUERY_PHRASES, MAX_PHRASE_WORDS } from './chunk-
|
|
2
|
+
import { NarsilError, ErrorCodes, MAX_QUERY_PHRASES, MAX_PHRASE_WORDS } from './chunk-35B42UFM.mjs';
|
|
3
3
|
|
|
4
4
|
// src/core/partition/phrase-matching.ts
|
|
5
5
|
var UINT32_BITS = 32;
|