@delali/narsil 0.3.1-next.140 → 0.3.1-next.143
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{analysis-change-JN23AZQP.mjs → analysis-change-HQUT2Z3I.mjs} +13 -13
- package/dist/{analysis-change-entry-5JSIMEKG.mjs → analysis-change-entry-TZTG4AZB.mjs} +1 -1
- package/dist/browser/{analysis-change-57MIRJWR.mjs → analysis-change-JPV775LR.mjs} +407 -6
- package/dist/browser/{analysis-change-entry-ICGRR3IN.mjs → analysis-change-entry-7HR4LWL3.mjs} +1 -1
- package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
- package/dist/browser/{chunk-ZAIZQOTM.mjs → chunk-5DB2VDGI.mjs} +951 -1134
- package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
- package/dist/browser/chunk-CCOAOO73.mjs +86 -0
- package/dist/browser/{chunk-3EYDURFD.mjs → chunk-HKX2G6WH.mjs} +1615 -2266
- package/dist/browser/{chunk-CGVBGHBX.mjs → chunk-KTOK275N.mjs} +531 -830
- package/dist/browser/{chunk-5LYWZJE2.mjs → chunk-LVY2LI35.mjs} +1 -1
- package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
- package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
- package/dist/browser/{chunk-67KSBUYR.mjs → chunk-R4WUDZQV.mjs} +8 -5
- package/dist/browser/{chunk-6BSB4N6B.mjs → chunk-TCLEOZ4P.mjs} +2 -2
- package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
- package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
- package/dist/browser/{chunk-E5SFTOQR.mjs → chunk-XNWPP5NS.mjs} +2 -2
- package/dist/browser/{chunk-QHNNULPW.mjs → chunk-ZEV26AQ6.mjs} +5 -0
- package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
- package/dist/browser/{durability-wiring-MTIAN6VT.mjs → durability-wiring-SNOWIT6Z.mjs} +231 -51
- package/dist/browser/{import-Q7MXD6BS.mjs → import-TLB2A5LJ.mjs} +6 -5
- package/dist/browser/module-BUFNZMEU.mjs +12 -0
- package/dist/browser/{module-5J3PEUEB.mjs → module-BVP4TKZU.mjs} +395 -108
- package/dist/{restore-queue-RYSRRFS4.mjs → browser/restore-queue-ONYNYUDE.mjs} +2 -2
- package/dist/browser/{snapshot-YBJ5MF2M.mjs → snapshot-QXBKHJAG.mjs} +108 -25
- package/dist/browser/{write-RTWI5MXL.mjs → write-WHFBDWQS.mjs} +63 -91
- package/dist/{capabilities-DXAmysqJ.d.ts → capabilities-BCdUwK_L.d.ts} +2 -2
- package/dist/{chunk-HMDT3I63.mjs → chunk-46NZOPHS.mjs} +2 -2
- package/dist/{chunk-XXKWEG53.mjs → chunk-5WYVUPGD.mjs} +1 -1
- package/dist/chunk-6SNBQESJ.mjs +214 -0
- package/dist/chunk-7AUASRE7.mjs +653 -0
- package/dist/{chunk-4GCN34JV.mjs → chunk-AST3SHFA.mjs} +26 -24
- package/dist/chunk-BOJZGUS6.mjs +177 -0
- package/dist/{chunk-IP4HJOGC.mjs → chunk-BRUUBXCB.mjs} +4 -3
- package/dist/{chunk-J3EHATEZ.mjs → chunk-DQNKVOJ2.mjs} +63 -63
- package/dist/chunk-FOCV4CBZ.mjs +87 -0
- package/dist/{chunk-FFVLPCLH.mjs → chunk-GLZHNSGB.mjs} +1 -1
- package/dist/chunk-GWOS5DYC.mjs +2637 -0
- package/dist/{chunk-W2XDK66Z.mjs → chunk-IO5F7ZBE.mjs} +112 -438
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/{chunk-LLDFIKGP.mjs → chunk-KEIYVZUX.mjs} +4 -4
- package/dist/{chunk-UBXJQJII.mjs → chunk-KGOXHLLZ.mjs} +5 -1
- package/dist/{chunk-FKARMUWH.mjs → chunk-KZL72VTC.mjs} +68 -16
- package/dist/{chunk-TCWQJL7O.mjs → chunk-L76W3IZJ.mjs} +5 -0
- package/dist/{chunk-CPCGEIOM.mjs → chunk-LA4WNDHL.mjs} +83 -799
- package/dist/{chunk-PNNVTKMT.mjs → chunk-NJJCPFXN.mjs} +2 -2
- package/dist/chunk-NN33CZRE.mjs +86 -0
- package/dist/chunk-NRUXBYPH.mjs +121 -0
- package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
- package/dist/{chunk-VSUYTPME.mjs → chunk-RHKRTYGO.mjs} +13 -4
- package/dist/{chunk-UEV53BBO.mjs → chunk-SRJBFG2T.mjs} +406 -5
- package/dist/{chunk-K3AQOZVM.mjs → chunk-TNBIK2LQ.mjs} +1638 -1644
- package/dist/chunk-UUZCQQG2.mjs +209 -0
- package/dist/{chunk-XMQNOD5V.mjs → chunk-VEFN6X6Q.mjs} +2 -2
- package/dist/{chunk-LK3MOTXF.mjs → chunk-VJEGOELZ.mjs} +8 -5
- package/dist/chunk-W2COJ5AH.mjs +148 -0
- package/dist/{chunk-IBATGS3J.mjs → chunk-WJGO5PGV.mjs} +241 -124
- package/dist/{chunk-CCCGEPJM.mjs → chunk-XZ3SRNDJ.mjs} +24 -63
- package/dist/{chunk-MTQTX4BP.mjs → chunk-YHUTF6EA.mjs} +400 -92
- package/dist/{chunk-B5I3IL4H.mjs → chunk-YLNSJQLX.mjs} +4 -4
- package/dist/{chunk-6VHWXAOI.mjs → chunk-Z4HFMYNX.mjs} +29 -24
- package/dist/{chunk-U4X4AWKL.mjs → chunk-ZPWS2SKJ.mjs} +293 -270
- package/dist/{client-DApBeLbk.d.ts → client-B12YFALP.d.ts} +3 -3
- package/dist/client.d.ts +7 -7
- package/dist/cold-module-4X7WJCFM.mjs +783 -0
- package/dist/{config-B0H-vp4U.d.ts → config-BhZxIeb1.d.ts} +6 -1
- package/dist/{config-Brysl9NO.d.ts → config-CIG5yDyg.d.ts} +2 -2
- package/dist/{direct-executor-J5UDLPL3.mjs → direct-executor-7XOQ7VQP.mjs} +5 -5
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +123 -42
- package/dist/{durability-wiring-AYOTAPNQ.mjs → durability-wiring-324D2DDD.mjs} +213 -40
- package/dist/{engine-CuLMbH1-.d.ts → engine-CtzbHOmG.d.ts} +2 -2
- package/dist/{import-QRZAQ5RH.mjs → import-OOBUW3RR.mjs} +14 -13
- package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +11 -11
- package/dist/{long-query-thread-U7K5A72G.mjs → long-query-thread-4SW7RWDE.mjs} +8 -8
- package/dist/{module-L2OZO5PQ.mjs → module-DXSJ5LT2.mjs} +374 -83
- package/dist/module-IYPLWHGC.mjs +48 -0
- package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
- package/dist/partial-words.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +5 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
- package/dist/react.d.ts +9 -9
- package/dist/{browser/restore-queue-DISFECE4.mjs → restore-queue-FEYAC4SB.mjs} +2 -2
- package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +8 -7
- package/dist/{snapshot-BTRZ3MZK.mjs → snapshot-CGM65VCD.mjs} +60 -25
- package/dist/{tasks-DjGjK70Y.d.ts → tasks-CjRJRGTb.d.ts} +6 -4
- package/dist/{thread-USB3GOPY.mjs → thread-KGNAIKPD.mjs} +8 -8
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +3 -2
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-GHTA56OW.mjs → write-2DZPEQL2.mjs} +70 -98
- package/openapi.json +13 -0
- package/package.json +12 -8
- package/dist/browser/chunk-XVMOQQ34.mjs +0 -384
- package/dist/chunk-BDIDUW2B.mjs +0 -387
- package/dist/browser/{chunk-4MFH226Q.mjs → chunk-D33A5VMU.mjs} +15 -15
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
import { VECTOR_WORKER_COPIES_ALLOWED, openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, osqBitsOf, noBuildHistory, createOsqQuantizer, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, ordinalFilterValues, handOverBuildTurns, threadsHoldCurrentLayout, filterForOptions, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, OSQ4_MIN_DIMENSION, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, WORKER_COPY_MIN_VECTORS, allLiveDocIds, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, BLOCK_MAX_BYTES, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, growBufferTo, SHUFFLED_REBUILD_MEMORY_RETRY_MS, recalibrateFromStore, signatureOf, nativeStoreScratchBytes, noteFallback, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, QUERY_STAGED, grownByteLength, createVectorBlock, ensureBlockSlots, createHNSWIndex, adoptGraph, topLayerInParts, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-3EYDURFD.mjs';
|
|
2
|
-
export { releaseLocksHeldBy } from './chunk-3EYDURFD.mjs';
|
|
3
|
-
import './chunk-OIZS5KIX.mjs';
|
|
4
1
|
import { spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
2
|
+
import { VECTOR_WORKER_COPIES_ALLOWED, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, noBuildHistory, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, handOverBuildTurns, filterForOptions, OSQ4_MIN_DIMENSION, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, allLiveDocIds, threadsHoldCurrentLayout, WORKER_COPY_MIN_VECTORS, SHUFFLED_REBUILD_MEMORY_RETRY_MS, recalibrateFromStore, signatureOf, adoptGraph, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-6RHTAVZX.mjs';
|
|
3
|
+
import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, osqBitsOf, createOsqQuantizer, ordinalFilterValues, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, BLOCK_MAX_BYTES, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, growBufferTo, nativeStoreScratchBytes, noteFallback, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, QUERY_STAGED, grownByteLength, createVectorBlock, ensureBlockSlots, createHNSWIndex, topLayerInParts, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy } from './chunk-HKX2G6WH.mjs';
|
|
4
|
+
export { releaseLocksHeldBy } from './chunk-HKX2G6WH.mjs';
|
|
5
|
+
import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-LVY2LI35.mjs';
|
|
6
|
+
import { createVectorFileReader } from './chunk-3ZBJYAS4.mjs';
|
|
7
|
+
import './chunk-OIZS5KIX.mjs';
|
|
8
|
+
import { MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, compareCodePoints, UNPLACED_FLAG_CHUNK_ORDINALS, MAX_VECTOR_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, resolveWorkerCount, resolveWorkerEntry, detectRuntime, noteThreadSlotLive, noteThreadSlotGone, WORKER_YOUNG_GENERATION_MB, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS } from './chunk-D33A5VMU.mjs';
|
|
7
9
|
import './chunk-MZI3TEOW.mjs';
|
|
8
10
|
import { NarsilError, ErrorCodes, isNarsilError } from './chunk-35B42UFM.mjs';
|
|
9
11
|
|
|
@@ -352,21 +354,6 @@ function createSharedVectorSearcher(options) {
|
|
|
352
354
|
};
|
|
353
355
|
}
|
|
354
356
|
|
|
355
|
-
// src/vector/vector-store/vector-file.browser.ts
|
|
356
|
-
function filesystemUnavailable() {
|
|
357
|
-
throw new NarsilError(
|
|
358
|
-
ErrorCodes.CONFIG_INVALID,
|
|
359
|
-
"Reading vectors from a checkpoint file requires a Node.js runtime; keep the field in memory in the browser"
|
|
360
|
-
);
|
|
361
|
-
}
|
|
362
|
-
function createVectorFileReader() {
|
|
363
|
-
return {
|
|
364
|
-
readInto: () => filesystemUnavailable(),
|
|
365
|
-
retainOnly: () => void 0,
|
|
366
|
-
close: () => void 0
|
|
367
|
-
};
|
|
368
|
-
}
|
|
369
|
-
|
|
370
357
|
// src/vector/vector-store/hot-block-distance.ts
|
|
371
358
|
function jsDistance(a, b, magA, magB, metric) {
|
|
372
359
|
switch (metric) {
|
|
@@ -378,6 +365,11 @@ function jsDistance(a, b, magA, magB, metric) {
|
|
|
378
365
|
return euclideanDistance(a, b);
|
|
379
366
|
}
|
|
380
367
|
}
|
|
368
|
+
function distanceFromQueryOf(source, coldScratch, query, queryMagnitude, ordinal, metric) {
|
|
369
|
+
if (!source.holdsOrdinal(ordinal)) return Number.POSITIVE_INFINITY;
|
|
370
|
+
const vector = source.isCold(ordinal) && source.readColdInto(ordinal, coldScratch) ? coldScratch : source.vectorAt(ordinal);
|
|
371
|
+
return jsDistance(query, vector, queryMagnitude, source.magnitudeAt(ordinal), metric);
|
|
372
|
+
}
|
|
381
373
|
function inMemory(reads, ordinal) {
|
|
382
374
|
return ordinal >= 0 && ordinal < reads.limit && reads.diskFile[ordinal] === IN_MEMORY;
|
|
383
375
|
}
|
|
@@ -474,9 +466,7 @@ function openSharedVectorStore(initial, threadSlot) {
|
|
|
474
466
|
}
|
|
475
467
|
return diskFile[ordinal];
|
|
476
468
|
}
|
|
477
|
-
|
|
478
|
-
return fileOf(ordinal) !== IN_MEMORY;
|
|
479
|
-
}
|
|
469
|
+
const isCold = (ordinal) => fileOf(ordinal) !== IN_MEMORY;
|
|
480
470
|
function holdState(ordinal) {
|
|
481
471
|
if (ordinal < 0) return NOT_HELD;
|
|
482
472
|
if (ordinal >= present.length) {
|
|
@@ -488,9 +478,7 @@ function openSharedVectorStore(initial, threadSlot) {
|
|
|
488
478
|
const index = Math.floor(ordinal / capacity);
|
|
489
479
|
return index < handles.blocks.length && handles.blocks[index] !== null ? HELD_HOT : NOT_HELD;
|
|
490
480
|
}
|
|
491
|
-
|
|
492
|
-
return holdState(ordinal) !== NOT_HELD;
|
|
493
|
-
}
|
|
481
|
+
const holds = (ordinal) => holdState(ordinal) !== NOT_HELD;
|
|
494
482
|
function readFromFile(file, offset, target) {
|
|
495
483
|
const path = handles.vectorFiles[file];
|
|
496
484
|
if (path === void 0 || path.length === 0) return false;
|
|
@@ -698,6 +686,7 @@ function openSharedVectorStore(initial, threadSlot) {
|
|
|
698
686
|
}
|
|
699
687
|
return jsDistance(currentQuery, vectorIn(block, local), prepared.magnitude, magnitudes[ordinal], metric);
|
|
700
688
|
},
|
|
689
|
+
distanceFromQuery: (query, queryMagnitude, ordinal, metric) => distanceFromQueryOf(view, coldScratchB, query, queryMagnitude, ordinal, metric),
|
|
701
690
|
hotBlockReads,
|
|
702
691
|
queryDistance: (prepared, metric) => queryDistanceOf(view, prepared, metric),
|
|
703
692
|
ordinalDistance: (from, metric) => ordinalDistanceOf(view, from, metric),
|
|
@@ -841,6 +830,11 @@ function createOrdinalReads(openStore, readable) {
|
|
|
841
830
|
if (open === null || !readable(ordinal)) return Number.POSITIVE_INFINITY;
|
|
842
831
|
return open.view.distanceFromArena(prepared, ordinal, metric);
|
|
843
832
|
},
|
|
833
|
+
distanceFromQuery(query, queryMagnitude, ordinal, metric) {
|
|
834
|
+
const open = openStore();
|
|
835
|
+
if (open === null || !readable(ordinal)) return Number.POSITIVE_INFINITY;
|
|
836
|
+
return open.view.distanceFromQuery(query, queryMagnitude, ordinal, metric);
|
|
837
|
+
},
|
|
844
838
|
queryDistance(prepared, metric) {
|
|
845
839
|
const open = openStore();
|
|
846
840
|
if (open === null) return () => Number.POSITIVE_INFINITY;
|
|
@@ -1017,13 +1011,12 @@ function blockFullyCold(store, blockIndex, documents, replacedOrdinals) {
|
|
|
1017
1011
|
}
|
|
1018
1012
|
return true;
|
|
1019
1013
|
}
|
|
1020
|
-
function fileHoldsSameBytes(store, ordinal, location) {
|
|
1021
|
-
const fromFile = new Float32Array(store.handles.dimension);
|
|
1014
|
+
function fileHoldsSameBytes(store, ordinal, location, fromFile) {
|
|
1022
1015
|
if (!store.view.readFromFile(location.fileIndex, location.offset, fromFile)) return false;
|
|
1023
1016
|
const held = store.view.vectorAt(ordinal);
|
|
1024
|
-
const
|
|
1025
|
-
const
|
|
1026
|
-
for (let i = 0; i <
|
|
1017
|
+
const fileWords = new Uint32Array(fromFile.buffer, fromFile.byteOffset, fromFile.length);
|
|
1018
|
+
const heldWords = new Uint32Array(held.buffer, held.byteOffset, held.length);
|
|
1019
|
+
for (let i = 0; i < fileWords.length; i++) if (fileWords[i] !== heldWords[i]) return false;
|
|
1027
1020
|
return true;
|
|
1028
1021
|
}
|
|
1029
1022
|
function forgetUnreferencedFiles(store, documents, replacedOrdinals) {
|
|
@@ -1074,6 +1067,7 @@ function createVectorStore(options) {
|
|
|
1074
1067
|
const replacedOrdinals = /* @__PURE__ */ new Set();
|
|
1075
1068
|
let unknownPartitions = 0;
|
|
1076
1069
|
let liveCount = 0;
|
|
1070
|
+
let readBackVector = null;
|
|
1077
1071
|
let open = options?.dimension === void 0 ? null : openHandles(options.dimension, codeBits, blockBytes);
|
|
1078
1072
|
function forgetDocuments() {
|
|
1079
1073
|
docToOrd.clear();
|
|
@@ -1218,7 +1212,10 @@ function createVectorStore(options) {
|
|
|
1218
1212
|
releaseToFile(ordinal, location) {
|
|
1219
1213
|
const store = requireOpen();
|
|
1220
1214
|
if (ordToDoc[ordinal] === void 0) return false;
|
|
1221
|
-
if (store.diskFile[ordinal] === IN_MEMORY
|
|
1215
|
+
if (store.diskFile[ordinal] === IN_MEMORY) {
|
|
1216
|
+
readBackVector ??= new Float32Array(store.handles.dimension);
|
|
1217
|
+
if (!fileHoldsSameBytes(store, ordinal, location, readBackVector)) return false;
|
|
1218
|
+
}
|
|
1222
1219
|
Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
|
|
1223
1220
|
store.diskOffset[ordinal] = location.offset;
|
|
1224
1221
|
store.diskFile[ordinal] = location.fileIndex;
|
|
@@ -2600,19 +2597,25 @@ function beginCheckpointRestore(state, shape) {
|
|
|
2600
2597
|
const dead = isDead(file.dead, i);
|
|
2601
2598
|
if (dead && !graphHoldsNodeFor(firstNumber + i)) continue;
|
|
2602
2599
|
const docId = payload.docIds[i];
|
|
2603
|
-
const vector = vectors.subarray(i * dimension, (i + 1) * dimension);
|
|
2604
2600
|
const offset = onDisk ? location.vectorsOffset + i * dimension * 4 : 0;
|
|
2601
|
+
const vectorHere = () => file.byParts === void 0 ? vectors.subarray(i * dimension, (i + 1) * dimension) : file.byParts.vectorAt(i);
|
|
2605
2602
|
if (dead) {
|
|
2606
|
-
ordinals[i] = state.store.insertReplaced(
|
|
2603
|
+
ordinals[i] = state.store.insertReplaced(
|
|
2604
|
+
docId,
|
|
2605
|
+
vectorHere(),
|
|
2606
|
+
fileIndex === null ? void 0 : { fileIndex, offset }
|
|
2607
|
+
);
|
|
2607
2608
|
tombstoned.add(ordinals[i]);
|
|
2608
2609
|
continue;
|
|
2609
2610
|
}
|
|
2610
2611
|
const replaced = state.store.getOrdinal(docId);
|
|
2611
2612
|
if (replaced !== void 0) state.osq?.removeOrdinal(replaced);
|
|
2612
2613
|
if (fileIndex !== null) {
|
|
2613
|
-
|
|
2614
|
+
const known = file.byParts?.magnitudes[i];
|
|
2615
|
+
ordinals[i] = state.store.insertCold(docId, known ?? magnitude(vectorHere()), { fileIndex, offset });
|
|
2614
2616
|
continue;
|
|
2615
2617
|
}
|
|
2618
|
+
const vector = vectorHere();
|
|
2616
2619
|
ordinals[i] = state.store.insert(docId, vector);
|
|
2617
2620
|
if (onDisk) state.pendingLocations.set(docId, { path: location.path, offset });
|
|
2618
2621
|
}
|
|
@@ -2654,6 +2657,223 @@ function beginCheckpointRestore(state, shape) {
|
|
|
2654
2657
|
return { addFile, finish };
|
|
2655
2658
|
}
|
|
2656
2659
|
|
|
2660
|
+
// src/vector/vector-index/cold.ts
|
|
2661
|
+
function beginColdHold(state, field, loaded, searchAnswered) {
|
|
2662
|
+
const hold = {
|
|
2663
|
+
field,
|
|
2664
|
+
loaded,
|
|
2665
|
+
searchAnswered,
|
|
2666
|
+
written: /* @__PURE__ */ new Map(),
|
|
2667
|
+
removedNumbers: /* @__PURE__ */ new Set(),
|
|
2668
|
+
removedIds: /* @__PURE__ */ new Set(),
|
|
2669
|
+
writes: [],
|
|
2670
|
+
live: field.liveCount
|
|
2671
|
+
};
|
|
2672
|
+
state.cold = hold;
|
|
2673
|
+
return hold;
|
|
2674
|
+
}
|
|
2675
|
+
function endColdHold(state) {
|
|
2676
|
+
const hold = state.cold;
|
|
2677
|
+
if (hold === null) return;
|
|
2678
|
+
state.cold = null;
|
|
2679
|
+
for (const write of hold.writes) write();
|
|
2680
|
+
hold.field.close();
|
|
2681
|
+
}
|
|
2682
|
+
function cancelColdHold(state) {
|
|
2683
|
+
const hold = state.cold;
|
|
2684
|
+
if (hold === null) return;
|
|
2685
|
+
state.cold = null;
|
|
2686
|
+
hold.field.close();
|
|
2687
|
+
}
|
|
2688
|
+
function openColdField(state, field, load) {
|
|
2689
|
+
cancelColdHold(state);
|
|
2690
|
+
let finished = () => void 0;
|
|
2691
|
+
let failed = () => void 0;
|
|
2692
|
+
const loaded = new Promise((resolve, reject) => {
|
|
2693
|
+
finished = resolve;
|
|
2694
|
+
failed = reject;
|
|
2695
|
+
});
|
|
2696
|
+
let searchAnswered = () => void 0;
|
|
2697
|
+
const firstSearchAnswered = new Promise((resolve) => {
|
|
2698
|
+
searchAnswered = resolve;
|
|
2699
|
+
});
|
|
2700
|
+
const hold = beginColdHold(state, field, loaded, searchAnswered);
|
|
2701
|
+
void load(
|
|
2702
|
+
(shape) => beginCheckpointRestore(state, shape),
|
|
2703
|
+
() => state.cold === hold,
|
|
2704
|
+
firstSearchAnswered
|
|
2705
|
+
).then(
|
|
2706
|
+
() => {
|
|
2707
|
+
if (state.cold === hold) endColdHold(state);
|
|
2708
|
+
finished();
|
|
2709
|
+
},
|
|
2710
|
+
(error) => failed(error)
|
|
2711
|
+
);
|
|
2712
|
+
loaded.catch(() => void 0);
|
|
2713
|
+
}
|
|
2714
|
+
function retireOlderCopy(hold, docId) {
|
|
2715
|
+
if (hold.written.delete(docId)) return true;
|
|
2716
|
+
if (hold.removedIds.has(docId)) return false;
|
|
2717
|
+
const number = hold.field.numberOf(docId);
|
|
2718
|
+
if (number === void 0) return false;
|
|
2719
|
+
hold.removedIds.add(docId);
|
|
2720
|
+
hold.removedNumbers.add(number);
|
|
2721
|
+
return true;
|
|
2722
|
+
}
|
|
2723
|
+
function coldInsert(hold, docId, vector, partitionId) {
|
|
2724
|
+
const replaced = retireOlderCopy(hold, docId);
|
|
2725
|
+
if (!replaced) hold.live += 1;
|
|
2726
|
+
hold.written.set(docId, { vector: vector.slice(), partitionId });
|
|
2727
|
+
return replaced;
|
|
2728
|
+
}
|
|
2729
|
+
function coldRemove(hold, docId) {
|
|
2730
|
+
const removed = retireOlderCopy(hold, docId);
|
|
2731
|
+
if (removed) hold.live -= 1;
|
|
2732
|
+
return removed;
|
|
2733
|
+
}
|
|
2734
|
+
function coldHas(hold, docId) {
|
|
2735
|
+
if (hold.written.has(docId)) return true;
|
|
2736
|
+
if (hold.removedIds.has(docId)) return false;
|
|
2737
|
+
return hold.field.numberOf(docId) !== void 0;
|
|
2738
|
+
}
|
|
2739
|
+
function coldVectorOf(hold, docId) {
|
|
2740
|
+
const written = hold.written.get(docId);
|
|
2741
|
+
if (written !== void 0) return written.vector.slice();
|
|
2742
|
+
if (hold.removedIds.has(docId)) return null;
|
|
2743
|
+
const number = hold.field.numberOf(docId);
|
|
2744
|
+
return number === void 0 ? null : hold.field.vectorOfNumber(number);
|
|
2745
|
+
}
|
|
2746
|
+
function isAdmitted(subsets, docId) {
|
|
2747
|
+
for (const subset of subsets) {
|
|
2748
|
+
const { docStore } = subset.owner;
|
|
2749
|
+
if (docStore.numberingGeneration() !== subset.numberingGeneration) continue;
|
|
2750
|
+
const internalId = docStore.getInternalId(docId);
|
|
2751
|
+
if (internalId === void 0) continue;
|
|
2752
|
+
const word = subset.bits[internalId >>> 5];
|
|
2753
|
+
if (word !== void 0 && (word & 1 << (internalId & 31)) !== 0 && docStore.resolver().isLive(internalId)) {
|
|
2754
|
+
return true;
|
|
2755
|
+
}
|
|
2756
|
+
}
|
|
2757
|
+
return false;
|
|
2758
|
+
}
|
|
2759
|
+
function admissionOf(options, hold) {
|
|
2760
|
+
const { filterSubsets, filterPartitions } = options;
|
|
2761
|
+
if (filterSubsets !== void 0) return (docId) => isAdmitted(filterSubsets, docId);
|
|
2762
|
+
if (filterPartitions !== void 0) {
|
|
2763
|
+
return (docId) => {
|
|
2764
|
+
const partition = hold.field.partitionOf(docId);
|
|
2765
|
+
return partition !== void 0 && filterPartitions.has(partition);
|
|
2766
|
+
};
|
|
2767
|
+
}
|
|
2768
|
+
return null;
|
|
2769
|
+
}
|
|
2770
|
+
function bitCount(word) {
|
|
2771
|
+
let value = word - (word >>> 1 & 1431655765);
|
|
2772
|
+
value = (value & 858993459) + (value >>> 2 & 858993459);
|
|
2773
|
+
return (value + (value >>> 4) & 252645135) * 16843009 >>> 24;
|
|
2774
|
+
}
|
|
2775
|
+
function admittedBySubsets(subsets) {
|
|
2776
|
+
let admitted = 0;
|
|
2777
|
+
for (const subset of subsets) {
|
|
2778
|
+
for (let at = 0; at < subset.bits.length; at++) admitted += bitCount(subset.bits[at]);
|
|
2779
|
+
}
|
|
2780
|
+
return admitted;
|
|
2781
|
+
}
|
|
2782
|
+
function* idsOfSubsets(subsets) {
|
|
2783
|
+
for (const subset of subsets) {
|
|
2784
|
+
const { bits } = subset;
|
|
2785
|
+
const { docStore } = subset.owner;
|
|
2786
|
+
if (docStore.numberingGeneration() !== subset.numberingGeneration) continue;
|
|
2787
|
+
for (let wordIndex = 0; wordIndex < bits.length; wordIndex++) {
|
|
2788
|
+
let word = bits[wordIndex];
|
|
2789
|
+
while (word !== 0) {
|
|
2790
|
+
const internalId = (wordIndex << 5) + (Math.clz32(word & -word) ^ 31);
|
|
2791
|
+
word &= word - 1;
|
|
2792
|
+
if (!docStore.resolver().isLive(internalId)) continue;
|
|
2793
|
+
const docId = docStore.getExternalId(internalId);
|
|
2794
|
+
if (docId !== void 0) yield docId;
|
|
2795
|
+
}
|
|
2796
|
+
}
|
|
2797
|
+
}
|
|
2798
|
+
}
|
|
2799
|
+
function scoreOf(query, qMag, vector, metric) {
|
|
2800
|
+
const vectorMagnitude = metric === "cosine" ? magnitude(vector) : 0;
|
|
2801
|
+
return toScore(toDistance(query, vector, qMag, vectorMagnitude, metric), metric);
|
|
2802
|
+
}
|
|
2803
|
+
function scanWritten(hold, query, k, options, admits) {
|
|
2804
|
+
if (hold.written.size === 0) return [];
|
|
2805
|
+
const qMag = magnitude(query);
|
|
2806
|
+
const heap = createBoundedMaxHeap(highScoreFirst, Math.max(k, 1));
|
|
2807
|
+
for (const [docId, { vector }] of hold.written) {
|
|
2808
|
+
if (admits !== null && !admits(docId)) continue;
|
|
2809
|
+
const score = scoreOf(query, qMag, vector, options.metric);
|
|
2810
|
+
if (score >= options.minSimilarity) heap.push({ docId, score });
|
|
2811
|
+
}
|
|
2812
|
+
return heap.toSortedArray().reverse();
|
|
2813
|
+
}
|
|
2814
|
+
function scanSubsets(hold, query, k, options, subsets) {
|
|
2815
|
+
const qMag = magnitude(query);
|
|
2816
|
+
const heap = createBoundedMaxHeap(highScoreFirst, Math.max(k, 1));
|
|
2817
|
+
let matched = 0;
|
|
2818
|
+
for (const docId of idsOfSubsets(subsets)) {
|
|
2819
|
+
const written = hold.written.get(docId);
|
|
2820
|
+
let score;
|
|
2821
|
+
if (written !== void 0) {
|
|
2822
|
+
score = scoreOf(query, qMag, written.vector, options.metric);
|
|
2823
|
+
} else {
|
|
2824
|
+
if (hold.removedIds.has(docId)) continue;
|
|
2825
|
+
const number = hold.field.numberOf(docId);
|
|
2826
|
+
if (number === void 0) continue;
|
|
2827
|
+
score = scoreOf(query, qMag, hold.field.reusedVectorOfNumber(number), options.metric);
|
|
2828
|
+
}
|
|
2829
|
+
if (score < options.minSimilarity) continue;
|
|
2830
|
+
matched += 1;
|
|
2831
|
+
heap.push({ docId, score });
|
|
2832
|
+
}
|
|
2833
|
+
return { results: heap.toSortedArray().reverse(), matched };
|
|
2834
|
+
}
|
|
2835
|
+
function outcomeWithoutGraphWalk(state, hold, query, k, options) {
|
|
2836
|
+
if (hold.live === 0) return { results: [], matched: 0, matchedExact: true };
|
|
2837
|
+
const subsets = options.filterSubsets;
|
|
2838
|
+
let admitted = hold.live;
|
|
2839
|
+
if (subsets !== void 0) {
|
|
2840
|
+
admitted = admittedBySubsets(subsets);
|
|
2841
|
+
if (admitted === 0) return { results: [], matched: 0, matchedExact: true };
|
|
2842
|
+
if (admitted / hold.live < state.filterThreshold) {
|
|
2843
|
+
const { results, matched } = scanSubsets(hold, query, k, options, subsets);
|
|
2844
|
+
return { results: results.slice(0, Math.max(k, 0)), matched, matchedExact: true };
|
|
2845
|
+
}
|
|
2846
|
+
}
|
|
2847
|
+
if (k <= 0) {
|
|
2848
|
+
return options.minSimilarity === Number.NEGATIVE_INFINITY ? { results: [], matched: admitted, matchedExact: true } : { results: [], matched: 0, matchedExact: false };
|
|
2849
|
+
}
|
|
2850
|
+
return { admits: admissionOf(options, hold), admitted };
|
|
2851
|
+
}
|
|
2852
|
+
function outcomeOfGraphWalk(hold, walk, fromGraph, query, k, options) {
|
|
2853
|
+
const results = mergeResults(fromGraph, scanWritten(hold, query, k, options, walk.admits), k);
|
|
2854
|
+
return options.minSimilarity === Number.NEGATIVE_INFINITY ? { results, matched: walk.admitted, matchedExact: true } : { results, matched: results.length, matchedExact: false };
|
|
2855
|
+
}
|
|
2856
|
+
function coldSearch(state, hold, query, k, options) {
|
|
2857
|
+
const walk = outcomeWithoutGraphWalk(state, hold, query, k, options);
|
|
2858
|
+
if ("results" in walk) {
|
|
2859
|
+
hold.searchAnswered();
|
|
2860
|
+
return walk;
|
|
2861
|
+
}
|
|
2862
|
+
const fromGraph = hold.field.search(query, k, options, walk.admits, hold.removedNumbers);
|
|
2863
|
+
hold.searchAnswered();
|
|
2864
|
+
return outcomeOfGraphWalk(hold, walk, fromGraph, query, k, options);
|
|
2865
|
+
}
|
|
2866
|
+
async function coldSearchConcurrently(state, hold, query, k, options) {
|
|
2867
|
+
const walk = outcomeWithoutGraphWalk(state, hold, query, k, options);
|
|
2868
|
+
if ("results" in walk) {
|
|
2869
|
+
hold.searchAnswered();
|
|
2870
|
+
return walk;
|
|
2871
|
+
}
|
|
2872
|
+
const fromGraph = await hold.field.searchConcurrently(query, k, options, walk.admits, hold.removedNumbers);
|
|
2873
|
+
hold.searchAnswered();
|
|
2874
|
+
return outcomeOfGraphWalk(hold, walk, fromGraph, query, k, options);
|
|
2875
|
+
}
|
|
2876
|
+
|
|
2657
2877
|
// src/vector/vector-index/flagged-documents.ts
|
|
2658
2878
|
var FlaggedDocuments = class extends Set {
|
|
2659
2879
|
constructor(store, layout) {
|
|
@@ -2805,7 +3025,7 @@ function completeGraph(state) {
|
|
|
2805
3025
|
}
|
|
2806
3026
|
function maintenanceStatus(state) {
|
|
2807
3027
|
const storeSize = state.store.size;
|
|
2808
|
-
const graphCount = state.hnsw ? 1 : 0;
|
|
3028
|
+
const graphCount = state.hnsw || state.cold ? 1 : 0;
|
|
2809
3029
|
const graphNodes = state.hnsw === null ? storeSize : state.hnsw.size + state.hnsw.tombstoneCount;
|
|
2810
3030
|
const tombstoned = state.tombstones.size + state.store.replacedVectorCount;
|
|
2811
3031
|
const tombstoneRatio = graphNodes > 0 ? tombstoned / graphNodes : 0;
|
|
@@ -2968,6 +3188,91 @@ function withBufferedVectors(state, query, k, options, filter, graphResults, buf
|
|
|
2968
3188
|
return mergeResults(graphResults, scan.results, k);
|
|
2969
3189
|
}
|
|
2970
3190
|
|
|
3191
|
+
// src/vector/vector-index/writes.ts
|
|
3192
|
+
function createVectorWrites(state) {
|
|
3193
|
+
function validateDimension(vector) {
|
|
3194
|
+
if (vector.length !== state.dimension) {
|
|
3195
|
+
throw new NarsilError(
|
|
3196
|
+
ErrorCodes.VECTOR_DIMENSION_MISMATCH,
|
|
3197
|
+
`Vector dimension mismatch: expected ${state.dimension}, got ${vector.length}`,
|
|
3198
|
+
{ expected: state.dimension, received: vector.length }
|
|
3199
|
+
);
|
|
3200
|
+
}
|
|
3201
|
+
}
|
|
3202
|
+
function heldVectorOrUndefinedWhereItsFileFailsToRead(ordinal) {
|
|
3203
|
+
try {
|
|
3204
|
+
return state.store.entryForOrdinal(ordinal)?.vector;
|
|
3205
|
+
} catch {
|
|
3206
|
+
return void 0;
|
|
3207
|
+
}
|
|
3208
|
+
}
|
|
3209
|
+
function holdsThisVectorAlready(docId, ordinal, vector) {
|
|
3210
|
+
if (state.tombstones.has(docId)) return false;
|
|
3211
|
+
const held = heldVectorOrUndefinedWhereItsFileFailsToRead(ordinal);
|
|
3212
|
+
if (held === void 0) return false;
|
|
3213
|
+
for (let i = 0; i < vector.length; i++) if (held[i] !== vector[i]) return false;
|
|
3214
|
+
return true;
|
|
3215
|
+
}
|
|
3216
|
+
function applyInsert(docId, vector, partitionId) {
|
|
3217
|
+
handOverBuildTurns(state);
|
|
3218
|
+
const previous = state.store.getOrdinal(docId);
|
|
3219
|
+
if (previous !== void 0 && holdsThisVectorAlready(docId, previous, vector)) {
|
|
3220
|
+
if (partitionId !== void 0 && state.store.partitionOfOrdinal(previous) !== partitionId) {
|
|
3221
|
+
noteWrite(state);
|
|
3222
|
+
state.store.setPartition(docId, partitionId);
|
|
3223
|
+
}
|
|
3224
|
+
return;
|
|
3225
|
+
}
|
|
3226
|
+
noteWrite(state);
|
|
3227
|
+
state.tombstones.delete(docId);
|
|
3228
|
+
const graphHoldsPrevious = previous !== void 0 && ((state.hnsw?.holdsNode(previous) ?? false) || (state.freshGraph?.holdsNode(previous) ?? false));
|
|
3229
|
+
state.store.insert(docId, vector, partitionId, graphHoldsPrevious);
|
|
3230
|
+
if (previous !== void 0) {
|
|
3231
|
+
state.hnsw?.markTombstoneOrdinal(previous);
|
|
3232
|
+
state.freshGraph?.markTombstoneOrdinal(previous);
|
|
3233
|
+
if (!graphHoldsPrevious) state.osq?.removeOrdinal(previous);
|
|
3234
|
+
}
|
|
3235
|
+
state.buffer.add(docId);
|
|
3236
|
+
state.pendingLocations.delete(docId);
|
|
3237
|
+
if (!threadsHoldCurrentLayout(state)) scheduleWorkerCopyLoad(state);
|
|
3238
|
+
}
|
|
3239
|
+
function applyRemove(docId) {
|
|
3240
|
+
handOverBuildTurns(state);
|
|
3241
|
+
if (!state.store.has(docId)) return;
|
|
3242
|
+
noteWrite(state);
|
|
3243
|
+
state.tombstones.add(docId);
|
|
3244
|
+
state.buffer.delete(docId);
|
|
3245
|
+
state.pendingLocations.delete(docId);
|
|
3246
|
+
if (state.hnsw) {
|
|
3247
|
+
state.hnsw.markTombstone(docId);
|
|
3248
|
+
}
|
|
3249
|
+
}
|
|
3250
|
+
return {
|
|
3251
|
+
insert(docId, vector, partitionId) {
|
|
3252
|
+
validateDimension(vector);
|
|
3253
|
+
const hold = state.cold;
|
|
3254
|
+
if (hold === null) {
|
|
3255
|
+
applyInsert(docId, vector, partitionId);
|
|
3256
|
+
return;
|
|
3257
|
+
}
|
|
3258
|
+
noteWrite(state);
|
|
3259
|
+
coldInsert(hold, docId, vector, partitionId);
|
|
3260
|
+
const held = vector.slice();
|
|
3261
|
+
hold.writes.push(() => applyInsert(docId, held, partitionId));
|
|
3262
|
+
},
|
|
3263
|
+
remove(docId) {
|
|
3264
|
+
const hold = state.cold;
|
|
3265
|
+
if (hold === null) {
|
|
3266
|
+
applyRemove(docId);
|
|
3267
|
+
return;
|
|
3268
|
+
}
|
|
3269
|
+
if (!coldRemove(hold, docId)) return;
|
|
3270
|
+
noteWrite(state);
|
|
3271
|
+
hold.writes.push(() => applyRemove(docId));
|
|
3272
|
+
}
|
|
3273
|
+
};
|
|
3274
|
+
}
|
|
3275
|
+
|
|
2971
3276
|
// src/vector/vector-index/index.ts
|
|
2972
3277
|
function defaultQuantizationFor(dimension) {
|
|
2973
3278
|
if (dimension >= OSQ4_MIN_DIMENSION) return "osq4";
|
|
@@ -3038,76 +3343,26 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
3038
3343
|
sharedHandles: /* @__PURE__ */ new Map(),
|
|
3039
3344
|
sharedLayoutRevision: 0,
|
|
3040
3345
|
sharing: Promise.resolve(),
|
|
3041
|
-
releaseToFilesInFlight: Promise.resolve()
|
|
3346
|
+
releaseToFilesInFlight: Promise.resolve(),
|
|
3347
|
+
cold: null
|
|
3042
3348
|
};
|
|
3043
|
-
|
|
3044
|
-
if (vector.length !== state.dimension) {
|
|
3045
|
-
throw new NarsilError(
|
|
3046
|
-
ErrorCodes.VECTOR_DIMENSION_MISMATCH,
|
|
3047
|
-
`Vector dimension mismatch: expected ${state.dimension}, got ${vector.length}`,
|
|
3048
|
-
{ expected: state.dimension, received: vector.length }
|
|
3049
|
-
);
|
|
3050
|
-
}
|
|
3051
|
-
}
|
|
3052
|
-
function heldVectorOrUndefinedWhereItsFileFailsToRead(ordinal) {
|
|
3053
|
-
try {
|
|
3054
|
-
return state.store.entryForOrdinal(ordinal)?.vector;
|
|
3055
|
-
} catch {
|
|
3056
|
-
return void 0;
|
|
3057
|
-
}
|
|
3058
|
-
}
|
|
3059
|
-
function holdsThisVectorAlready(docId, ordinal, vector) {
|
|
3060
|
-
if (state.tombstones.has(docId)) return false;
|
|
3061
|
-
const held = heldVectorOrUndefinedWhereItsFileFailsToRead(ordinal);
|
|
3062
|
-
if (held === void 0) return false;
|
|
3063
|
-
for (let i = 0; i < vector.length; i++) if (held[i] !== vector[i]) return false;
|
|
3064
|
-
return true;
|
|
3065
|
-
}
|
|
3066
|
-
function insert(docId, vector, partitionId) {
|
|
3067
|
-
validateDimension(vector);
|
|
3068
|
-
handOverBuildTurns(state);
|
|
3069
|
-
const previous = state.store.getOrdinal(docId);
|
|
3070
|
-
if (previous !== void 0 && holdsThisVectorAlready(docId, previous, vector)) {
|
|
3071
|
-
if (partitionId !== void 0 && state.store.partitionOfOrdinal(previous) !== partitionId) {
|
|
3072
|
-
noteWrite(state);
|
|
3073
|
-
state.store.setPartition(docId, partitionId);
|
|
3074
|
-
}
|
|
3075
|
-
return;
|
|
3076
|
-
}
|
|
3077
|
-
noteWrite(state);
|
|
3078
|
-
state.tombstones.delete(docId);
|
|
3079
|
-
const graphHoldsPrevious = previous !== void 0 && ((state.hnsw?.holdsNode(previous) ?? false) || (state.freshGraph?.holdsNode(previous) ?? false));
|
|
3080
|
-
state.store.insert(docId, vector, partitionId, graphHoldsPrevious);
|
|
3081
|
-
if (previous !== void 0) {
|
|
3082
|
-
state.hnsw?.markTombstoneOrdinal(previous);
|
|
3083
|
-
state.freshGraph?.markTombstoneOrdinal(previous);
|
|
3084
|
-
if (!graphHoldsPrevious) state.osq?.removeOrdinal(previous);
|
|
3085
|
-
}
|
|
3086
|
-
state.buffer.add(docId);
|
|
3087
|
-
state.pendingLocations.delete(docId);
|
|
3088
|
-
if (!threadsHoldCurrentLayout(state)) scheduleWorkerCopyLoad(state);
|
|
3089
|
-
}
|
|
3090
|
-
function remove(docId) {
|
|
3091
|
-
handOverBuildTurns(state);
|
|
3092
|
-
if (!state.store.has(docId)) return;
|
|
3093
|
-
noteWrite(state);
|
|
3094
|
-
state.tombstones.add(docId);
|
|
3095
|
-
state.buffer.delete(docId);
|
|
3096
|
-
state.pendingLocations.delete(docId);
|
|
3097
|
-
if (state.hnsw) {
|
|
3098
|
-
state.hnsw.markTombstone(docId);
|
|
3099
|
-
}
|
|
3100
|
-
}
|
|
3349
|
+
const { insert, remove } = createVectorWrites(state);
|
|
3101
3350
|
function getVector(docId) {
|
|
3351
|
+
if (state.cold !== null) return coldVectorOf(state.cold, docId);
|
|
3102
3352
|
if (state.tombstones.has(docId)) return null;
|
|
3103
3353
|
const entry = state.store.get(docId);
|
|
3104
3354
|
if (!entry) return null;
|
|
3105
3355
|
return new Float32Array(entry.vector);
|
|
3106
3356
|
}
|
|
3107
3357
|
function has(docId) {
|
|
3358
|
+
if (state.cold !== null) return coldHas(state.cold, docId);
|
|
3108
3359
|
return state.store.has(docId) && !state.tombstones.has(docId);
|
|
3109
3360
|
}
|
|
3361
|
+
async function whenLoaded() {
|
|
3362
|
+
while (state.cold !== null) await state.cold.loaded;
|
|
3363
|
+
}
|
|
3110
3364
|
async function searchParallel(query, k, options) {
|
|
3365
|
+
if (state.cold !== null) return coldSearchConcurrently(state, state.cold, query, k, options);
|
|
3111
3366
|
handOverBuildTurns(state);
|
|
3112
3367
|
const confined = options.filterSubsets !== void 0 || options.filterPartitions !== void 0;
|
|
3113
3368
|
let filter = filterForOptions(state, options);
|
|
@@ -3154,44 +3409,73 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
3154
3409
|
get storage() {
|
|
3155
3410
|
return state.storage;
|
|
3156
3411
|
},
|
|
3412
|
+
get metric() {
|
|
3413
|
+
return state.metric;
|
|
3414
|
+
},
|
|
3157
3415
|
insert,
|
|
3158
3416
|
remove,
|
|
3159
|
-
partitionsKnown: () => state.store.partitionsKnown,
|
|
3160
|
-
assignPartitions: (resolve) =>
|
|
3161
|
-
|
|
3162
|
-
|
|
3163
|
-
|
|
3164
|
-
|
|
3165
|
-
|
|
3417
|
+
partitionsKnown: () => state.cold !== null || state.store.partitionsKnown,
|
|
3418
|
+
assignPartitions: (resolve) => {
|
|
3419
|
+
if (state.cold === null) assignStorePartitions(state, resolve);
|
|
3420
|
+
},
|
|
3421
|
+
scheduleBuild: () => {
|
|
3422
|
+
if (state.cold === null) scheduleBuild(state);
|
|
3423
|
+
},
|
|
3424
|
+
awaitPendingBuild: async () => {
|
|
3425
|
+
await whenLoaded();
|
|
3426
|
+
await awaitBuilds(state);
|
|
3427
|
+
},
|
|
3428
|
+
completeGraph: async () => {
|
|
3429
|
+
await whenLoaded();
|
|
3430
|
+
await completeGraph(state);
|
|
3431
|
+
},
|
|
3432
|
+
dispose: () => {
|
|
3433
|
+
cancelColdHold(state);
|
|
3434
|
+
disposeIndex(state);
|
|
3435
|
+
},
|
|
3436
|
+
search: (query, k, options) => state.cold === null ? search(state, query, k, options) : coldSearch(state, state.cold, query, k, options),
|
|
3166
3437
|
searchParallel,
|
|
3167
|
-
refreshWorkerCopies: () =>
|
|
3438
|
+
refreshWorkerCopies: () => {
|
|
3439
|
+
if (state.cold === null) refreshWorkerCopies(state);
|
|
3440
|
+
},
|
|
3168
3441
|
getVector,
|
|
3169
3442
|
has,
|
|
3170
3443
|
compact: () => {
|
|
3444
|
+
if (state.cold !== null) return;
|
|
3171
3445
|
noteWrite(state);
|
|
3172
3446
|
compact(state);
|
|
3173
3447
|
},
|
|
3174
3448
|
optimize: async () => {
|
|
3449
|
+
await whenLoaded();
|
|
3175
3450
|
noteWrite(state);
|
|
3176
3451
|
await optimize(state);
|
|
3177
3452
|
},
|
|
3178
|
-
maintenanceStatus: () => maintenanceStatus(state),
|
|
3179
|
-
estimateMemoryBytes: () => estimateMemoryBytes(state),
|
|
3453
|
+
maintenanceStatus: () => ({ ...maintenanceStatus(state), loadingFromFiles: state.cold !== null }),
|
|
3454
|
+
estimateMemoryBytes: () => estimateMemoryBytes(state) + (state.cold?.field.residentBytes ?? 0),
|
|
3180
3455
|
growthBytesFor: (vectors) => state.store.growthBytesFor(vectors),
|
|
3181
3456
|
deserialize: (parts) => {
|
|
3457
|
+
cancelColdHold(state);
|
|
3182
3458
|
invalidateWorkerCopies(state);
|
|
3183
3459
|
deserialize(state, parts);
|
|
3184
3460
|
},
|
|
3185
3461
|
planCheckpoint: (listedKeys, rewritesGraph) => planCheckpoint(state, listedKeys, rewritesGraph),
|
|
3186
3462
|
recordCheckpoint: (plan, written) => recordCheckpoint(state, plan, written),
|
|
3187
3463
|
restoreCheckpoint: (shape) => {
|
|
3464
|
+
cancelColdHold(state);
|
|
3188
3465
|
invalidateWorkerCopies(state);
|
|
3189
3466
|
return beginCheckpointRestore(state, shape);
|
|
3190
3467
|
},
|
|
3191
|
-
|
|
3192
|
-
|
|
3468
|
+
openColdField: (field, load) => openColdField(state, field, load),
|
|
3469
|
+
startShuffledRebuildWhenDue: () => {
|
|
3470
|
+
if (state.cold === null) startShuffledRebuild(state);
|
|
3471
|
+
},
|
|
3472
|
+
vectorFilesInUse: () => {
|
|
3473
|
+
const stored = state.store.dimension === 0 ? [] : state.store.handles.vectorFiles.filter((path) => path !== "");
|
|
3474
|
+
return state.cold === null ? stored : [...stored, ...state.cold.field.filePaths()];
|
|
3475
|
+
},
|
|
3193
3476
|
adoptDiskLayout: (layout) => adoptDiskLayout(state, layout),
|
|
3194
3477
|
releaseVectorFiles: async () => {
|
|
3478
|
+
await whenLoaded();
|
|
3195
3479
|
await withdrawWorkerCopies(state);
|
|
3196
3480
|
state.store.closeFiles();
|
|
3197
3481
|
},
|
|
@@ -3201,7 +3485,10 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
3201
3485
|
state.pendingLocations.set(docId, { ...location, path: rewrite(location.path) });
|
|
3202
3486
|
}
|
|
3203
3487
|
},
|
|
3204
|
-
holdWorkerCopies: () =>
|
|
3488
|
+
holdWorkerCopies: async () => {
|
|
3489
|
+
await whenLoaded();
|
|
3490
|
+
await holdWorkerCopies(state);
|
|
3491
|
+
},
|
|
3205
3492
|
resumeWorkerCopies: () => {
|
|
3206
3493
|
state.workerCopiesHeld = false;
|
|
3207
3494
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { abandonChangeOf } from './chunk-
|
|
1
|
+
import { abandonChangeOf } from './chunk-XNWPP5NS.mjs';
|
|
2
2
|
|
|
3
3
|
// src/narsil/restore-queue.ts
|
|
4
4
|
function restoreDepsOf(core, host) {
|
|
@@ -24,7 +24,7 @@ function restoreDepsOf(core, host) {
|
|
|
24
24
|
function queueRestore(core, host, restoresInFlight, indexName, data) {
|
|
25
25
|
const previous = restoresInFlight.get(indexName) ?? Promise.resolve();
|
|
26
26
|
const restored = previous.then(async () => {
|
|
27
|
-
const { restoreFromSnapshot } = await import('./snapshot-
|
|
27
|
+
const { restoreFromSnapshot } = await import('./snapshot-QXBKHJAG.mjs');
|
|
28
28
|
await restoreFromSnapshot(indexName, data, restoreDepsOf(core, host));
|
|
29
29
|
});
|
|
30
30
|
const settled = restored.then(
|