@delali/narsil 0.3.1-next.91 → 0.3.1-next.95
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/browser/{chunk-66KYAOIK.mjs → chunk-3U636ADD.mjs} +2 -2
- package/dist/browser/{chunk-NOUZ6RH3.mjs → chunk-D3CC2NMZ.mjs} +238 -114
- package/dist/browser/{chunk-3ZRMZIH4.mjs → chunk-FYFQJKFW.mjs} +208 -78
- package/dist/browser/{chunk-FHFFWLEK.mjs → chunk-HARBSUWP.mjs} +3 -4
- package/dist/browser/{chunk-B6XE7KXY.mjs → chunk-LULHQXCK.mjs} +4 -4
- package/dist/browser/{chunk-NQW2YXEX.mjs → chunk-WKPJ4E73.mjs} +1 -1
- package/dist/browser/{durability-wiring-F4OGQUSU.mjs → durability-wiring-3ZRKSPJD.mjs} +498 -71
- package/dist/browser/{module-E25UDADM.mjs → module-IEOQI5GI.mjs} +102 -54
- package/dist/browser/release-76MXRPGV.mjs +33 -0
- package/dist/browser/replace-RURXTGVG.mjs +57 -0
- package/dist/browser/restore-queue-RBULSOVX.mjs +35 -0
- package/dist/browser/{snapshot-5EJ7I7QY.mjs → snapshot-Y3NGUJHX.mjs} +362 -103
- package/dist/browser/{write-K5V4XQ72.mjs → write-ZUBDNLCH.mjs} +4 -4
- package/dist/{capabilities-CvExBThq.d.ts → capabilities-BvxvFzVA.d.ts} +2 -2
- package/dist/{chunk-NVL4ZR3K.mjs → chunk-2GZGBOQI.mjs} +2 -2
- package/dist/{chunk-XLJ2S2BH.mjs → chunk-3LYY62M6.mjs} +40 -21
- package/dist/{chunk-CVQWQVFX.mjs → chunk-4ZJWRETC.mjs} +2 -2
- package/dist/{chunk-T4DB5YFB.mjs → chunk-54HYC5FT.mjs} +6 -6
- package/dist/{chunk-ZIJ63TBT.mjs → chunk-7KC56574.mjs} +1 -1
- package/dist/{chunk-67Z2V6TC.mjs → chunk-7QONURHW.mjs} +3 -3
- package/dist/{chunk-BVOJIYRV.mjs → chunk-BB7MASRQ.mjs} +1 -1
- package/dist/{chunk-6VZ3Q73N.mjs → chunk-CHOGQSHK.mjs} +44 -25
- package/dist/{chunk-QZN3MIJ3.mjs → chunk-ES33L3HO.mjs} +2 -2
- package/dist/{chunk-YCJLVNWR.mjs → chunk-F52QGNZ2.mjs} +1 -1
- package/dist/{chunk-43ORORN6.mjs → chunk-GJZHU4FC.mjs} +3 -3
- package/dist/{chunk-R7OSDAFK.mjs → chunk-HBVLF3HA.mjs} +1 -1
- package/dist/{chunk-ULRPNYAG.mjs → chunk-ICL2AELD.mjs} +120 -62
- package/dist/{chunk-3RDS4UHC.mjs → chunk-J7SDAOTP.mjs} +4 -4
- package/dist/{chunk-A4G7JSYO.mjs → chunk-JFUE6F2Y.mjs} +98 -49
- package/dist/chunk-KOKM4PQN.mjs +52 -0
- package/dist/{chunk-7OQHSDK6.mjs → chunk-L26LEDHO.mjs} +7 -7
- package/dist/{chunk-EONIHG3V.mjs → chunk-MASE2LTJ.mjs} +7 -7
- package/dist/{chunk-MXS5HWIH.mjs → chunk-MWT5WCJ5.mjs} +89 -17
- package/dist/{chunk-3TTYK5A7.mjs → chunk-Q4UTU5ZE.mjs} +9 -9
- package/dist/{chunk-T6CNO72J.mjs → chunk-QM3VPX4Z.mjs} +2 -2
- package/dist/chunk-QY7GC2MW.mjs +33 -0
- package/dist/{chunk-BJ2YKRNX.mjs → chunk-R36MIGBU.mjs} +86 -54
- package/dist/{chunk-RVM7AODE.mjs → chunk-RDJKYKMQ.mjs} +2 -2
- package/dist/{chunk-UXQE2ROB.mjs → chunk-RUIXODYU.mjs} +3 -2
- package/dist/{chunk-5ER245FI.mjs → chunk-SBCEHIPL.mjs} +3 -3
- package/dist/{chunk-TPJU7NDJ.mjs → chunk-SYWGORQB.mjs} +1 -0
- package/dist/{chunk-UKSUENHK.mjs → chunk-T2QN2FA4.mjs} +40 -26
- package/dist/{chunk-CO4MRKST.mjs → chunk-WLOF6HSP.mjs} +2 -2
- package/dist/{chunk-MKWUXWQH.mjs → chunk-ZHS44YKF.mjs} +1 -1
- package/dist/{client-Cc42b42j.d.ts → client-DKwterMb.d.ts} +2 -2
- package/dist/client.d.ts +6 -6
- package/dist/client.mjs +1 -0
- package/dist/{config-DyRowvVP.d.ts → config-6YBtxekG.d.ts} +1 -1
- package/dist/{config-BYne-yCI.d.ts → config-DPphZfuM.d.ts} +1 -1
- package/dist/direct-executor-5KJGVQHM.mjs +22 -0
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +5 -5
- package/dist/distribution.mjs +26 -26
- package/dist/{durability-wiring-6AN4QTT3.mjs → durability-wiring-FRPV6JLZ.mjs} +518 -91
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/{engine-96_kLtSY.d.ts → engine-BGt9lI1V.d.ts} +11 -11
- package/dist/index.browser.mjs +2 -2
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +18 -18
- package/dist/{memory-dd3GESdc.d.ts → memory-COdoGthn.d.ts} +96 -76
- package/dist/{module-WIDYIJLY.mjs → module-IUTNXWMV.mjs} +72 -38
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +21 -21
- package/dist/react.d.ts +6 -6
- package/dist/react.mjs +1 -0
- package/dist/release-BH3DKW3T.mjs +33 -0
- package/dist/replace-CEO5KJJ3.mjs +57 -0
- package/dist/restore-queue-I5SI6AUQ.mjs +35 -0
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +10 -10
- package/dist/snapshot-YC6TS4OP.mjs +738 -0
- package/dist/synonyms.mjs +3 -3
- package/dist/{tasks-DA1Ff0G4.d.ts → tasks-BJmamiC4.d.ts} +1 -1
- package/dist/{thread-VTDLLYXC.mjs → thread-U7XYJWGG.mjs} +15 -12
- package/dist/vector/search-worker.mjs +6 -6
- package/dist/workers/entry.mjs +5 -5
- package/dist/{write-OJR3H4BI.mjs → write-M2OALSA4.mjs} +20 -20
- package/package.json +7 -7
- package/dist/chunk-Y2F66IQJ.mjs +0 -74
- package/dist/chunk-ZJ4CSAB2.mjs +0 -13
- package/dist/direct-executor-CHXVCBN7.mjs +0 -22
- package/dist/snapshot-JFZVOGZV.mjs +0 -479
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
|
|
2
|
-
import { VECTOR_WORKER_COPIES_ALLOWED, openSharedQuantizer, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, osqBitsOf, createOsqQuantizer, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterValues, threadsHoldCurrentLayout, filterForOptions, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, OSQ4_MIN_DIMENSION, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY,
|
|
3
|
-
export { releaseLocksHeldBy } from './chunk-
|
|
4
|
-
import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-
|
|
5
|
-
import { NarsilError, ErrorCodes, DISK_STORAGE_BLOCK_BYTES, compareCodePoints, UNPLACED_FLAG_CHUNK_ORDINALS, MAX_VECTOR_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, resolveWorkerCount, resolveWorkerEntry, detectRuntime, WORKER_YOUNG_GENERATION_MB, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS } from './chunk-
|
|
2
|
+
import { VECTOR_WORKER_COPIES_ALLOWED, openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, osqBitsOf, createOsqQuantizer, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, ordinalFilterValues, threadsHoldCurrentLayout, filterForOptions, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, OSQ4_MIN_DIMENSION, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, WORKER_COPY_MIN_VECTORS, allLiveDocIds, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, BLOCK_MAX_BYTES, growBufferTo, unplacedChunkCount, STORE_DOC_ID_BYTES, recalibrateFromStore, signatureOf, nativeStoreScratchBytes, noteFallback, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, QUERY_STAGED, createVectorBlock, ensureBlockSlots, createHNSWIndex, adoptGraph, topLayerInParts, calibrateQuantizer, fieldHandlesOf, sharedMemoryAvailable, requireWholeNeighbourLists, BUILD_CHUNK_SIZE, yieldToEventLoop, releaseLocksHeldBy, DISK_ADOPTION_YIELD_INTERVAL } from './chunk-FYFQJKFW.mjs';
|
|
3
|
+
export { releaseLocksHeldBy } from './chunk-FYFQJKFW.mjs';
|
|
4
|
+
import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-WKPJ4E73.mjs';
|
|
5
|
+
import { NarsilError, ErrorCodes, MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, compareCodePoints, UNPLACED_FLAG_CHUNK_ORDINALS, MAX_VECTOR_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, resolveWorkerCount, resolveWorkerEntry, detectRuntime, WORKER_YOUNG_GENERATION_MB, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS } from './chunk-HARBSUWP.mjs';
|
|
6
6
|
|
|
7
7
|
// src/core/heap.ts
|
|
8
8
|
function createMaxHeap(compare) {
|
|
@@ -205,14 +205,21 @@ function scoreCandidates(store, candidates, query, k, metric, minSimilarity) {
|
|
|
205
205
|
function createSharedVectorSearcher(options) {
|
|
206
206
|
const { fieldName, view, holdsDocument } = options;
|
|
207
207
|
const flags = readStoreFlags(view.store);
|
|
208
|
-
function
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
208
|
+
function ordinalLayoutGeneration() {
|
|
209
|
+
return Atomics.load(view.store.handles.header, STORE_ORDINAL_LAYOUT_GENERATION);
|
|
210
|
+
}
|
|
211
|
+
function ordinalSpace() {
|
|
212
|
+
const store = view.store;
|
|
213
|
+
const layoutGeneration = ordinalLayoutGeneration();
|
|
214
|
+
return {
|
|
215
|
+
fieldName,
|
|
216
|
+
layoutGeneration,
|
|
217
|
+
slots: store.slots,
|
|
218
|
+
header: store.handles.header,
|
|
219
|
+
ordinalOf: (docId) => view.ordinalOf(docId),
|
|
220
|
+
holdsCurrentOrdinal: (ordinal) => store.holdsOrdinal(ordinal) && store.partitionAt(ordinal) !== void 0,
|
|
221
|
+
isRemoved: (ordinal) => flags.isRemoved(ordinal)
|
|
222
|
+
};
|
|
216
223
|
}
|
|
217
224
|
function filterForPartitions(allowed) {
|
|
218
225
|
const slots = view.store.slots;
|
|
@@ -226,7 +233,9 @@ function createSharedVectorSearcher(options) {
|
|
|
226
233
|
return filter;
|
|
227
234
|
}
|
|
228
235
|
function filterFor(searchOptions) {
|
|
229
|
-
if (searchOptions.
|
|
236
|
+
if (searchOptions.filterSubsets !== void 0) {
|
|
237
|
+
return ordinalFilterForSubsets(searchOptions.filterSubsets, ordinalSpace());
|
|
238
|
+
}
|
|
230
239
|
if (searchOptions.filterPartitions !== void 0) return filterForPartitions(searchOptions.filterPartitions);
|
|
231
240
|
return void 0;
|
|
232
241
|
}
|
|
@@ -242,6 +251,11 @@ function createSharedVectorSearcher(options) {
|
|
|
242
251
|
}
|
|
243
252
|
return candidates;
|
|
244
253
|
}
|
|
254
|
+
function answerOf(outcome) {
|
|
255
|
+
const results = outcome.results.filter((result) => holdsDocument(result.docId));
|
|
256
|
+
const holdsEveryHit = outcome.resolvedEveryHit && results.length === outcome.results.length;
|
|
257
|
+
return { results, matched: outcome.matched, matchedExact: outcome.matchedExact && holdsEveryHit };
|
|
258
|
+
}
|
|
245
259
|
function scanned(scan) {
|
|
246
260
|
return { results: scan.results, matched: scan.matched, matchedExact: true, resolvedEveryHit: true };
|
|
247
261
|
}
|
|
@@ -300,21 +314,25 @@ function createSharedVectorSearcher(options) {
|
|
|
300
314
|
{ expected: view.handles.dimension, received: query.length }
|
|
301
315
|
);
|
|
302
316
|
}
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
317
|
+
for (let attempt = 1; ; attempt++) {
|
|
318
|
+
const layoutGeneration = ordinalLayoutGeneration();
|
|
319
|
+
const outcome = search2(query, k, searchOptions, filterFor(searchOptions));
|
|
320
|
+
if (view.graphRetired) {
|
|
321
|
+
throw new NarsilError(
|
|
322
|
+
ErrorCodes.WORKER_BUSY,
|
|
323
|
+
`The main thread replaced the graph of vector field "${fieldName}" while this thread searched it`,
|
|
324
|
+
{ fieldName }
|
|
325
|
+
);
|
|
326
|
+
}
|
|
327
|
+
if (ordinalLayoutGeneration() === layoutGeneration) return Promise.resolve(answerOf(outcome));
|
|
328
|
+
if (attempt >= MAX_RENUMBERED_SEARCH_ATTEMPTS) {
|
|
329
|
+
throw new NarsilError(
|
|
330
|
+
ErrorCodes.WORKER_BUSY,
|
|
331
|
+
`The main thread renumbered the vectors of field "${fieldName}" during each of ${attempt} searches on this thread`,
|
|
332
|
+
{ fieldName, attempts: attempt }
|
|
333
|
+
);
|
|
334
|
+
}
|
|
310
335
|
}
|
|
311
|
-
const results = outcome.results.filter((result) => holdsDocument(result.docId));
|
|
312
|
-
const holdsEveryHit = outcome.resolvedEveryHit && results.length === outcome.results.length;
|
|
313
|
-
return Promise.resolve({
|
|
314
|
-
results,
|
|
315
|
-
matched: outcome.matched,
|
|
316
|
-
matchedExact: outcome.matchedExact && holdsEveryHit
|
|
317
|
-
});
|
|
318
336
|
}
|
|
319
337
|
};
|
|
320
338
|
}
|
|
@@ -686,6 +704,7 @@ function openSharedVectorField(initial, threadSlot) {
|
|
|
686
704
|
let graph = null;
|
|
687
705
|
let ordinals = null;
|
|
688
706
|
let scanned = 0;
|
|
707
|
+
let scannedLayoutGeneration = Atomics.load(store.handles.header, STORE_ORDINAL_LAYOUT_GENERATION);
|
|
689
708
|
let placed = 0;
|
|
690
709
|
function openGraph(next) {
|
|
691
710
|
graph = next.graph === null ? null : openGraphState(next.graph, next.dimension, store, quantizer, threadSlot);
|
|
@@ -696,6 +715,12 @@ function openSharedVectorField(initial, threadSlot) {
|
|
|
696
715
|
return store.docIdAt(ordinal);
|
|
697
716
|
}
|
|
698
717
|
function ordinalOf(docId) {
|
|
718
|
+
const layoutGeneration = Atomics.load(store.handles.header, STORE_ORDINAL_LAYOUT_GENERATION);
|
|
719
|
+
if (layoutGeneration !== scannedLayoutGeneration) {
|
|
720
|
+
ordinals = null;
|
|
721
|
+
scanned = 0;
|
|
722
|
+
scannedLayoutGeneration = layoutGeneration;
|
|
723
|
+
}
|
|
699
724
|
const slots = store.slots;
|
|
700
725
|
if (ordinals === null) ordinals = /* @__PURE__ */ new Map();
|
|
701
726
|
for (; scanned < slots; scanned++) {
|
|
@@ -863,6 +888,31 @@ function relayout(store) {
|
|
|
863
888
|
store.handles.layoutRevision += 1;
|
|
864
889
|
store.view.adoptHandles(store.handles);
|
|
865
890
|
}
|
|
891
|
+
function claimSlot(store, docId, cold) {
|
|
892
|
+
const ordinal = Atomics.load(store.handles.header, STORE_SLOTS);
|
|
893
|
+
ensureOrdinal(store, ordinal, cold);
|
|
894
|
+
store.diskFile[ordinal] = IN_MEMORY;
|
|
895
|
+
appendDocId(store.handles, ordinal, docId);
|
|
896
|
+
return ordinal;
|
|
897
|
+
}
|
|
898
|
+
function publishOrdinal(store, ordinal) {
|
|
899
|
+
Atomics.store(store.handles.header, STORE_SLOTS, ordinal + 1);
|
|
900
|
+
Atomics.store(store.present, ordinal, 1);
|
|
901
|
+
}
|
|
902
|
+
function publishOnDisk(store, ordinal, vectorMagnitude, location) {
|
|
903
|
+
store.magnitudes[ordinal] = vectorMagnitude;
|
|
904
|
+
store.diskOffset[ordinal] = location.offset;
|
|
905
|
+
store.diskFile[ordinal] = location.fileIndex;
|
|
906
|
+
publishOrdinal(store, ordinal);
|
|
907
|
+
}
|
|
908
|
+
function relocateFiles(store, rewrite) {
|
|
909
|
+
const files = store.handles.vectorFiles;
|
|
910
|
+
if (files.length === 0) return;
|
|
911
|
+
for (let file = 0; file < files.length; file++) {
|
|
912
|
+
if (files[file].length > 0) files[file] = rewrite(files[file]);
|
|
913
|
+
}
|
|
914
|
+
relayout(store);
|
|
915
|
+
}
|
|
866
916
|
function ensureCodeSlot(store, ordinal) {
|
|
867
917
|
const { handles } = store;
|
|
868
918
|
if (handles.codeLayout === null) return;
|
|
@@ -1031,13 +1081,6 @@ function createVectorStore(options) {
|
|
|
1031
1081
|
Atomics.store(store.present, ordinal, 1);
|
|
1032
1082
|
replacedOrdinals.add(ordinal);
|
|
1033
1083
|
}
|
|
1034
|
-
function claimSlot(store, docId, cold) {
|
|
1035
|
-
const ordinal = Atomics.load(store.handles.header, STORE_SLOTS);
|
|
1036
|
-
ensureOrdinal(store, ordinal, cold);
|
|
1037
|
-
store.diskFile[ordinal] = IN_MEMORY;
|
|
1038
|
-
appendDocId(store.handles, ordinal, docId);
|
|
1039
|
-
return ordinal;
|
|
1040
|
-
}
|
|
1041
1084
|
function claimOrdinal(store, docId, cold, partitionId) {
|
|
1042
1085
|
const ordinal = claimSlot(store, docId, cold);
|
|
1043
1086
|
recordPartition(store, ordinal, partitionId);
|
|
@@ -1045,16 +1088,6 @@ function createVectorStore(options) {
|
|
|
1045
1088
|
docToOrd.set(docId, ordinal);
|
|
1046
1089
|
return ordinal;
|
|
1047
1090
|
}
|
|
1048
|
-
function publishOrdinal(store, ordinal) {
|
|
1049
|
-
Atomics.store(store.handles.header, STORE_SLOTS, ordinal + 1);
|
|
1050
|
-
Atomics.store(store.present, ordinal, 1);
|
|
1051
|
-
}
|
|
1052
|
-
function publishOnDisk(store, ordinal, vectorMagnitude, location) {
|
|
1053
|
-
store.magnitudes[ordinal] = vectorMagnitude;
|
|
1054
|
-
store.diskOffset[ordinal] = location.offset;
|
|
1055
|
-
store.diskFile[ordinal] = location.fileIndex;
|
|
1056
|
-
publishOrdinal(store, ordinal);
|
|
1057
|
-
}
|
|
1058
1091
|
function appendOrdinal(store, docId, vector, partitionId) {
|
|
1059
1092
|
const ordinal = claimOrdinal(store, docId, false, partitionId);
|
|
1060
1093
|
writeVector(store, ordinal, vector);
|
|
@@ -1225,19 +1258,18 @@ function createVectorStore(options) {
|
|
|
1225
1258
|
relayout(open);
|
|
1226
1259
|
}
|
|
1227
1260
|
resetDocIds(open.handles);
|
|
1228
|
-
|
|
1229
|
-
Atomics.store(open.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 0);
|
|
1230
|
-
Atomics.store(open.handles.header, STORE_LIVE_COUNT, 0);
|
|
1231
|
-
Atomics.store(open.handles.header, STORE_CODE_COUNT, 0);
|
|
1232
|
-
Atomics.store(open.handles.header, STORE_CALIBRATED, 0);
|
|
1233
|
-
Atomics.add(open.handles.header, STORE_CALIBRATION_GENERATION, 1);
|
|
1261
|
+
resetStoreHeader(open.handles.header);
|
|
1234
1262
|
},
|
|
1235
1263
|
release() {
|
|
1236
1264
|
forgetDocuments();
|
|
1265
|
+
if (open !== null) Atomics.add(open.handles.header, STORE_ORDINAL_LAYOUT_GENERATION, 1);
|
|
1237
1266
|
open?.view.close();
|
|
1238
1267
|
open = null;
|
|
1239
1268
|
},
|
|
1240
1269
|
closeFiles: () => open?.view.close(),
|
|
1270
|
+
relocateVectorFiles(rewrite) {
|
|
1271
|
+
if (open !== null) relocateFiles(open, rewrite);
|
|
1272
|
+
},
|
|
1241
1273
|
getOrdinal(docId) {
|
|
1242
1274
|
return docToOrd.get(docId);
|
|
1243
1275
|
},
|
|
@@ -1606,6 +1638,11 @@ async function withdrawWorkerCopies(state) {
|
|
|
1606
1638
|
await Promise.all([...handles].map((handle) => dropHandle(state, handle)));
|
|
1607
1639
|
if (state.workerCopies.host === void 0 && state.sharedHandles.size === 0) state.workerCopyPool = null;
|
|
1608
1640
|
}
|
|
1641
|
+
async function holdWorkerCopies(state) {
|
|
1642
|
+
state.workerCopiesHeld = true;
|
|
1643
|
+
await state.sharing;
|
|
1644
|
+
await withdrawWorkerCopies(state);
|
|
1645
|
+
}
|
|
1609
1646
|
function noteWrite(state) {
|
|
1610
1647
|
state.revision += 1;
|
|
1611
1648
|
if (state.workerCopyMode === "clone") invalidateWorkerCopies(state);
|
|
@@ -1704,7 +1741,7 @@ async function loadOnPool(state, graph, searchable) {
|
|
|
1704
1741
|
}
|
|
1705
1742
|
function shareGraph(state, graph, searchable) {
|
|
1706
1743
|
const run = state.sharing.then(async () => {
|
|
1707
|
-
if (!state.workerCopies.enabled || state.disposed) return null;
|
|
1744
|
+
if (!state.workerCopies.enabled || state.disposed || state.workerCopiesHeld) return null;
|
|
1708
1745
|
const existing = state.sharedHandles.get(graph);
|
|
1709
1746
|
const layoutUnchanged = threadsHoldCurrentLayout(state);
|
|
1710
1747
|
if (existing !== void 0 && layoutUnchanged && (existing.searchable || !searchable)) return existing.handle;
|
|
@@ -1719,7 +1756,7 @@ function shareGraph(state, graph, searchable) {
|
|
|
1719
1756
|
}
|
|
1720
1757
|
function resendSharedHandles(state) {
|
|
1721
1758
|
const run = state.sharing.then(async () => {
|
|
1722
|
-
if (!state.workerCopies.enabled || state.disposed) return;
|
|
1759
|
+
if (!state.workerCopies.enabled || state.disposed || state.workerCopiesHeld) return;
|
|
1723
1760
|
const host = state.workerCopies.host;
|
|
1724
1761
|
for (const [graph, shared] of [...state.sharedHandles]) {
|
|
1725
1762
|
if (host !== void 0) await loadOnHost(state, host, graph, shared.searchable);
|
|
@@ -1733,7 +1770,7 @@ function resendSharedHandles(state) {
|
|
|
1733
1770
|
return run;
|
|
1734
1771
|
}
|
|
1735
1772
|
function scheduleWorkerCopyLoad(state) {
|
|
1736
|
-
if (!state.workerCopies.enabled) return;
|
|
1773
|
+
if (!state.workerCopies.enabled || state.workerCopiesHeld) return;
|
|
1737
1774
|
if (state.disposed || state.workerCopyLoading || state.building) return;
|
|
1738
1775
|
const graph = state.hnsw;
|
|
1739
1776
|
const host = state.workerCopies.host;
|
|
@@ -2581,6 +2618,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
2581
2618
|
workerCopyRevision: -1,
|
|
2582
2619
|
workerCopyMode: null,
|
|
2583
2620
|
workerCopyLoading: false,
|
|
2621
|
+
workerCopiesHeld: false,
|
|
2584
2622
|
sharedHandles: /* @__PURE__ */ new Map(),
|
|
2585
2623
|
sharedLayoutRevision: 0,
|
|
2586
2624
|
sharing: Promise.resolve(),
|
|
@@ -2678,7 +2716,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
2678
2716
|
releaseHeldMemory();
|
|
2679
2717
|
}
|
|
2680
2718
|
async function searchParallel(query, k, options) {
|
|
2681
|
-
const confined = options.
|
|
2719
|
+
const confined = options.filterSubsets !== void 0 || options.filterPartitions !== void 0;
|
|
2682
2720
|
let filter = filterForOptions(state, options);
|
|
2683
2721
|
if (filter !== void 0) {
|
|
2684
2722
|
if (state.hnsw) {
|
|
@@ -2759,6 +2797,16 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
2759
2797
|
releaseVectorFiles: async () => {
|
|
2760
2798
|
await withdrawWorkerCopies(state);
|
|
2761
2799
|
state.store.closeFiles();
|
|
2800
|
+
},
|
|
2801
|
+
relocateVectorFiles: (rewrite) => {
|
|
2802
|
+
state.store.relocateVectorFiles(rewrite);
|
|
2803
|
+
for (const [docId, location] of state.pendingLocations) {
|
|
2804
|
+
state.pendingLocations.set(docId, { ...location, path: rewrite(location.path) });
|
|
2805
|
+
}
|
|
2806
|
+
},
|
|
2807
|
+
holdWorkerCopies: () => holdWorkerCopies(state),
|
|
2808
|
+
resumeWorkerCopies: () => {
|
|
2809
|
+
state.workerCopiesHeld = false;
|
|
2762
2810
|
}
|
|
2763
2811
|
};
|
|
2764
2812
|
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
// src/engine/orchestration/release.ts
|
|
2
|
+
async function releaseIndexCopies(state, indexName, steps) {
|
|
3
|
+
const wasScaledOut = state.scaledOutIndexes.delete(indexName);
|
|
4
|
+
const loading = state.copyTransitions.get(indexName);
|
|
5
|
+
if (loading !== void 0) await loading.done.catch(() => void 0);
|
|
6
|
+
await steps.awaitReplicationIdle(state, indexName);
|
|
7
|
+
steps.cancelIdleMerge(state, indexName);
|
|
8
|
+
const reloads = wasScaledOut || loading !== void 0 && loading.kind !== "drop";
|
|
9
|
+
const pool = state.workerPool;
|
|
10
|
+
if (pool === null) return reloads;
|
|
11
|
+
pool.removeIndex(indexName);
|
|
12
|
+
state.segmentLedger.delete(indexName);
|
|
13
|
+
state.unbroadcastSegments.delete(indexName);
|
|
14
|
+
await Promise.all(
|
|
15
|
+
pool.executorEntries().map(
|
|
16
|
+
({ executor }) => executor.execute({ type: "dropIndex", indexName, requestId: `restore-release-${indexName}` }).catch((error) => {
|
|
17
|
+
if (!steps.heldNowhere(error)) throw error;
|
|
18
|
+
})
|
|
19
|
+
)
|
|
20
|
+
);
|
|
21
|
+
return reloads;
|
|
22
|
+
}
|
|
23
|
+
async function endCopyRelease(state, indexName, reloads, steps) {
|
|
24
|
+
const hold = state.copyReleaseHolds.get(indexName);
|
|
25
|
+
if (hold === void 0) return steps.resyncIndex(indexName, reloads);
|
|
26
|
+
hold.holders -= 1;
|
|
27
|
+
hold.reloads ||= reloads;
|
|
28
|
+
if (hold.holders > 0) return;
|
|
29
|
+
state.copyReleaseHolds.delete(indexName);
|
|
30
|
+
await steps.resyncIndex(indexName, hold.reloads);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export { endCopyRelease, releaseIndexCopies };
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
// src/engine/index-state/replace.ts
|
|
2
|
+
function runReplacement(context, entry, body, settle) {
|
|
3
|
+
const previous = entry.state;
|
|
4
|
+
entry.state = "replacing";
|
|
5
|
+
const run = (async () => {
|
|
6
|
+
await context.waitForDrain(entry);
|
|
7
|
+
await body();
|
|
8
|
+
})().then(
|
|
9
|
+
() => settle(true, previous),
|
|
10
|
+
(error) => {
|
|
11
|
+
settle(false, previous);
|
|
12
|
+
throw error;
|
|
13
|
+
}
|
|
14
|
+
).finally(() => {
|
|
15
|
+
entry.replacePromise = null;
|
|
16
|
+
});
|
|
17
|
+
entry.replacePromise = run;
|
|
18
|
+
return run;
|
|
19
|
+
}
|
|
20
|
+
function replaceUnregistered(context, indexName, entry, action) {
|
|
21
|
+
return runReplacement(context, entry, action, (succeeded) => {
|
|
22
|
+
if (!succeeded) {
|
|
23
|
+
context.forget(indexName);
|
|
24
|
+
return;
|
|
25
|
+
}
|
|
26
|
+
entry.state = "open";
|
|
27
|
+
entry.lastAccessAt = Date.now();
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
async function replaceExclusively(context, indexName, action) {
|
|
31
|
+
let entry = context.entryOf(indexName);
|
|
32
|
+
while (true) {
|
|
33
|
+
const pending = entry.dropPromise ?? entry.replacePromise ?? entry.reopenPromise ?? entry.closePromise;
|
|
34
|
+
if (pending === null) break;
|
|
35
|
+
await pending.catch(() => void 0);
|
|
36
|
+
entry = context.entryOf(indexName);
|
|
37
|
+
}
|
|
38
|
+
const wasOpen = entry.state === "open";
|
|
39
|
+
await runReplacement(
|
|
40
|
+
context,
|
|
41
|
+
entry,
|
|
42
|
+
() => action(wasOpen),
|
|
43
|
+
(succeeded, previous) => {
|
|
44
|
+
if (!succeeded) {
|
|
45
|
+
entry.state = previous;
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
entry.state = "open";
|
|
49
|
+
entry.cachedError = null;
|
|
50
|
+
entry.reopenFailures = 0;
|
|
51
|
+
entry.nextReopenAt = 0;
|
|
52
|
+
entry.lastAccessAt = Date.now();
|
|
53
|
+
}
|
|
54
|
+
);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export { replaceExclusively, replaceUnregistered };
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
// src/narsil/restore-queue.ts
|
|
2
|
+
function restoreDepsOf(core, host) {
|
|
3
|
+
return {
|
|
4
|
+
...host,
|
|
5
|
+
executor: core.executor,
|
|
6
|
+
indexRegistry: core.indexRegistry,
|
|
7
|
+
durability: core.durability,
|
|
8
|
+
filesystemDurability: core.filesystemDurability,
|
|
9
|
+
embeddingAdapters: core.embeddingAdapters,
|
|
10
|
+
markAnalysisStale: core.analysisRebuild.markStale,
|
|
11
|
+
clearAnalysisStale: core.analysisRebuild.clearStale,
|
|
12
|
+
abandonAnalysisRebuild: core.analysisRebuild.abandon,
|
|
13
|
+
indexState: core.indexState,
|
|
14
|
+
copies: core.orchestrator,
|
|
15
|
+
forgetWatermark: core.watermarkNotifier.forget
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
function queueRestore(core, host, restoresInFlight, indexName, data) {
|
|
19
|
+
const previous = restoresInFlight.get(indexName) ?? Promise.resolve();
|
|
20
|
+
const restored = previous.then(async () => {
|
|
21
|
+
const { restoreFromSnapshot } = await import('./snapshot-Y3NGUJHX.mjs');
|
|
22
|
+
await restoreFromSnapshot(indexName, data, restoreDepsOf(core, host));
|
|
23
|
+
});
|
|
24
|
+
const settled = restored.then(
|
|
25
|
+
() => void 0,
|
|
26
|
+
() => void 0
|
|
27
|
+
);
|
|
28
|
+
restoresInFlight.set(indexName, settled);
|
|
29
|
+
void settled.then(() => {
|
|
30
|
+
if (restoresInFlight.get(indexName) === settled) restoresInFlight.delete(indexName);
|
|
31
|
+
});
|
|
32
|
+
return restored;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export { queueRestore };
|