@delali/narsil 0.3.1-next.90 → 0.3.1-next.94

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/dist/adapters/filesystem.mjs +1 -1
  2. package/dist/adapters/indexeddb.mjs +1 -1
  3. package/dist/browser/{chunk-3GZQKGVQ.mjs → chunk-4JTBL63P.mjs} +13 -5
  4. package/dist/browser/{chunk-3ZRMZIH4.mjs → chunk-5S74HQRT.mjs} +208 -78
  5. package/dist/browser/{chunk-FHFFWLEK.mjs → chunk-HARBSUWP.mjs} +3 -4
  6. package/dist/browser/{chunk-RAHUQOJL.mjs → chunk-HC6DMR4T.mjs} +2445 -2107
  7. package/dist/browser/{chunk-JACCR3BJ.mjs → chunk-PP6N5CCS.mjs} +2 -2
  8. package/dist/browser/{chunk-NQW2YXEX.mjs → chunk-WKPJ4E73.mjs} +1 -1
  9. package/dist/browser/{durability-wiring-VSUBLUDK.mjs → durability-wiring-7DKYHKT5.mjs} +498 -71
  10. package/dist/browser/{module-E25UDADM.mjs → module-O5WH32EL.mjs} +102 -54
  11. package/dist/browser/release-76MXRPGV.mjs +33 -0
  12. package/dist/browser/replace-RURXTGVG.mjs +57 -0
  13. package/dist/browser/restore-queue-Z7FZZNKH.mjs +35 -0
  14. package/dist/browser/{snapshot-5MXBEDXF.mjs → snapshot-PPTZ565G.mjs} +368 -103
  15. package/dist/browser/{write-6SBYTS62.mjs → write-2ZAA7UDH.mjs} +4 -4
  16. package/dist/{capabilities-SJPttcgQ.d.ts → capabilities-xsJfdFoQ.d.ts} +11 -3
  17. package/dist/chunk-2GZGBOQI.mjs +426 -0
  18. package/dist/{chunk-CVQWQVFX.mjs → chunk-4ZJWRETC.mjs} +2 -2
  19. package/dist/chunk-7KC56574.mjs +507 -0
  20. package/dist/{chunk-MXS5HWIH.mjs → chunk-7VALMHZF.mjs} +89 -17
  21. package/dist/chunk-7X5JST5L.mjs +114 -0
  22. package/dist/{chunk-UKSUENHK.mjs → chunk-AF2CIKYC.mjs} +40 -26
  23. package/dist/{chunk-BVOJIYRV.mjs → chunk-BB7MASRQ.mjs} +1 -1
  24. package/dist/{chunk-R6BMMQXH.mjs → chunk-C45FUYPI.mjs} +193 -79
  25. package/dist/{chunk-EMXAYPTQ.mjs → chunk-CHOGQSHK.mjs} +2326 -3503
  26. package/dist/{chunk-QZN3MIJ3.mjs → chunk-ES33L3HO.mjs} +2 -2
  27. package/dist/{chunk-VY4DXZ5P.mjs → chunk-EWBXO45L.mjs} +3 -3
  28. package/dist/{chunk-QAMUC2BJ.mjs → chunk-EX62BQT6.mjs} +10 -9
  29. package/dist/{chunk-MWJIAREY.mjs → chunk-EZD4MWOU.mjs} +4 -4
  30. package/dist/{chunk-YCJLVNWR.mjs → chunk-F52QGNZ2.mjs} +1 -1
  31. package/dist/{chunk-URGLSNKR.mjs → chunk-GJZHU4FC.mjs} +202 -16
  32. package/dist/{chunk-S4LQDSUL.mjs → chunk-J53IHLG7.mjs} +7 -7
  33. package/dist/{chunk-R7OSDAFK.mjs → chunk-K6A4FCVR.mjs} +1 -1
  34. package/dist/chunk-KOKM4PQN.mjs +52 -0
  35. package/dist/{chunk-GCM3JFVA.mjs → chunk-OHP7QR33.mjs} +2 -1
  36. package/dist/{chunk-T6CNO72J.mjs → chunk-QM3VPX4Z.mjs} +2 -2
  37. package/dist/{chunk-V7YUSKKZ.mjs → chunk-QWWCLRXX.mjs} +10 -6
  38. package/dist/chunk-QY7GC2MW.mjs +33 -0
  39. package/dist/{chunk-DCK4P5BI.mjs → chunk-R36MIGBU.mjs} +100 -59
  40. package/dist/{chunk-YRX6XMOQ.mjs → chunk-R6PR4WYZ.mjs} +1 -0
  41. package/dist/{chunk-RVM7AODE.mjs → chunk-RDJKYKMQ.mjs} +2 -2
  42. package/dist/{chunk-OOSIDHA3.mjs → chunk-RSRVGAIQ.mjs} +1 -1
  43. package/dist/{chunk-XSZVS4S3.mjs → chunk-RUIXODYU.mjs} +9 -4
  44. package/dist/{chunk-A3UB2MXL.mjs → chunk-SBCEHIPL.mjs} +4 -507
  45. package/dist/{chunk-TPJU7NDJ.mjs → chunk-SYWGORQB.mjs} +1 -0
  46. package/dist/chunk-UAUZGXJP.mjs +3591 -0
  47. package/dist/{chunk-CL4OM7GB.mjs → chunk-WBS3EYC4.mjs} +12 -8
  48. package/dist/{chunk-J2BPXTD7.mjs → chunk-WLOF6HSP.mjs} +3 -3
  49. package/dist/chunk-X5CV2E2B.mjs +1 -0
  50. package/dist/{chunk-ULRPNYAG.mjs → chunk-Z3MOCHG5.mjs} +120 -62
  51. package/dist/chunk-ZHS44YKF.mjs +35 -0
  52. package/dist/{client-BMK53IR2.d.ts → client-CgayPw2R.d.ts} +27 -3
  53. package/dist/client.d.ts +9 -8
  54. package/dist/client.mjs +16 -2
  55. package/dist/{config-UJx1dyX1.d.ts → config-3kBsQ0Pi.d.ts} +2 -2
  56. package/dist/{config-BUR-7eVQ.d.ts → config-B-RnO6dr.d.ts} +2 -2
  57. package/dist/direct-executor-IJTRU3YN.mjs +22 -0
  58. package/dist/distribution/coordinator/etcd.d.ts +3 -2
  59. package/dist/distribution/coordinator/etcd.mjs +5 -1
  60. package/dist/distribution/coordinator/in-memory.d.ts +3 -2
  61. package/dist/distribution/coordinator/in-memory.mjs +4 -0
  62. package/dist/distribution/transport/grpc.mjs +3 -3
  63. package/dist/distribution/transport/in-memory.mjs +1 -1
  64. package/dist/distribution/transport/tcp.mjs +2 -2
  65. package/dist/distribution.d.ts +43 -9
  66. package/dist/distribution.mjs +4620 -4293
  67. package/dist/{durability-wiring-ZLRQU6FG.mjs → durability-wiring-Y55SCPFC.mjs} +524 -93
  68. package/dist/embeddings/openai.mjs +1 -1
  69. package/dist/{engine-9X6W5naj.d.ts → engine-DSUfY2F0.d.ts} +53 -17
  70. package/dist/{index-vCMu8OeQ.d.ts → index-ONAEaqtx.d.ts} +8 -1
  71. package/dist/index.browser.mjs +2 -2
  72. package/dist/index.d.ts +44 -25
  73. package/dist/index.mjs +22 -18
  74. package/dist/{memory-CV9NAxLY.d.ts → memory-wuXcYOD1.d.ts} +99 -79
  75. package/dist/{module-WIDYIJLY.mjs → module-VITS4ZR5.mjs} +72 -38
  76. package/dist/partial-words.mjs +5 -4
  77. package/dist/pattern.mjs +3 -3
  78. package/dist/persistence/durability/checkpoint-worker.d.ts +60 -58
  79. package/dist/persistence/durability/checkpoint-worker.mjs +27 -23
  80. package/dist/react.d.ts +8 -7
  81. package/dist/react.mjs +1 -0
  82. package/dist/release-BH3DKW3T.mjs +33 -0
  83. package/dist/replace-CEO5KJJ3.mjs +57 -0
  84. package/dist/restore-queue-4SG45BLF.mjs +35 -0
  85. package/dist/{schema-DtsgmNfU.d.ts → schema-DttzM-ln.d.ts} +12 -0
  86. package/dist/server.d.ts +11 -10
  87. package/dist/server.mjs +38 -11
  88. package/dist/snapshot-XBKE7PUG.mjs +738 -0
  89. package/dist/synonyms-CpOAHPa-.d.ts +52 -0
  90. package/dist/synonyms.d.ts +22 -0
  91. package/dist/synonyms.mjs +12 -0
  92. package/dist/{tasks-BGaZJyoY.d.ts → tasks-Bj9WBlFG.d.ts} +4 -1
  93. package/dist/{thread-ZP3UMIZT.mjs → thread-U7XYJWGG.mjs} +17 -12
  94. package/dist/vector/search-worker.d.ts +2 -1
  95. package/dist/vector/search-worker.mjs +6 -6
  96. package/dist/workers/entry.mjs +6 -6
  97. package/dist/{write-PXDKPG2R.mjs → write-BMNV7P2H.mjs} +24 -20
  98. package/package.json +15 -7
  99. package/dist/chunk-PR4O7CAP.mjs +0 -18
  100. package/dist/chunk-X4SEXL4U.mjs +0 -2310
  101. package/dist/chunk-Y2F66IQJ.mjs +0 -74
  102. package/dist/chunk-ZJ4CSAB2.mjs +0 -13
  103. package/dist/direct-executor-EGESD26S.mjs +0 -18
  104. package/dist/snapshot-22CUEYYW.mjs +0 -469
@@ -1,8 +1,8 @@
1
1
  import { spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
2
- import { VECTOR_WORKER_COPIES_ALLOWED, openSharedQuantizer, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, osqBitsOf, createOsqQuantizer, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterValues, threadsHoldCurrentLayout, filterForOptions, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, OSQ4_MIN_DIMENSION, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, STORE_RELEASED_VECTORS_TO_DISK, STORE_CODE_COUNT, STORE_CALIBRATED, STORE_CALIBRATION_GENERATION, magnitude, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, WORKER_COPY_MIN_VECTORS, allLiveDocIds, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, BLOCK_MAX_BYTES, growBufferTo, unplacedChunkCount, STORE_DOC_ID_BYTES, recalibrateFromStore, signatureOf, nativeStoreScratchBytes, noteFallback, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, QUERY_STAGED, createVectorBlock, ensureBlockSlots, createHNSWIndex, adoptGraph, topLayerInParts, calibrateQuantizer, fieldHandlesOf, sharedMemoryAvailable, requireWholeNeighbourLists, BUILD_CHUNK_SIZE, yieldToEventLoop, releaseLocksHeldBy, DISK_ADOPTION_YIELD_INTERVAL } from './chunk-3ZRMZIH4.mjs';
3
- export { releaseLocksHeldBy } from './chunk-3ZRMZIH4.mjs';
4
- import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-NQW2YXEX.mjs';
5
- import { NarsilError, ErrorCodes, DISK_STORAGE_BLOCK_BYTES, compareCodePoints, UNPLACED_FLAG_CHUNK_ORDINALS, MAX_VECTOR_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, resolveWorkerCount, resolveWorkerEntry, detectRuntime, WORKER_YOUNG_GENERATION_MB, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS } from './chunk-FHFFWLEK.mjs';
2
+ import { VECTOR_WORKER_COPIES_ALLOWED, openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, osqBitsOf, createOsqQuantizer, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, ordinalFilterValues, threadsHoldCurrentLayout, filterForOptions, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, OSQ4_MIN_DIMENSION, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, WORKER_COPY_MIN_VECTORS, allLiveDocIds, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, BLOCK_MAX_BYTES, growBufferTo, unplacedChunkCount, STORE_DOC_ID_BYTES, recalibrateFromStore, signatureOf, nativeStoreScratchBytes, noteFallback, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, QUERY_STAGED, createVectorBlock, ensureBlockSlots, createHNSWIndex, adoptGraph, topLayerInParts, calibrateQuantizer, fieldHandlesOf, sharedMemoryAvailable, requireWholeNeighbourLists, BUILD_CHUNK_SIZE, yieldToEventLoop, releaseLocksHeldBy, DISK_ADOPTION_YIELD_INTERVAL } from './chunk-5S74HQRT.mjs';
3
+ export { releaseLocksHeldBy } from './chunk-5S74HQRT.mjs';
4
+ import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-WKPJ4E73.mjs';
5
+ import { NarsilError, ErrorCodes, MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, compareCodePoints, UNPLACED_FLAG_CHUNK_ORDINALS, MAX_VECTOR_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, resolveWorkerCount, resolveWorkerEntry, detectRuntime, WORKER_YOUNG_GENERATION_MB, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS } from './chunk-HARBSUWP.mjs';
6
6
 
7
7
  // src/core/heap.ts
8
8
  function createMaxHeap(compare) {
@@ -205,14 +205,21 @@ function scoreCandidates(store, candidates, query, k, metric, minSimilarity) {
205
205
  function createSharedVectorSearcher(options) {
206
206
  const { fieldName, view, holdsDocument } = options;
207
207
  const flags = readStoreFlags(view.store);
208
- function filterForDocIds(allowed) {
209
- const filter = createOrdinalFilter(view.store.slots);
210
- for (const docId of allowed) {
211
- const ordinal = view.ordinalOf(docId);
212
- if (ordinal === void 0 || flags.isRemoved(ordinal)) continue;
213
- addToOrdinalFilter(filter, ordinal);
214
- }
215
- return filter;
208
+ function ordinalLayoutGeneration() {
209
+ return Atomics.load(view.store.handles.header, STORE_ORDINAL_LAYOUT_GENERATION);
210
+ }
211
+ function ordinalSpace() {
212
+ const store = view.store;
213
+ const layoutGeneration = ordinalLayoutGeneration();
214
+ return {
215
+ fieldName,
216
+ layoutGeneration,
217
+ slots: store.slots,
218
+ header: store.handles.header,
219
+ ordinalOf: (docId) => view.ordinalOf(docId),
220
+ holdsCurrentOrdinal: (ordinal) => store.holdsOrdinal(ordinal) && store.partitionAt(ordinal) !== void 0,
221
+ isRemoved: (ordinal) => flags.isRemoved(ordinal)
222
+ };
216
223
  }
217
224
  function filterForPartitions(allowed) {
218
225
  const slots = view.store.slots;
@@ -226,7 +233,9 @@ function createSharedVectorSearcher(options) {
226
233
  return filter;
227
234
  }
228
235
  function filterFor(searchOptions) {
229
- if (searchOptions.filterDocIds !== void 0) return filterForDocIds(searchOptions.filterDocIds);
236
+ if (searchOptions.filterSubsets !== void 0) {
237
+ return ordinalFilterForSubsets(searchOptions.filterSubsets, ordinalSpace());
238
+ }
230
239
  if (searchOptions.filterPartitions !== void 0) return filterForPartitions(searchOptions.filterPartitions);
231
240
  return void 0;
232
241
  }
@@ -242,6 +251,11 @@ function createSharedVectorSearcher(options) {
242
251
  }
243
252
  return candidates;
244
253
  }
254
+ function answerOf(outcome) {
255
+ const results = outcome.results.filter((result) => holdsDocument(result.docId));
256
+ const holdsEveryHit = outcome.resolvedEveryHit && results.length === outcome.results.length;
257
+ return { results, matched: outcome.matched, matchedExact: outcome.matchedExact && holdsEveryHit };
258
+ }
245
259
  function scanned(scan) {
246
260
  return { results: scan.results, matched: scan.matched, matchedExact: true, resolvedEveryHit: true };
247
261
  }
@@ -300,21 +314,25 @@ function createSharedVectorSearcher(options) {
300
314
  { expected: view.handles.dimension, received: query.length }
301
315
  );
302
316
  }
303
- const outcome = search2(query, k, searchOptions, filterFor(searchOptions));
304
- if (view.graphRetired) {
305
- throw new NarsilError(
306
- ErrorCodes.INDEX_NOT_FOUND,
307
- `The main thread replaced the graph of vector field "${fieldName}" while this thread searched it`,
308
- { fieldName }
309
- );
317
+ for (let attempt = 1; ; attempt++) {
318
+ const layoutGeneration = ordinalLayoutGeneration();
319
+ const outcome = search2(query, k, searchOptions, filterFor(searchOptions));
320
+ if (view.graphRetired) {
321
+ throw new NarsilError(
322
+ ErrorCodes.WORKER_BUSY,
323
+ `The main thread replaced the graph of vector field "${fieldName}" while this thread searched it`,
324
+ { fieldName }
325
+ );
326
+ }
327
+ if (ordinalLayoutGeneration() === layoutGeneration) return Promise.resolve(answerOf(outcome));
328
+ if (attempt >= MAX_RENUMBERED_SEARCH_ATTEMPTS) {
329
+ throw new NarsilError(
330
+ ErrorCodes.WORKER_BUSY,
331
+ `The main thread renumbered the vectors of field "${fieldName}" during each of ${attempt} searches on this thread`,
332
+ { fieldName, attempts: attempt }
333
+ );
334
+ }
310
335
  }
311
- const results = outcome.results.filter((result) => holdsDocument(result.docId));
312
- const holdsEveryHit = outcome.resolvedEveryHit && results.length === outcome.results.length;
313
- return Promise.resolve({
314
- results,
315
- matched: outcome.matched,
316
- matchedExact: outcome.matchedExact && holdsEveryHit
317
- });
318
336
  }
319
337
  };
320
338
  }
@@ -686,6 +704,7 @@ function openSharedVectorField(initial, threadSlot) {
686
704
  let graph = null;
687
705
  let ordinals = null;
688
706
  let scanned = 0;
707
+ let scannedLayoutGeneration = Atomics.load(store.handles.header, STORE_ORDINAL_LAYOUT_GENERATION);
689
708
  let placed = 0;
690
709
  function openGraph(next) {
691
710
  graph = next.graph === null ? null : openGraphState(next.graph, next.dimension, store, quantizer, threadSlot);
@@ -696,6 +715,12 @@ function openSharedVectorField(initial, threadSlot) {
696
715
  return store.docIdAt(ordinal);
697
716
  }
698
717
  function ordinalOf(docId) {
718
+ const layoutGeneration = Atomics.load(store.handles.header, STORE_ORDINAL_LAYOUT_GENERATION);
719
+ if (layoutGeneration !== scannedLayoutGeneration) {
720
+ ordinals = null;
721
+ scanned = 0;
722
+ scannedLayoutGeneration = layoutGeneration;
723
+ }
699
724
  const slots = store.slots;
700
725
  if (ordinals === null) ordinals = /* @__PURE__ */ new Map();
701
726
  for (; scanned < slots; scanned++) {
@@ -863,6 +888,31 @@ function relayout(store) {
863
888
  store.handles.layoutRevision += 1;
864
889
  store.view.adoptHandles(store.handles);
865
890
  }
891
+ function claimSlot(store, docId, cold) {
892
+ const ordinal = Atomics.load(store.handles.header, STORE_SLOTS);
893
+ ensureOrdinal(store, ordinal, cold);
894
+ store.diskFile[ordinal] = IN_MEMORY;
895
+ appendDocId(store.handles, ordinal, docId);
896
+ return ordinal;
897
+ }
898
+ function publishOrdinal(store, ordinal) {
899
+ Atomics.store(store.handles.header, STORE_SLOTS, ordinal + 1);
900
+ Atomics.store(store.present, ordinal, 1);
901
+ }
902
+ function publishOnDisk(store, ordinal, vectorMagnitude, location) {
903
+ store.magnitudes[ordinal] = vectorMagnitude;
904
+ store.diskOffset[ordinal] = location.offset;
905
+ store.diskFile[ordinal] = location.fileIndex;
906
+ publishOrdinal(store, ordinal);
907
+ }
908
+ function relocateFiles(store, rewrite) {
909
+ const files = store.handles.vectorFiles;
910
+ if (files.length === 0) return;
911
+ for (let file = 0; file < files.length; file++) {
912
+ if (files[file].length > 0) files[file] = rewrite(files[file]);
913
+ }
914
+ relayout(store);
915
+ }
866
916
  function ensureCodeSlot(store, ordinal) {
867
917
  const { handles } = store;
868
918
  if (handles.codeLayout === null) return;
@@ -1031,13 +1081,6 @@ function createVectorStore(options) {
1031
1081
  Atomics.store(store.present, ordinal, 1);
1032
1082
  replacedOrdinals.add(ordinal);
1033
1083
  }
1034
- function claimSlot(store, docId, cold) {
1035
- const ordinal = Atomics.load(store.handles.header, STORE_SLOTS);
1036
- ensureOrdinal(store, ordinal, cold);
1037
- store.diskFile[ordinal] = IN_MEMORY;
1038
- appendDocId(store.handles, ordinal, docId);
1039
- return ordinal;
1040
- }
1041
1084
  function claimOrdinal(store, docId, cold, partitionId) {
1042
1085
  const ordinal = claimSlot(store, docId, cold);
1043
1086
  recordPartition(store, ordinal, partitionId);
@@ -1045,16 +1088,6 @@ function createVectorStore(options) {
1045
1088
  docToOrd.set(docId, ordinal);
1046
1089
  return ordinal;
1047
1090
  }
1048
- function publishOrdinal(store, ordinal) {
1049
- Atomics.store(store.handles.header, STORE_SLOTS, ordinal + 1);
1050
- Atomics.store(store.present, ordinal, 1);
1051
- }
1052
- function publishOnDisk(store, ordinal, vectorMagnitude, location) {
1053
- store.magnitudes[ordinal] = vectorMagnitude;
1054
- store.diskOffset[ordinal] = location.offset;
1055
- store.diskFile[ordinal] = location.fileIndex;
1056
- publishOrdinal(store, ordinal);
1057
- }
1058
1091
  function appendOrdinal(store, docId, vector, partitionId) {
1059
1092
  const ordinal = claimOrdinal(store, docId, false, partitionId);
1060
1093
  writeVector(store, ordinal, vector);
@@ -1225,19 +1258,18 @@ function createVectorStore(options) {
1225
1258
  relayout(open);
1226
1259
  }
1227
1260
  resetDocIds(open.handles);
1228
- Atomics.store(open.handles.header, STORE_SLOTS, 0);
1229
- Atomics.store(open.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 0);
1230
- Atomics.store(open.handles.header, STORE_LIVE_COUNT, 0);
1231
- Atomics.store(open.handles.header, STORE_CODE_COUNT, 0);
1232
- Atomics.store(open.handles.header, STORE_CALIBRATED, 0);
1233
- Atomics.add(open.handles.header, STORE_CALIBRATION_GENERATION, 1);
1261
+ resetStoreHeader(open.handles.header);
1234
1262
  },
1235
1263
  release() {
1236
1264
  forgetDocuments();
1265
+ if (open !== null) Atomics.add(open.handles.header, STORE_ORDINAL_LAYOUT_GENERATION, 1);
1237
1266
  open?.view.close();
1238
1267
  open = null;
1239
1268
  },
1240
1269
  closeFiles: () => open?.view.close(),
1270
+ relocateVectorFiles(rewrite) {
1271
+ if (open !== null) relocateFiles(open, rewrite);
1272
+ },
1241
1273
  getOrdinal(docId) {
1242
1274
  return docToOrd.get(docId);
1243
1275
  },
@@ -1606,6 +1638,11 @@ async function withdrawWorkerCopies(state) {
1606
1638
  await Promise.all([...handles].map((handle) => dropHandle(state, handle)));
1607
1639
  if (state.workerCopies.host === void 0 && state.sharedHandles.size === 0) state.workerCopyPool = null;
1608
1640
  }
1641
+ async function holdWorkerCopies(state) {
1642
+ state.workerCopiesHeld = true;
1643
+ await state.sharing;
1644
+ await withdrawWorkerCopies(state);
1645
+ }
1609
1646
  function noteWrite(state) {
1610
1647
  state.revision += 1;
1611
1648
  if (state.workerCopyMode === "clone") invalidateWorkerCopies(state);
@@ -1704,7 +1741,7 @@ async function loadOnPool(state, graph, searchable) {
1704
1741
  }
1705
1742
  function shareGraph(state, graph, searchable) {
1706
1743
  const run = state.sharing.then(async () => {
1707
- if (!state.workerCopies.enabled || state.disposed) return null;
1744
+ if (!state.workerCopies.enabled || state.disposed || state.workerCopiesHeld) return null;
1708
1745
  const existing = state.sharedHandles.get(graph);
1709
1746
  const layoutUnchanged = threadsHoldCurrentLayout(state);
1710
1747
  if (existing !== void 0 && layoutUnchanged && (existing.searchable || !searchable)) return existing.handle;
@@ -1719,7 +1756,7 @@ function shareGraph(state, graph, searchable) {
1719
1756
  }
1720
1757
  function resendSharedHandles(state) {
1721
1758
  const run = state.sharing.then(async () => {
1722
- if (!state.workerCopies.enabled || state.disposed) return;
1759
+ if (!state.workerCopies.enabled || state.disposed || state.workerCopiesHeld) return;
1723
1760
  const host = state.workerCopies.host;
1724
1761
  for (const [graph, shared] of [...state.sharedHandles]) {
1725
1762
  if (host !== void 0) await loadOnHost(state, host, graph, shared.searchable);
@@ -1733,7 +1770,7 @@ function resendSharedHandles(state) {
1733
1770
  return run;
1734
1771
  }
1735
1772
  function scheduleWorkerCopyLoad(state) {
1736
- if (!state.workerCopies.enabled) return;
1773
+ if (!state.workerCopies.enabled || state.workerCopiesHeld) return;
1737
1774
  if (state.disposed || state.workerCopyLoading || state.building) return;
1738
1775
  const graph = state.hnsw;
1739
1776
  const host = state.workerCopies.host;
@@ -2581,6 +2618,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
2581
2618
  workerCopyRevision: -1,
2582
2619
  workerCopyMode: null,
2583
2620
  workerCopyLoading: false,
2621
+ workerCopiesHeld: false,
2584
2622
  sharedHandles: /* @__PURE__ */ new Map(),
2585
2623
  sharedLayoutRevision: 0,
2586
2624
  sharing: Promise.resolve(),
@@ -2678,7 +2716,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
2678
2716
  releaseHeldMemory();
2679
2717
  }
2680
2718
  async function searchParallel(query, k, options) {
2681
- const confined = options.filterDocIds !== void 0 || options.filterPartitions !== void 0;
2719
+ const confined = options.filterSubsets !== void 0 || options.filterPartitions !== void 0;
2682
2720
  let filter = filterForOptions(state, options);
2683
2721
  if (filter !== void 0) {
2684
2722
  if (state.hnsw) {
@@ -2759,6 +2797,16 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
2759
2797
  releaseVectorFiles: async () => {
2760
2798
  await withdrawWorkerCopies(state);
2761
2799
  state.store.closeFiles();
2800
+ },
2801
+ relocateVectorFiles: (rewrite) => {
2802
+ state.store.relocateVectorFiles(rewrite);
2803
+ for (const [docId, location] of state.pendingLocations) {
2804
+ state.pendingLocations.set(docId, { ...location, path: rewrite(location.path) });
2805
+ }
2806
+ },
2807
+ holdWorkerCopies: () => holdWorkerCopies(state),
2808
+ resumeWorkerCopies: () => {
2809
+ state.workerCopiesHeld = false;
2762
2810
  }
2763
2811
  };
2764
2812
  }
@@ -0,0 +1,33 @@
1
+ // src/engine/orchestration/release.ts
2
+ async function releaseIndexCopies(state, indexName, steps) {
3
+ const wasScaledOut = state.scaledOutIndexes.delete(indexName);
4
+ const loading = state.copyTransitions.get(indexName);
5
+ if (loading !== void 0) await loading.done.catch(() => void 0);
6
+ await steps.awaitReplicationIdle(state, indexName);
7
+ steps.cancelIdleMerge(state, indexName);
8
+ const reloads = wasScaledOut || loading !== void 0 && loading.kind !== "drop";
9
+ const pool = state.workerPool;
10
+ if (pool === null) return reloads;
11
+ pool.removeIndex(indexName);
12
+ state.segmentLedger.delete(indexName);
13
+ state.unbroadcastSegments.delete(indexName);
14
+ await Promise.all(
15
+ pool.executorEntries().map(
16
+ ({ executor }) => executor.execute({ type: "dropIndex", indexName, requestId: `restore-release-${indexName}` }).catch((error) => {
17
+ if (!steps.heldNowhere(error)) throw error;
18
+ })
19
+ )
20
+ );
21
+ return reloads;
22
+ }
23
+ async function endCopyRelease(state, indexName, reloads, steps) {
24
+ const hold = state.copyReleaseHolds.get(indexName);
25
+ if (hold === void 0) return steps.resyncIndex(indexName, reloads);
26
+ hold.holders -= 1;
27
+ hold.reloads ||= reloads;
28
+ if (hold.holders > 0) return;
29
+ state.copyReleaseHolds.delete(indexName);
30
+ await steps.resyncIndex(indexName, hold.reloads);
31
+ }
32
+
33
+ export { endCopyRelease, releaseIndexCopies };
@@ -0,0 +1,57 @@
1
+ // src/engine/index-state/replace.ts
2
+ function runReplacement(context, entry, body, settle) {
3
+ const previous = entry.state;
4
+ entry.state = "replacing";
5
+ const run = (async () => {
6
+ await context.waitForDrain(entry);
7
+ await body();
8
+ })().then(
9
+ () => settle(true, previous),
10
+ (error) => {
11
+ settle(false, previous);
12
+ throw error;
13
+ }
14
+ ).finally(() => {
15
+ entry.replacePromise = null;
16
+ });
17
+ entry.replacePromise = run;
18
+ return run;
19
+ }
20
+ function replaceUnregistered(context, indexName, entry, action) {
21
+ return runReplacement(context, entry, action, (succeeded) => {
22
+ if (!succeeded) {
23
+ context.forget(indexName);
24
+ return;
25
+ }
26
+ entry.state = "open";
27
+ entry.lastAccessAt = Date.now();
28
+ });
29
+ }
30
+ async function replaceExclusively(context, indexName, action) {
31
+ let entry = context.entryOf(indexName);
32
+ while (true) {
33
+ const pending = entry.dropPromise ?? entry.replacePromise ?? entry.reopenPromise ?? entry.closePromise;
34
+ if (pending === null) break;
35
+ await pending.catch(() => void 0);
36
+ entry = context.entryOf(indexName);
37
+ }
38
+ const wasOpen = entry.state === "open";
39
+ await runReplacement(
40
+ context,
41
+ entry,
42
+ () => action(wasOpen),
43
+ (succeeded, previous) => {
44
+ if (!succeeded) {
45
+ entry.state = previous;
46
+ return;
47
+ }
48
+ entry.state = "open";
49
+ entry.cachedError = null;
50
+ entry.reopenFailures = 0;
51
+ entry.nextReopenAt = 0;
52
+ entry.lastAccessAt = Date.now();
53
+ }
54
+ );
55
+ }
56
+
57
+ export { replaceExclusively, replaceUnregistered };
@@ -0,0 +1,35 @@
1
+ // src/narsil/restore-queue.ts
2
+ function restoreDepsOf(core, host) {
3
+ return {
4
+ ...host,
5
+ executor: core.executor,
6
+ indexRegistry: core.indexRegistry,
7
+ durability: core.durability,
8
+ filesystemDurability: core.filesystemDurability,
9
+ embeddingAdapters: core.embeddingAdapters,
10
+ markAnalysisStale: core.analysisRebuild.markStale,
11
+ clearAnalysisStale: core.analysisRebuild.clearStale,
12
+ abandonAnalysisRebuild: core.analysisRebuild.abandon,
13
+ indexState: core.indexState,
14
+ copies: core.orchestrator,
15
+ forgetWatermark: core.watermarkNotifier.forget
16
+ };
17
+ }
18
+ function queueRestore(core, host, restoresInFlight, indexName, data) {
19
+ const previous = restoresInFlight.get(indexName) ?? Promise.resolve();
20
+ const restored = previous.then(async () => {
21
+ const { restoreFromSnapshot } = await import('./snapshot-PPTZ565G.mjs');
22
+ await restoreFromSnapshot(indexName, data, restoreDepsOf(core, host));
23
+ });
24
+ const settled = restored.then(
25
+ () => void 0,
26
+ () => void 0
27
+ );
28
+ restoresInFlight.set(indexName, settled);
29
+ void settled.then(() => {
30
+ if (restoresInFlight.get(indexName) === settled) restoresInFlight.delete(indexName);
31
+ });
32
+ return restored;
33
+ }
34
+
35
+ export { queueRestore };