@delali/narsil 0.3.1-next.91 → 0.3.1-next.94

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/dist/adapters/filesystem.mjs +1 -1
  2. package/dist/adapters/indexeddb.mjs +1 -1
  3. package/dist/browser/{chunk-B6XE7KXY.mjs → chunk-4JTBL63P.mjs} +4 -4
  4. package/dist/browser/{chunk-3ZRMZIH4.mjs → chunk-5S74HQRT.mjs} +208 -78
  5. package/dist/browser/{chunk-FHFFWLEK.mjs → chunk-HARBSUWP.mjs} +3 -4
  6. package/dist/browser/{chunk-NOUZ6RH3.mjs → chunk-HC6DMR4T.mjs} +238 -114
  7. package/dist/browser/{chunk-66KYAOIK.mjs → chunk-PP6N5CCS.mjs} +2 -2
  8. package/dist/browser/{chunk-NQW2YXEX.mjs → chunk-WKPJ4E73.mjs} +1 -1
  9. package/dist/browser/{durability-wiring-F4OGQUSU.mjs → durability-wiring-7DKYHKT5.mjs} +498 -71
  10. package/dist/browser/{module-E25UDADM.mjs → module-O5WH32EL.mjs} +102 -54
  11. package/dist/browser/release-76MXRPGV.mjs +33 -0
  12. package/dist/browser/replace-RURXTGVG.mjs +57 -0
  13. package/dist/browser/restore-queue-Z7FZZNKH.mjs +35 -0
  14. package/dist/browser/{snapshot-5EJ7I7QY.mjs → snapshot-PPTZ565G.mjs} +362 -103
  15. package/dist/browser/{write-K5V4XQ72.mjs → write-2ZAA7UDH.mjs} +4 -4
  16. package/dist/{capabilities-CvExBThq.d.ts → capabilities-xsJfdFoQ.d.ts} +2 -2
  17. package/dist/{chunk-NVL4ZR3K.mjs → chunk-2GZGBOQI.mjs} +2 -2
  18. package/dist/{chunk-CVQWQVFX.mjs → chunk-4ZJWRETC.mjs} +2 -2
  19. package/dist/{chunk-ZIJ63TBT.mjs → chunk-7KC56574.mjs} +1 -1
  20. package/dist/{chunk-MXS5HWIH.mjs → chunk-7VALMHZF.mjs} +89 -17
  21. package/dist/{chunk-UKSUENHK.mjs → chunk-AF2CIKYC.mjs} +40 -26
  22. package/dist/{chunk-BVOJIYRV.mjs → chunk-BB7MASRQ.mjs} +1 -1
  23. package/dist/{chunk-A4G7JSYO.mjs → chunk-C45FUYPI.mjs} +98 -49
  24. package/dist/{chunk-6VZ3Q73N.mjs → chunk-CHOGQSHK.mjs} +44 -25
  25. package/dist/{chunk-QZN3MIJ3.mjs → chunk-ES33L3HO.mjs} +2 -2
  26. package/dist/{chunk-67Z2V6TC.mjs → chunk-EWBXO45L.mjs} +3 -3
  27. package/dist/{chunk-3TTYK5A7.mjs → chunk-EX62BQT6.mjs} +9 -9
  28. package/dist/{chunk-3RDS4UHC.mjs → chunk-EZD4MWOU.mjs} +4 -4
  29. package/dist/{chunk-YCJLVNWR.mjs → chunk-F52QGNZ2.mjs} +1 -1
  30. package/dist/{chunk-43ORORN6.mjs → chunk-GJZHU4FC.mjs} +3 -3
  31. package/dist/{chunk-7OQHSDK6.mjs → chunk-J53IHLG7.mjs} +7 -7
  32. package/dist/{chunk-R7OSDAFK.mjs → chunk-K6A4FCVR.mjs} +1 -1
  33. package/dist/chunk-KOKM4PQN.mjs +52 -0
  34. package/dist/{chunk-T6CNO72J.mjs → chunk-QM3VPX4Z.mjs} +2 -2
  35. package/dist/{chunk-T4DB5YFB.mjs → chunk-QWWCLRXX.mjs} +6 -6
  36. package/dist/chunk-QY7GC2MW.mjs +33 -0
  37. package/dist/{chunk-BJ2YKRNX.mjs → chunk-R36MIGBU.mjs} +86 -54
  38. package/dist/{chunk-RVM7AODE.mjs → chunk-RDJKYKMQ.mjs} +2 -2
  39. package/dist/{chunk-UXQE2ROB.mjs → chunk-RUIXODYU.mjs} +3 -2
  40. package/dist/{chunk-5ER245FI.mjs → chunk-SBCEHIPL.mjs} +3 -3
  41. package/dist/{chunk-TPJU7NDJ.mjs → chunk-SYWGORQB.mjs} +1 -0
  42. package/dist/{chunk-XLJ2S2BH.mjs → chunk-UAUZGXJP.mjs} +40 -21
  43. package/dist/{chunk-EONIHG3V.mjs → chunk-WBS3EYC4.mjs} +7 -7
  44. package/dist/{chunk-CO4MRKST.mjs → chunk-WLOF6HSP.mjs} +2 -2
  45. package/dist/{chunk-ULRPNYAG.mjs → chunk-Z3MOCHG5.mjs} +120 -62
  46. package/dist/{chunk-MKWUXWQH.mjs → chunk-ZHS44YKF.mjs} +1 -1
  47. package/dist/{client-Cc42b42j.d.ts → client-CgayPw2R.d.ts} +2 -2
  48. package/dist/client.d.ts +6 -6
  49. package/dist/client.mjs +1 -0
  50. package/dist/{config-DyRowvVP.d.ts → config-3kBsQ0Pi.d.ts} +1 -1
  51. package/dist/{config-BYne-yCI.d.ts → config-B-RnO6dr.d.ts} +1 -1
  52. package/dist/direct-executor-IJTRU3YN.mjs +22 -0
  53. package/dist/distribution/coordinator/etcd.mjs +1 -1
  54. package/dist/distribution/transport/grpc.mjs +1 -1
  55. package/dist/distribution.d.ts +5 -5
  56. package/dist/distribution.mjs +26 -26
  57. package/dist/{durability-wiring-6AN4QTT3.mjs → durability-wiring-Y55SCPFC.mjs} +518 -91
  58. package/dist/embeddings/openai.mjs +1 -1
  59. package/dist/{engine-96_kLtSY.d.ts → engine-DSUfY2F0.d.ts} +11 -11
  60. package/dist/index.browser.mjs +2 -2
  61. package/dist/index.d.ts +5 -5
  62. package/dist/index.mjs +18 -18
  63. package/dist/{memory-dd3GESdc.d.ts → memory-wuXcYOD1.d.ts} +96 -76
  64. package/dist/{module-WIDYIJLY.mjs → module-VITS4ZR5.mjs} +72 -38
  65. package/dist/partial-words.mjs +5 -5
  66. package/dist/pattern.mjs +3 -3
  67. package/dist/persistence/durability/checkpoint-worker.mjs +21 -21
  68. package/dist/react.d.ts +6 -6
  69. package/dist/react.mjs +1 -0
  70. package/dist/release-BH3DKW3T.mjs +33 -0
  71. package/dist/replace-CEO5KJJ3.mjs +57 -0
  72. package/dist/restore-queue-4SG45BLF.mjs +35 -0
  73. package/dist/server.d.ts +8 -8
  74. package/dist/server.mjs +10 -10
  75. package/dist/snapshot-XBKE7PUG.mjs +738 -0
  76. package/dist/synonyms.mjs +3 -3
  77. package/dist/{tasks-DA1Ff0G4.d.ts → tasks-Bj9WBlFG.d.ts} +1 -1
  78. package/dist/{thread-VTDLLYXC.mjs → thread-U7XYJWGG.mjs} +15 -12
  79. package/dist/vector/search-worker.mjs +6 -6
  80. package/dist/workers/entry.mjs +5 -5
  81. package/dist/{write-OJR3H4BI.mjs → write-BMNV7P2H.mjs} +20 -20
  82. package/package.json +7 -7
  83. package/dist/chunk-Y2F66IQJ.mjs +0 -74
  84. package/dist/chunk-ZJ4CSAB2.mjs +0 -13
  85. package/dist/direct-executor-CHXVCBN7.mjs +0 -22
  86. package/dist/snapshot-JFZVOGZV.mjs +0 -479
@@ -1,9 +1,76 @@
1
- import { VECTOR_FILE_MAX_VECTORS, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors, VECTOR_GRAPH_PART_VECTORS, VECTOR_GRAPH_PART_PAYLOAD_VERSION, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, vectorsToBytes } from './chunk-YCJLVNWR.mjs';
2
- import { createOrdinalFilter, addToOrdinalFilter, osqBitsOf, removeFromOrdinalFilter, createHNSWIndex, STORE_CALIBRATION_GENERATION, graphWithoutNodes } from './chunk-ULRPNYAG.mjs';
1
+ import { VECTOR_FILE_MAX_VECTORS, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors, VECTOR_GRAPH_PART_VECTORS, VECTOR_GRAPH_PART_PAYLOAD_VERSION, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, vectorsToBytes } from './chunk-F52QGNZ2.mjs';
2
+ import { createOrdinalFilter, addToOrdinalFilter, osqBitsOf, STORE_ORDINAL_LAYOUT_GENERATION, removeFromOrdinalFilter, createHNSWIndex, STORE_CALIBRATION_GENERATION, graphWithoutNodes } from './chunk-Z3MOCHG5.mjs';
3
3
  import { osqRecordBytes } from './chunk-37CD3OKS.mjs';
4
- import { MAX_FILE_BYTES } from './chunk-RVM7AODE.mjs';
4
+ import { MAX_FILE_BYTES } from './chunk-RDJKYKMQ.mjs';
5
5
  import { MAX_DOC_ID_LENGTH } from './chunk-Q5DC6WAB.mjs';
6
- import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
6
+ import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
7
+
8
+ // src/vector/filter-ordinals.ts
9
+ var UNRESOLVED_ORDINAL = -1;
10
+ var NO_VECTOR_MARK = -2;
11
+ function freshOrdinals(length) {
12
+ return new Int32Array(length).fill(UNRESOLVED_ORDINAL);
13
+ }
14
+ function mapsOf(owner) {
15
+ owner.vectorOrdinalMaps ??= { bytes: 0, byField: /* @__PURE__ */ new Map() };
16
+ return owner.vectorOrdinalMaps;
17
+ }
18
+ function mapFor(maps, docStore, space, numberingGeneration) {
19
+ const held = maps.byField.get(space.fieldName);
20
+ if (held !== void 0 && held.header === space.header && held.layoutGeneration === space.layoutGeneration && held.numberingGeneration === numberingGeneration) {
21
+ return held;
22
+ }
23
+ const map = {
24
+ header: space.header,
25
+ layoutGeneration: space.layoutGeneration,
26
+ numberingGeneration,
27
+ ordinals: freshOrdinals(docStore.internalIdCapacity())
28
+ };
29
+ maps.bytes += map.ordinals.byteLength - (held?.ordinals.byteLength ?? 0);
30
+ maps.byField.set(space.fieldName, map);
31
+ return map;
32
+ }
33
+ function growTo(maps, map, length) {
34
+ const grown = freshOrdinals(Math.max(length, map.ordinals.length * 2));
35
+ grown.set(map.ordinals);
36
+ maps.bytes += grown.byteLength - map.ordinals.byteLength;
37
+ map.ordinals = grown;
38
+ }
39
+ function ordinalAt(maps, map, internalId, docStore, space) {
40
+ if (internalId >= map.ordinals.length) growTo(maps, map, internalId + 1);
41
+ const held = map.ordinals[internalId];
42
+ if (held >= 0 && space.holdsCurrentOrdinal(held)) return held;
43
+ if (held <= NO_VECTOR_MARK && NO_VECTOR_MARK - held === space.slots) return UNRESOLVED_ORDINAL;
44
+ const docId = docStore.getExternalId(internalId);
45
+ const resolved = docId === void 0 ? void 0 : space.ordinalOf(docId);
46
+ map.ordinals[internalId] = resolved ?? NO_VECTOR_MARK - space.slots;
47
+ return resolved ?? UNRESOLVED_ORDINAL;
48
+ }
49
+ function ordinalFilterForSubsets(subsets, space) {
50
+ const filter = createOrdinalFilter(space.slots);
51
+ for (const subset of subsets) {
52
+ const docStore = subset.owner.docStore;
53
+ if (docStore.numberingGeneration() !== subset.numberingGeneration) continue;
54
+ const resolver = docStore.resolver();
55
+ const maps = mapsOf(subset.owner);
56
+ const map = mapFor(maps, docStore, space, subset.numberingGeneration);
57
+ const bits = subset.bits;
58
+ for (let wordIndex = 0; wordIndex < bits.length; wordIndex++) {
59
+ let word = bits[wordIndex];
60
+ if (word === 0) continue;
61
+ const base = wordIndex << 5;
62
+ while (word !== 0) {
63
+ const internalId = base + (Math.clz32(word & -word) ^ 31);
64
+ word &= word - 1;
65
+ if (!resolver.isLive(internalId)) continue;
66
+ const ordinal = ordinalAt(maps, map, internalId, docStore, space);
67
+ if (ordinal < 0 || space.isRemoved(ordinal)) continue;
68
+ addToOrdinalFilter(filter, ordinal);
69
+ }
70
+ }
71
+ }
72
+ return filter;
73
+ }
7
74
 
8
75
  // src/vector/vector-index/constants.ts
9
76
  var DEFAULT_PROMOTION_THRESHOLD = 1024;
@@ -94,25 +161,30 @@ function ordinalFilterForPartitions(state, partitionIds) {
94
161
  }
95
162
  return filter;
96
163
  }
164
+ function ordinalSpaceOf(state) {
165
+ const store = state.store;
166
+ const handles = store.handles;
167
+ const removed = new Uint8Array(handles.removed);
168
+ return {
169
+ fieldName: state.fieldName,
170
+ slots: store.slots,
171
+ header: handles.header,
172
+ layoutGeneration: Atomics.load(handles.header, STORE_ORDINAL_LAYOUT_GENERATION),
173
+ ordinalOf: (docId) => store.getOrdinal(docId),
174
+ holdsCurrentOrdinal: (ordinal) => store.holdsOrdinal(ordinal),
175
+ isRemoved: (ordinal) => ordinal < removed.length && removed[ordinal] === 1
176
+ };
177
+ }
97
178
  function filterForOptions(state, options) {
98
- if (options.filterDocIds !== void 0) {
99
- return ordinalFilterForDocIds(state, options.filterDocIds);
179
+ if (options.filterSubsets !== void 0) {
180
+ if (state.store.dimension === 0) return createOrdinalFilter(0);
181
+ return ordinalFilterForSubsets(options.filterSubsets, ordinalSpaceOf(state));
100
182
  }
101
183
  if (options.filterPartitions !== void 0) {
102
184
  return ordinalFilterForPartitions(state, options.filterPartitions);
103
185
  }
104
186
  return void 0;
105
187
  }
106
- function ordinalFilterForDocIds(state, docIds) {
107
- const filter = createOrdinalFilter(state.store.slots);
108
- for (const docId of docIds) {
109
- if (state.tombstones.has(docId)) continue;
110
- const ordinal = state.store.getOrdinal(docId);
111
- if (ordinal === void 0) continue;
112
- addToOrdinalFilter(filter, ordinal);
113
- }
114
- return filter;
115
- }
116
188
  function liveOrdinals(state) {
117
189
  const ordinals = [];
118
190
  for (let ordinal = 0; ordinal < state.store.slots; ordinal++) {
@@ -512,4 +584,4 @@ function recordCheckpoint(state, plan, written) {
512
584
  state.savedSignature = plan.signature;
513
585
  }
514
586
 
515
- export { BUILD_CHUNK_SIZE, DEFAULT_FILTER_THRESHOLD, DEFAULT_PROMOTION_THRESHOLD, DISK_ADOPTION_YIELD_INTERVAL, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, OSQ4_MIN_DIMENSION, VECTOR_WORKER_COPIES_ALLOWED, WORKER_COPY_MIN_VECTORS, adoptGraph, allLiveDocIds, assignStorePartitions, calibrateQuantizer, deserialize, emptyFieldBeforeRestore, fieldHandlesOf, filterForOptions, graphNeedsRebuild, liveSize, planCheckpoint, recalibrateFromStore, recordCheckpoint, releaseReplacedVectors, signatureOf, threadsHoldCurrentLayout, vectorsPerFile, yieldToEventLoop };
587
+ export { BUILD_CHUNK_SIZE, DEFAULT_FILTER_THRESHOLD, DEFAULT_PROMOTION_THRESHOLD, DISK_ADOPTION_YIELD_INTERVAL, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, OSQ4_MIN_DIMENSION, VECTOR_WORKER_COPIES_ALLOWED, WORKER_COPY_MIN_VECTORS, adoptGraph, allLiveDocIds, assignStorePartitions, calibrateQuantizer, deserialize, emptyFieldBeforeRestore, fieldHandlesOf, filterForOptions, graphNeedsRebuild, liveSize, ordinalFilterForSubsets, planCheckpoint, recalibrateFromStore, recordCheckpoint, releaseReplacedVectors, signatureOf, threadsHoldCurrentLayout, vectorsPerFile, yieldToEventLoop };
@@ -1,6 +1,6 @@
1
- import { openSharedQuantizer, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, BLOCK_MAX_BYTES, sharedVectorStoreBytes, STORE_SLOTS, STORE_LIVE_COUNT, IN_MEMORY, STORE_RELEASED_VECTORS_TO_DISK, STORE_CODE_COUNT, STORE_CALIBRATED, STORE_CALIBRATION_GENERATION, magnitude, openGraphState, fixedView, createSharedVectorStoreHandles, growBufferTo, unplacedChunkCount, STORE_DOC_ID_BYTES, createOrdinalFilter, addToOrdinalFilter, slotByteOffset, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createVectorBlock, ensureBlockSlots, QUERY_STAGED, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes } from './chunk-ULRPNYAG.mjs';
2
- import { MAX_VECTOR_ORDINALS } from './chunk-Y2F66IQJ.mjs';
3
- import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
1
+ import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, BLOCK_MAX_BYTES, sharedVectorStoreBytes, STORE_SLOTS, STORE_LIVE_COUNT, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, openGraphState, fixedView, createSharedVectorStoreHandles, growBufferTo, unplacedChunkCount, STORE_DOC_ID_BYTES, createOrdinalFilter, addToOrdinalFilter, slotByteOffset, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createVectorBlock, ensureBlockSlots, QUERY_STAGED, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes } from './chunk-Z3MOCHG5.mjs';
2
+ import { MAX_VECTOR_ORDINALS } from './chunk-QY7GC2MW.mjs';
3
+ import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
4
4
  import { readSync, closeSync, openSync } from 'fs';
5
5
 
6
6
  function createVectorFileReader() {
@@ -404,6 +404,7 @@ function openSharedVectorField(initial, threadSlot) {
404
404
  let graph = null;
405
405
  let ordinals = null;
406
406
  let scanned = 0;
407
+ let scannedLayoutGeneration = Atomics.load(store.handles.header, STORE_ORDINAL_LAYOUT_GENERATION);
407
408
  let placed = 0;
408
409
  function openGraph(next) {
409
410
  graph = next.graph === null ? null : openGraphState(next.graph, next.dimension, store, quantizer, threadSlot);
@@ -414,6 +415,12 @@ function openSharedVectorField(initial, threadSlot) {
414
415
  return store.docIdAt(ordinal);
415
416
  }
416
417
  function ordinalOf(docId) {
418
+ const layoutGeneration = Atomics.load(store.handles.header, STORE_ORDINAL_LAYOUT_GENERATION);
419
+ if (layoutGeneration !== scannedLayoutGeneration) {
420
+ ordinals = null;
421
+ scanned = 0;
422
+ scannedLayoutGeneration = layoutGeneration;
423
+ }
417
424
  const slots = store.slots;
418
425
  if (ordinals === null) ordinals = /* @__PURE__ */ new Map();
419
426
  for (; scanned < slots; scanned++) {
@@ -581,6 +588,31 @@ function relayout(store) {
581
588
  store.handles.layoutRevision += 1;
582
589
  store.view.adoptHandles(store.handles);
583
590
  }
591
+ function claimSlot(store, docId, cold) {
592
+ const ordinal = Atomics.load(store.handles.header, STORE_SLOTS);
593
+ ensureOrdinal(store, ordinal, cold);
594
+ store.diskFile[ordinal] = IN_MEMORY;
595
+ appendDocId(store.handles, ordinal, docId);
596
+ return ordinal;
597
+ }
598
+ function publishOrdinal(store, ordinal) {
599
+ Atomics.store(store.handles.header, STORE_SLOTS, ordinal + 1);
600
+ Atomics.store(store.present, ordinal, 1);
601
+ }
602
+ function publishOnDisk(store, ordinal, vectorMagnitude, location) {
603
+ store.magnitudes[ordinal] = vectorMagnitude;
604
+ store.diskOffset[ordinal] = location.offset;
605
+ store.diskFile[ordinal] = location.fileIndex;
606
+ publishOrdinal(store, ordinal);
607
+ }
608
+ function relocateFiles(store, rewrite) {
609
+ const files = store.handles.vectorFiles;
610
+ if (files.length === 0) return;
611
+ for (let file = 0; file < files.length; file++) {
612
+ if (files[file].length > 0) files[file] = rewrite(files[file]);
613
+ }
614
+ relayout(store);
615
+ }
584
616
  function ensureCodeSlot(store, ordinal) {
585
617
  const { handles } = store;
586
618
  if (handles.codeLayout === null) return;
@@ -749,13 +781,6 @@ function createVectorStore(options) {
749
781
  Atomics.store(store.present, ordinal, 1);
750
782
  replacedOrdinals.add(ordinal);
751
783
  }
752
- function claimSlot(store, docId, cold) {
753
- const ordinal = Atomics.load(store.handles.header, STORE_SLOTS);
754
- ensureOrdinal(store, ordinal, cold);
755
- store.diskFile[ordinal] = IN_MEMORY;
756
- appendDocId(store.handles, ordinal, docId);
757
- return ordinal;
758
- }
759
784
  function claimOrdinal(store, docId, cold, partitionId) {
760
785
  const ordinal = claimSlot(store, docId, cold);
761
786
  recordPartition(store, ordinal, partitionId);
@@ -763,16 +788,6 @@ function createVectorStore(options) {
763
788
  docToOrd.set(docId, ordinal);
764
789
  return ordinal;
765
790
  }
766
- function publishOrdinal(store, ordinal) {
767
- Atomics.store(store.handles.header, STORE_SLOTS, ordinal + 1);
768
- Atomics.store(store.present, ordinal, 1);
769
- }
770
- function publishOnDisk(store, ordinal, vectorMagnitude, location) {
771
- store.magnitudes[ordinal] = vectorMagnitude;
772
- store.diskOffset[ordinal] = location.offset;
773
- store.diskFile[ordinal] = location.fileIndex;
774
- publishOrdinal(store, ordinal);
775
- }
776
791
  function appendOrdinal(store, docId, vector, partitionId) {
777
792
  const ordinal = claimOrdinal(store, docId, false, partitionId);
778
793
  writeVector(store, ordinal, vector);
@@ -943,19 +958,18 @@ function createVectorStore(options) {
943
958
  relayout(open);
944
959
  }
945
960
  resetDocIds(open.handles);
946
- Atomics.store(open.handles.header, STORE_SLOTS, 0);
947
- Atomics.store(open.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 0);
948
- Atomics.store(open.handles.header, STORE_LIVE_COUNT, 0);
949
- Atomics.store(open.handles.header, STORE_CODE_COUNT, 0);
950
- Atomics.store(open.handles.header, STORE_CALIBRATED, 0);
951
- Atomics.add(open.handles.header, STORE_CALIBRATION_GENERATION, 1);
961
+ resetStoreHeader(open.handles.header);
952
962
  },
953
963
  release() {
954
964
  forgetDocuments();
965
+ if (open !== null) Atomics.add(open.handles.header, STORE_ORDINAL_LAYOUT_GENERATION, 1);
955
966
  open?.view.close();
956
967
  open = null;
957
968
  },
958
969
  closeFiles: () => open?.view.close(),
970
+ relocateVectorFiles(rewrite) {
971
+ if (open !== null) relocateFiles(open, rewrite);
972
+ },
959
973
  getOrdinal(docId) {
960
974
  return docToOrd.get(docId);
961
975
  },
@@ -1,5 +1,5 @@
1
1
  import { PARSED_FIELD_TYPE_CACHE_ENTRIES, VECTOR_PATTERN, MAX_STORED_VECTOR_DIMENSION } from './chunk-L5ORED7V.mjs';
2
- import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
2
+ import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
3
3
 
4
4
  // src/schema/validator/field-type.ts
5
5
  var FIELD_OPTION_ORDER = ["sortable", "pattern", "partial"];
@@ -1,21 +1,21 @@
1
- import { splitWorkerBudget, resolveWorkerCount, NATIVE_CRC32_MIN_BYTES, resolveWorkerEntry, missingWorkerEntry, MAX_FILE_BYTES, resolveRequestThreadCount, CHECKSUM_YIELD_CHUNK_BYTES, workerResourceLimits, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH } from './chunk-RVM7AODE.mjs';
2
- import { isEmptySynonymSet, synonymModule, createDirectExecutor, createPartitionRouter, readHeapStatistics, availableSynonymModule, mergeFrozenSegments, createSharedFrozenSegment, freezeSegmentShared, resolvePartitionInsertOptions, createFrozenSegment } from './chunk-XLJ2S2BH.mjs';
3
- import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, queryBindingOf, executeQuery, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-BJ2YKRNX.mjs';
4
- import { requirePatternSearchFor, requirePartialWordSearchFor, collectQueryTermStats, mergePartitionStats, createPartitionIndex, oversampledFacetConfig, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, isCompositePartition, fnv1a, encodeSegmentState } from './chunk-6VZ3Q73N.mjs';
5
- import { resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-CO4MRKST.mjs';
1
+ import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, queryBindingOf, executeQuery, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-R36MIGBU.mjs';
2
+ import { splitWorkerBudget, resolveWorkerCount, NATIVE_CRC32_MIN_BYTES, resolveWorkerEntry, missingWorkerEntry, MAX_FILE_BYTES, DEFAULT_REQUEST_TIMEOUT_MS, resolveRequestThreadCount, CHECKSUM_YIELD_CHUNK_BYTES, workerResourceLimits, DEFAULT_BACKPRESSURE_LIMIT, SHUTDOWN_TIMEOUT_MS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH } from './chunk-RDJKYKMQ.mjs';
3
+ import { isEmptySynonymSet, synonymModule, createDirectExecutor, createPartitionRouter, readHeapStatistics, availableSynonymModule, mergeFrozenSegments, createSharedFrozenSegment, freezeSegmentShared, resolvePartitionInsertOptions, createFrozenSegment } from './chunk-UAUZGXJP.mjs';
4
+ import { requirePatternSearchFor, requirePartialWordSearchFor, collectQueryTermStats, mergePartitionStats, createPartitionIndex, oversampledFacetConfig, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, isCompositePartition, fnv1a, encodeSegmentState } from './chunk-CHOGQSHK.mjs';
5
+ import { resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-WLOF6HSP.mjs';
6
6
  import { detectRuntime } from './chunk-AV4IJKD2.mjs';
7
- import { generateId, getLanguage } from './chunk-MKWUXWQH.mjs';
8
- import { MAX_INDEX_CONFIGURATION_BYTES, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, FOREIGN_STATS_TTL_MS, MAX_IMPORT_REFUSALS_REPORTED, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE } from './chunk-Q5DC6WAB.mjs';
9
- import { requireSearchBackendSettings } from './chunk-QZN3MIJ3.mjs';
10
- import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-T6CNO72J.mjs';
11
- import { getNestedValue } from './chunk-5ER245FI.mjs';
12
- import { flattenSchema, raiseEarlierReleaseGraphSetting, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, extractVectorFieldsFromSchema, schemaFieldsOf, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-CVQWQVFX.mjs';
13
- import { isSingleStringFieldType } from './chunk-BVOJIYRV.mjs';
7
+ import { MAX_INDEX_CONFIGURATION_BYTES, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, HEAP_PRESSURE_WARNING_FRACTION, FOREIGN_STATS_TTL_MS, MAX_IMPORT_REFUSALS_REPORTED, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE } from './chunk-Q5DC6WAB.mjs';
8
+ import { generateId, getLanguage } from './chunk-ZHS44YKF.mjs';
9
+ import { requireSearchBackendSettings } from './chunk-ES33L3HO.mjs';
10
+ import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-QM3VPX4Z.mjs';
11
+ import { getNestedValue } from './chunk-SBCEHIPL.mjs';
12
+ import { flattenSchema, raiseEarlierReleaseGraphSetting, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, extractVectorFieldsFromSchema, schemaFieldsOf, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-4ZJWRETC.mjs';
13
+ import { isSingleStringFieldType } from './chunk-BB7MASRQ.mjs';
14
14
  import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
15
15
  import { createRequestId, isValidWorkerResponse, threadSlotOfWorker } from './chunk-R6PR4WYZ.mjs';
16
- import { loadVectorCodeFor, requireLoadedVectorCode } from './chunk-R7OSDAFK.mjs';
16
+ import { loadVectorCodeFor, requireLoadedVectorCode } from './chunk-K6A4FCVR.mjs';
17
17
  import { MAX_DOCUMENT_NESTING_DEPTH } from './chunk-L5ORED7V.mjs';
18
- import { ErrorCodes, NarsilError } from './chunk-TPJU7NDJ.mjs';
18
+ import { ErrorCodes, NarsilError } from './chunk-SYWGORQB.mjs';
19
19
  import { encode, decode } from '@msgpack/msgpack';
20
20
  import * as zlib from 'zlib';
21
21
 
@@ -993,6 +993,7 @@ function createLifecycleEntry(state) {
993
993
  waiterCount: 0,
994
994
  closePromise: null,
995
995
  dropPromise: null,
996
+ replacePromise: null,
996
997
  drainResolvers: /* @__PURE__ */ new Set()
997
998
  };
998
999
  }
@@ -1032,7 +1033,7 @@ function createIndexStateCoordinator(config, callbacks) {
1032
1033
  return entry;
1033
1034
  }
1034
1035
  function publicState(entry) {
1035
- return entry.state === "closing" || entry.state === "dropping" ? "open" : entry.state;
1036
+ return entry.state === "closing" || entry.state === "dropping" || entry.state === "replacing" ? "open" : entry.state;
1036
1037
  }
1037
1038
  async function waitForDrain(entry) {
1038
1039
  if (entry.activeOperations === 0) return;
@@ -1072,6 +1073,11 @@ function createIndexStateCoordinator(config, callbacks) {
1072
1073
  entry = entryOf(indexName);
1073
1074
  continue;
1074
1075
  }
1076
+ if (entry.state === "replacing" && entry.replacePromise !== null) {
1077
+ await entry.replacePromise.catch(() => void 0);
1078
+ entry = entryOf(indexName);
1079
+ continue;
1080
+ }
1075
1081
  if (entry.state === "closing" && entry.closePromise !== null) {
1076
1082
  await waitForTransition(
1077
1083
  indexName,
@@ -1087,6 +1093,7 @@ function createIndexStateCoordinator(config, callbacks) {
1087
1093
  function transitionPending(indexName, entry) {
1088
1094
  if (entries.get(indexName) !== entry) return true;
1089
1095
  if (entry.state === "dropping" && entry.dropPromise !== null) return true;
1096
+ if (entry.state === "replacing" && entry.replacePromise !== null) return true;
1090
1097
  return entry.state === "closing" && entry.closePromise !== null;
1091
1098
  }
1092
1099
  async function reopen(indexName, explicit) {
@@ -1134,9 +1141,10 @@ function createIndexStateCoordinator(config, callbacks) {
1134
1141
  entry = entryOf(indexName);
1135
1142
  continue;
1136
1143
  }
1137
- if (entry.reopenPromise !== null) {
1144
+ const settling = entry.reopenPromise ?? entry.replacePromise;
1145
+ if (settling !== null) {
1138
1146
  if (automatic) return false;
1139
- await entry.reopenPromise.catch(() => void 0);
1147
+ await settling.catch(() => void 0);
1140
1148
  entry = entryOf(indexName);
1141
1149
  continue;
1142
1150
  }
@@ -1178,13 +1186,9 @@ function createIndexStateCoordinator(config, callbacks) {
1178
1186
  await entry.dropPromise;
1179
1187
  return;
1180
1188
  }
1181
- if (entry.reopenPromise !== null) {
1182
- await entry.reopenPromise.catch(() => void 0);
1183
- entry = entryOf(indexName);
1184
- continue;
1185
- }
1186
- if (entry.closePromise !== null) {
1187
- await entry.closePromise.catch(() => void 0);
1189
+ const pending = entry.replacePromise ?? entry.reopenPromise ?? entry.closePromise;
1190
+ if (pending !== null) {
1191
+ await pending.catch(() => void 0);
1188
1192
  entry = entryOf(indexName);
1189
1193
  continue;
1190
1194
  }
@@ -1205,6 +1209,11 @@ function createIndexStateCoordinator(config, callbacks) {
1205
1209
  entry.dropPromise = run;
1206
1210
  await run;
1207
1211
  }
1212
+ const replaceContext = {
1213
+ entryOf,
1214
+ waitForDrain,
1215
+ forget: (indexName) => entries.delete(indexName)
1216
+ };
1208
1217
  function exceedsLimits() {
1209
1218
  let open = 0;
1210
1219
  let bytes = 0;
@@ -1298,6 +1307,25 @@ function createIndexStateCoordinator(config, callbacks) {
1298
1307
  drop(indexName, action) {
1299
1308
  return dropEntry(indexName, action);
1300
1309
  },
1310
+ async replace(indexName, action) {
1311
+ const { replaceExclusively } = await import('./replace-CEO5KJJ3.mjs');
1312
+ await replaceExclusively(replaceContext, indexName, action);
1313
+ },
1314
+ async registerReplacing(indexName, action) {
1315
+ const entry = createLifecycleEntry("open");
1316
+ entries.set(indexName, entry);
1317
+ entry.state = "replacing";
1318
+ const run = import('./replace-CEO5KJJ3.mjs').then(
1319
+ ({ replaceUnregistered }) => replaceUnregistered(replaceContext, indexName, entry, action),
1320
+ (error) => {
1321
+ if (entries.get(indexName) === entry) entries.delete(indexName);
1322
+ throw error;
1323
+ }
1324
+ );
1325
+ entry.replacePromise = run;
1326
+ await run;
1327
+ await enforceLimits(indexName);
1328
+ },
1301
1329
  stateOf(indexName) {
1302
1330
  return publicState(entryOf(indexName));
1303
1331
  },
@@ -1309,7 +1337,7 @@ function createIndexStateCoordinator(config, callbacks) {
1309
1337
  let closed = 0;
1310
1338
  let reopens = 0;
1311
1339
  for (const entry of entries.values()) {
1312
- if (entry.state === "open" || entry.state === "closing") open += 1;
1340
+ if (entry.state === "open" || entry.state === "closing" || entry.state === "replacing") open += 1;
1313
1341
  else closed += 1;
1314
1342
  reopens += entry.reopenCount;
1315
1343
  }
@@ -3718,6 +3746,7 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
3718
3746
  reportedIneligible: /* @__PURE__ */ new Set(),
3719
3747
  scaledOutIndexes: /* @__PURE__ */ new Set(),
3720
3748
  desyncedIndexes: /* @__PURE__ */ new Set(),
3749
+ copyReleaseHolds: /* @__PURE__ */ new Map(),
3721
3750
  copyLoadBuffers: /* @__PURE__ */ new Map(),
3722
3751
  copyTransitions: /* @__PURE__ */ new Map(),
3723
3752
  droppedCopies: /* @__PURE__ */ new Map(),
@@ -3752,10 +3781,21 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
3752
3781
  for (let pending = settlingCopyWork(indexName); pending !== void 0; pending = settlingCopyWork(indexName)) {
3753
3782
  await pending;
3754
3783
  }
3784
+ const hold = state.copyReleaseHolds.get(indexName);
3785
+ if (hold !== void 0) {
3786
+ hold.reloads ||= wasScaledOut;
3787
+ return;
3788
+ }
3755
3789
  state.desyncedIndexes.delete(indexName);
3756
3790
  if (!wasScaledOut) return;
3757
3791
  await scaleOutIndex(state, indexName, copyThresholdReason(state, indexName));
3758
3792
  }
3793
+ const copyReleaseSteps = {
3794
+ awaitReplicationIdle,
3795
+ cancelIdleMerge,
3796
+ heldNowhere: (error) => error instanceof NarsilError && error.code === ErrorCodes.INDEX_NOT_FOUND,
3797
+ resyncIndex
3798
+ };
3759
3799
  async function openIndex(indexName) {
3760
3800
  state.droppedCopies.delete(indexName);
3761
3801
  if (!indexReadyForCopies(state, indexName)) return;
@@ -3865,7 +3905,7 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
3865
3905
  pool.removeIndex(indexName);
3866
3906
  }
3867
3907
  state.droppedCopies.delete(indexName);
3868
- state.desyncedIndexes.delete(indexName);
3908
+ if (!state.copyReleaseHolds.has(indexName)) state.desyncedIndexes.delete(indexName);
3869
3909
  state.lastAccessAt.delete(indexName);
3870
3910
  state.replicationQueues.delete(indexName);
3871
3911
  state.segmentLedger.delete(indexName);
@@ -3911,6 +3951,18 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
3911
3951
  if (wasScaledOut) withdrawCopies(indexName);
3912
3952
  return wasScaledOut || loading !== void 0 && loading.kind !== "drop";
3913
3953
  },
3954
+ async releaseIndexCopies(indexName) {
3955
+ const hold = state.copyReleaseHolds.get(indexName);
3956
+ if (hold === void 0) state.copyReleaseHolds.set(indexName, { holders: 1, reloads: false });
3957
+ else hold.holders += 1;
3958
+ state.desyncedIndexes.add(indexName);
3959
+ const { releaseIndexCopies } = await import('./release-BH3DKW3T.mjs');
3960
+ return releaseIndexCopies(state, indexName, copyReleaseSteps);
3961
+ },
3962
+ async endCopyRelease(indexName, reloads) {
3963
+ const { endCopyRelease } = await import('./release-BH3DKW3T.mjs');
3964
+ await endCopyRelease(state, indexName, reloads, copyReleaseSteps);
3965
+ },
3914
3966
  resyncIndex,
3915
3967
  noteAccess: (indexName) => noteAccess(state, indexName),
3916
3968
  workerCopies,
@@ -3956,7 +4008,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
3956
4008
  if (config?.lifecycle !== void 0 && durabilityTier === null) {
3957
4009
  throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
3958
4010
  }
3959
- const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-6AN4QTT3.mjs');
4011
+ const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-Y55SCPFC.mjs');
3960
4012
  const filesystemDurability = durabilityTier?.kind === "wal";
3961
4013
  const executor = createDirectExecutor({
3962
4014
  vectorWorkerCopies: vectorCopyPolicy,
@@ -4680,6 +4732,9 @@ async function runLifecycleHook(core, hookName, context) {
4680
4732
  async function announceIndexCreated(core, name, config) {
4681
4733
  await runLifecycleHook(core, "onIndexCreate", { indexName: name, config });
4682
4734
  }
4735
+ async function announceIndexDropped(core, name, config) {
4736
+ await runLifecycleHook(core, "onIndexDrop", { indexName: name, config });
4737
+ }
4683
4738
  async function createEngineIndex(core, config, name, requestedConfig, creation = {}) {
4684
4739
  const { indexUuid, adapterBinding = "required", appliesCreationLimits = false } = creation;
4685
4740
  const loadingVectorCode = loadVectorCodeFor(requestedConfig.schema);
@@ -6834,6 +6889,14 @@ function portabilityOperations(core, config, engine) {
6834
6889
  release();
6835
6890
  }
6836
6891
  }
6892
+ const restoresInFlight = /* @__PURE__ */ new Map();
6893
+ const restoreHost = {
6894
+ getVectorFieldPaths,
6895
+ announceIndexCreated: (name, restoredConfig) => announceIndexCreated(core, name, restoredConfig),
6896
+ announceIndexDropped: (name, replacedConfig) => announceIndexDropped(core, name, replacedConfig),
6897
+ defaultEmbeddingAdapter: config?.embedding ?? null,
6898
+ releaseTimeoutMs: DEFAULT_REQUEST_TIMEOUT_MS
6899
+ };
6837
6900
  function snapshotDepsOf(indexName) {
6838
6901
  return {
6839
6902
  indexName,
@@ -6842,41 +6905,27 @@ function portabilityOperations(core, config, engine) {
6842
6905
  durability: core.durability
6843
6906
  };
6844
6907
  }
6845
- async function restoreIndex(indexName, data) {
6846
- const { restoreFromSnapshot } = await import('./snapshot-JFZVOGZV.mjs');
6847
- await restoreFromSnapshot(indexName, data, {
6848
- executor: core.executor,
6849
- indexRegistry: core.indexRegistry,
6850
- getVectorFieldPaths,
6851
- dropIndex: (name) => dropEngineIndex(core, name),
6852
- announceIndexCreated: (name, restoredConfig) => announceIndexCreated(core, name, restoredConfig),
6853
- requireManager: core.requireManager,
6854
- durability: core.durability,
6855
- filesystemDurability: core.filesystemDurability,
6856
- embeddingAdapters: core.embeddingAdapters,
6857
- defaultEmbeddingAdapter: config?.embedding ?? null,
6858
- markAnalysisStale: core.analysisRebuild.markStale,
6859
- clearAnalysisStale: core.analysisRebuild.clearStale,
6860
- indexState: core.indexState
6861
- });
6862
- }
6863
6908
  return {
6864
6909
  snapshot(indexName) {
6865
6910
  return withOpenIndex(indexName, async () => {
6866
- const { createSnapshot } = await import('./snapshot-JFZVOGZV.mjs');
6911
+ const { createSnapshot } = await import('./snapshot-XBKE7PUG.mjs');
6867
6912
  return createSnapshot(snapshotDepsOf(indexName));
6868
6913
  });
6869
6914
  },
6870
6915
  async restore(indexName, data) {
6871
6916
  core.guardShutdown();
6872
- await emitWriteEventAfter(core, indexName, restoreIndex(indexName, data));
6917
+ validateIndexName(indexName);
6918
+ const restored = import('./restore-queue-4SG45BLF.mjs').then(
6919
+ ({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
6920
+ );
6921
+ await emitWriteEventAfter(core, indexName, restored);
6873
6922
  core.heapPressureNotifier.check(indexName);
6874
6923
  },
6875
6924
  exportDocuments(indexName) {
6876
6925
  return withOpenIndex(indexName, async () => {
6877
6926
  const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
6878
- import('./snapshot-JFZVOGZV.mjs'),
6879
- import('./write-OJR3H4BI.mjs')
6927
+ import('./snapshot-XBKE7PUG.mjs'),
6928
+ import('./write-BMNV7P2H.mjs')
6880
6929
  ]);
6881
6930
  const header = indexConfigurationOf(core.requireIndex(indexName));
6882
6931
  return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));