@delali/narsil 0.3.1-next.80 → 0.3.1-next.83

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/dist/adapters/filesystem.mjs +1 -1
  2. package/dist/adapters/indexeddb.mjs +1 -1
  3. package/dist/{capabilities-CcTDWh_P.d.ts → capabilities-CAdkp2P-.d.ts} +2 -2
  4. package/dist/{chunk-HJASKX4F.mjs → chunk-2453RJYK.mjs} +2 -2
  5. package/dist/{chunk-4QDQEJDV.mjs → chunk-4D6OH4EF.mjs} +6 -6
  6. package/dist/{chunk-P2PYEZZE.mjs → chunk-4VBG6PEH.mjs} +3 -3
  7. package/dist/{chunk-Y3L5MDRV.mjs → chunk-5XNIRLFC.mjs} +2 -2
  8. package/dist/{chunk-NSXQCE4K.mjs → chunk-7JZZ3HNV.mjs} +208 -48
  9. package/dist/{chunk-2E35UM3G.mjs → chunk-B3UOQJD3.mjs} +1 -1
  10. package/dist/{chunk-K4FZ55Q2.mjs → chunk-CUP2E4OH.mjs} +175 -99
  11. package/dist/{chunk-MEOOYKWL.mjs → chunk-GNYXCLJS.mjs} +4 -4
  12. package/dist/{chunk-OSF3SGJV.mjs → chunk-KJ7LB3TR.mjs} +2 -2
  13. package/dist/{chunk-DQABW5UQ.mjs → chunk-LS5IRM23.mjs} +1 -1
  14. package/dist/{chunk-4MQ5F4IL.mjs → chunk-OLXM65YH.mjs} +1 -1
  15. package/dist/{chunk-23HJ5QXQ.mjs → chunk-RZXNWQSM.mjs} +183 -94
  16. package/dist/{chunk-R2XCJIO7.mjs → chunk-SREPGC6O.mjs} +2 -2
  17. package/dist/{chunk-I64LXKO3.mjs → chunk-TPJU7NDJ.mjs} +1 -0
  18. package/dist/{chunk-YGLSR7BC.mjs → chunk-XYDP7CCJ.mjs} +5 -5
  19. package/dist/{chunk-QPYV22XH.mjs → chunk-XZF4LZ3X.mjs} +4 -3
  20. package/dist/{chunk-GG2FBGVZ.mjs → chunk-ZCKPRCYI.mjs} +1 -1
  21. package/dist/{client-CsybN673.d.ts → client-CKUtkLz3.d.ts} +2 -2
  22. package/dist/client.d.ts +7 -7
  23. package/dist/client.mjs +1 -0
  24. package/dist/{config-CFonR_gY.d.ts → config-BuIkeM5C.d.ts} +39 -3
  25. package/dist/{config-BCNQtszf.d.ts → config-CSDEPraV.d.ts} +1 -1
  26. package/dist/direct-executor-6HASZ36Y.mjs +19 -0
  27. package/dist/distribution/coordinator/etcd.d.ts +1 -1
  28. package/dist/distribution/coordinator/etcd.mjs +1 -1
  29. package/dist/distribution/coordinator/in-memory.d.ts +1 -1
  30. package/dist/distribution/transport/grpc.mjs +1 -1
  31. package/dist/distribution.d.ts +7 -7
  32. package/dist/distribution.mjs +756 -64
  33. package/dist/embeddings/openai.mjs +1 -1
  34. package/dist/{engine-B0oIF1_r.d.ts → engine-DMNZJAUq.d.ts} +1 -1
  35. package/dist/{types-ChrVy4lH.d.ts → index-DzONZUKE.d.ts} +79 -37
  36. package/dist/index.browser.mjs +426 -249
  37. package/dist/index.d.ts +5 -5
  38. package/dist/index.mjs +14 -14
  39. package/dist/{memory-B6SBOKlS.d.ts → memory-BPKNx9Tl.d.ts} +1 -0
  40. package/dist/partial-words.mjs +4 -4
  41. package/dist/pattern.mjs +3 -3
  42. package/dist/persistence/durability/checkpoint-worker.mjs +14 -14
  43. package/dist/react.d.ts +6 -6
  44. package/dist/react.mjs +1 -0
  45. package/dist/server.d.ts +9 -9
  46. package/dist/server.mjs +18 -13
  47. package/dist/{tasks-CYAQe6qd.d.ts → tasks-DGyz586z.d.ts} +1 -1
  48. package/dist/{thread-P6QJ2Q5O.mjs → thread-33M5AOEI.mjs} +10 -10
  49. package/dist/vector/search-worker.d.ts +2 -0
  50. package/dist/vector/search-worker.mjs +8 -7
  51. package/dist/workers/entry.mjs +5 -5
  52. package/package.json +11 -7
  53. package/dist/direct-executor-VNMQ6NIH.mjs +0 -19
@@ -49,6 +49,7 @@ var ErrorCodes = {
49
49
  QUERY_NODE_TIMEOUT: "QUERY_NODE_TIMEOUT",
50
50
  QUERY_NO_ACTIVE_REPLICA: "QUERY_NO_ACTIVE_REPLICA",
51
51
  CLUSTER_OPERATION_UNSUPPORTED: "CLUSTER_OPERATION_UNSUPPORTED",
52
+ CLUSTER_WRITE_BLOCKED: "CLUSTER_WRITE_BLOCKED",
52
53
  ALLOCATION_NO_DATA_NODES: "ALLOCATION_NO_DATA_NODES",
53
54
  ALLOCATION_INVALID_CONFIG: "ALLOCATION_INVALID_CONFIG",
54
55
  ALLOCATION_FAILED: "ALLOCATION_FAILED",
@@ -12919,9 +12920,17 @@ function createOsqQuantizer(dimension, bits, metric, store) {
12919
12920
  if (view === null || ordinal === void 0 || !view.holdsOrdinal(ordinal)) return void 0;
12920
12921
  return view.recordAt(ordinal);
12921
12922
  },
12923
+ recordAtOrdinal(ordinal) {
12924
+ const view = shared2();
12925
+ if (view === null || !view.holdsOrdinal(ordinal)) return void 0;
12926
+ return view.recordAt(ordinal);
12927
+ },
12922
12928
  restoreRecord(docId, record) {
12923
12929
  requireShared().restoreRecord(ordinalOf(docId), record);
12924
12930
  },
12931
+ restoreRecordAtOrdinal(ordinal, record) {
12932
+ requireShared().restoreRecord(ordinal, record);
12933
+ },
12925
12934
  restoreCentroid(centroid) {
12926
12935
  requireShared().writeCentroid(centroid);
12927
12936
  },
@@ -12954,6 +12963,43 @@ function createOsqQuantizer(dimension, bits, metric, store) {
12954
12963
  };
12955
12964
  }
12956
12965
 
12966
+ // src/vector/similarity.ts
12967
+ function magnitude(v) {
12968
+ const result = simdMagnitude(v);
12969
+ if (result !== null) return result;
12970
+ let sum = 0;
12971
+ for (let i = 0; i < v.length; i++) {
12972
+ sum += v[i] * v[i];
12973
+ }
12974
+ return Math.sqrt(sum);
12975
+ }
12976
+ function cosineSimilarityWithMagnitudes(a, b, magA, magB) {
12977
+ if (magA === 0 || magB === 0) return 0;
12978
+ return dotProduct(a, b) / (magA * magB);
12979
+ }
12980
+ function dotProduct(a, b) {
12981
+ const result = simdDotProduct(a, b);
12982
+ if (result !== null) return result;
12983
+ let sum = 0;
12984
+ for (let i = 0; i < a.length; i++) {
12985
+ sum += a[i] * b[i];
12986
+ }
12987
+ return sum;
12988
+ }
12989
+ function euclideanDistance(a, b) {
12990
+ return Math.sqrt(squaredEuclideanDistance(a, b));
12991
+ }
12992
+ function squaredEuclideanDistance(a, b) {
12993
+ const result = simdSquaredEuclideanDistance(a, b);
12994
+ if (result !== null) return result;
12995
+ let sum = 0;
12996
+ for (let i = 0; i < a.length; i++) {
12997
+ const diff = a[i] - b[i];
12998
+ sum += diff * diff;
12999
+ }
13000
+ return sum;
13001
+ }
13002
+
12957
13003
  // src/vector/vector-store/doc-ids.ts
12958
13004
  var encoder2 = new TextEncoder();
12959
13005
  function appendDocId(handles, ordinal, docId) {
@@ -12973,6 +13019,78 @@ function resetDocIds(handles) {
12973
13019
  Atomics.store(handles.header, STORE_DOC_ID_BYTES, 0);
12974
13020
  }
12975
13021
 
13022
+ // src/vector/vector-store/ordinal-reads.ts
13023
+ function entryAt(store, ordinal) {
13024
+ return { vector: store.view.vectorAt(ordinal), magnitude: store.magnitudes[ordinal] };
13025
+ }
13026
+ function createOrdinalReads(openStore, readable) {
13027
+ function pairDistance(metric) {
13028
+ const open = openStore();
13029
+ if (open === null) return () => Number.POSITIVE_INFINITY;
13030
+ const inner = open.view.pairDistance(metric);
13031
+ return (ordA, ordB) => readable(ordA) && readable(ordB) ? inner(ordA, ordB) : Number.POSITIVE_INFINITY;
13032
+ }
13033
+ return {
13034
+ entryForOrdinal(ordinal) {
13035
+ const open = openStore();
13036
+ if (open === null || !readable(ordinal)) return void 0;
13037
+ return entryAt(open, ordinal);
13038
+ },
13039
+ distanceByOrdinal(ordA, ordB, metric) {
13040
+ const open = openStore();
13041
+ if (open === null || !readable(ordA) || !readable(ordB)) return Number.POSITIVE_INFINITY;
13042
+ return open.view.distanceByOrdinal(ordA, ordB, metric);
13043
+ },
13044
+ distanceFromArena(prepared, ordinal, metric) {
13045
+ const open = openStore();
13046
+ if (open === null || !readable(ordinal)) return Number.POSITIVE_INFINITY;
13047
+ return open.view.distanceFromArena(prepared, ordinal, metric);
13048
+ },
13049
+ queryDistance(prepared, metric) {
13050
+ const open = openStore();
13051
+ if (open === null) return () => Number.POSITIVE_INFINITY;
13052
+ const inner = open.view.queryDistance(prepared, metric);
13053
+ return (ordinal) => readable(ordinal) ? inner(ordinal) : Number.POSITIVE_INFINITY;
13054
+ },
13055
+ ordinalDistance(from, metric) {
13056
+ const pair = pairDistance(metric);
13057
+ return (ordinal) => pair(from, ordinal);
13058
+ },
13059
+ pairDistance
13060
+ };
13061
+ }
13062
+
13063
+ // src/vector/vector-store/listing.ts
13064
+ function* entriesOf(open, documents) {
13065
+ if (open === null) return;
13066
+ for (let ordinal = 0; ordinal < documents.length; ordinal++) {
13067
+ const docId = documents[ordinal];
13068
+ if (docId === void 0) continue;
13069
+ yield [docId, entryAt(open, ordinal)];
13070
+ }
13071
+ }
13072
+ function* docIdsOf(open, documents) {
13073
+ if (open === null) return;
13074
+ for (let ordinal = 0; ordinal < documents.length; ordinal++) {
13075
+ const docId = documents[ordinal];
13076
+ if (docId !== void 0) yield docId;
13077
+ }
13078
+ }
13079
+ function snapshotOf(open, documents, replacedOrdinals) {
13080
+ const slots = documents.length;
13081
+ const dimension = open === null ? 0 : open.handles.dimension;
13082
+ const vectors = new Float32Array(slots * dimension);
13083
+ const magnitudes = new Float64Array(slots);
13084
+ const docIds = new Array(slots);
13085
+ for (let ordinal = 0; ordinal < slots; ordinal++) {
13086
+ docIds[ordinal] = documents[ordinal] ?? null;
13087
+ if (open === null || documents[ordinal] === void 0 && !replacedOrdinals.has(ordinal)) continue;
13088
+ vectors.set(open.view.vectorAt(ordinal), ordinal * dimension);
13089
+ magnitudes[ordinal] = open.magnitudes[ordinal];
13090
+ }
13091
+ return { dimension, slots, vectors, magnitudes, docIds, replacedOrdinals: [...replacedOrdinals] };
13092
+ }
13093
+
12976
13094
  // src/vector/ordinal-filter.ts
12977
13095
  function createOrdinalFilter(slots) {
12978
13096
  return { bits: new Uint8Array(slots + 7 >>> 3), count: 0 };
@@ -13009,43 +13127,6 @@ function* ordinalFilterValues(filter) {
13009
13127
  }
13010
13128
  }
13011
13129
 
13012
- // src/vector/similarity.ts
13013
- function magnitude(v) {
13014
- const result = simdMagnitude(v);
13015
- if (result !== null) return result;
13016
- let sum = 0;
13017
- for (let i = 0; i < v.length; i++) {
13018
- sum += v[i] * v[i];
13019
- }
13020
- return Math.sqrt(sum);
13021
- }
13022
- function cosineSimilarityWithMagnitudes(a, b, magA, magB) {
13023
- if (magA === 0 || magB === 0) return 0;
13024
- return dotProduct(a, b) / (magA * magB);
13025
- }
13026
- function dotProduct(a, b) {
13027
- const result = simdDotProduct(a, b);
13028
- if (result !== null) return result;
13029
- let sum = 0;
13030
- for (let i = 0; i < a.length; i++) {
13031
- sum += a[i] * b[i];
13032
- }
13033
- return sum;
13034
- }
13035
- function euclideanDistance(a, b) {
13036
- return Math.sqrt(squaredEuclideanDistance(a, b));
13037
- }
13038
- function squaredEuclideanDistance(a, b) {
13039
- const result = simdSquaredEuclideanDistance(a, b);
13040
- if (result !== null) return result;
13041
- let sum = 0;
13042
- for (let i = 0; i < a.length; i++) {
13043
- const diff = a[i] - b[i];
13044
- sum += diff * diff;
13045
- }
13046
- return sum;
13047
- }
13048
-
13049
13130
  // src/vector/vector-store/vector-file.browser.ts
13050
13131
  function filesystemUnavailable() {
13051
13132
  throw new NarsilError(
@@ -13479,13 +13560,14 @@ function writeVector(store, ordinal, vector) {
13479
13560
  block.float32(base + vector.length).set(vector, base);
13480
13561
  store.magnitudes[ordinal] = magnitude(vector);
13481
13562
  }
13482
- function blockFullyCold(store, blockIndex, documents) {
13563
+ function blockFullyCold(store, blockIndex, documents, replacedOrdinals) {
13483
13564
  const capacity = store.handles.layout.capacity;
13484
13565
  const slots = Atomics.load(store.handles.header, STORE_SLOTS);
13485
13566
  const end = Math.min(slots, (blockIndex + 1) * capacity);
13486
13567
  if (end < (blockIndex + 1) * capacity) return false;
13487
13568
  for (let ordinal = blockIndex * capacity; ordinal < end; ordinal++) {
13488
- if (documents[ordinal] !== void 0 && store.diskFile[ordinal] === IN_MEMORY) return false;
13569
+ const held = documents[ordinal] !== void 0 || replacedOrdinals.has(ordinal);
13570
+ if (held && store.diskFile[ordinal] === IN_MEMORY) return false;
13489
13571
  }
13490
13572
  return true;
13491
13573
  }
@@ -13498,10 +13580,11 @@ function fileHoldsSameBytes(store, ordinal, location) {
13498
13580
  for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
13499
13581
  return true;
13500
13582
  }
13501
- function forgetUnreferencedFiles(store, documents) {
13583
+ function forgetUnreferencedFiles(store, documents, replacedOrdinals) {
13502
13584
  const referenced = /* @__PURE__ */ new Set();
13503
13585
  for (let ordinal = 0; ordinal < documents.length; ordinal++) {
13504
- if (documents[ordinal] !== void 0 && store.diskFile[ordinal] !== IN_MEMORY) {
13586
+ const held = documents[ordinal] !== void 0 || replacedOrdinals.has(ordinal);
13587
+ if (held && store.diskFile[ordinal] !== IN_MEMORY) {
13505
13588
  referenced.add(store.diskFile[ordinal]);
13506
13589
  }
13507
13590
  }
@@ -13542,15 +13625,20 @@ function createVectorStore(options) {
13542
13625
  const blockBytes2 = options?.blockBytes ?? BLOCK_MAX_BYTES;
13543
13626
  const docToOrd = /* @__PURE__ */ new Map();
13544
13627
  const ordToDoc = [];
13628
+ const replacedOrdinals = /* @__PURE__ */ new Set();
13545
13629
  let unknownPartitions = 0;
13546
13630
  let liveCount = 0;
13547
13631
  let open = options?.dimension === void 0 ? null : openHandles(options.dimension, codeBits, blockBytes2);
13548
13632
  function forgetDocuments() {
13549
13633
  docToOrd.clear();
13550
13634
  ordToDoc.length = 0;
13635
+ replacedOrdinals.clear();
13551
13636
  unknownPartitions = 0;
13552
13637
  liveCount = 0;
13553
13638
  }
13639
+ function readable(ordinal) {
13640
+ return ordToDoc[ordinal] !== void 0 || replacedOrdinals.has(ordinal);
13641
+ }
13554
13642
  function requireOpen() {
13555
13643
  if (open === null) {
13556
13644
  throw new NarsilError(ErrorCodes.VECTOR_DIMENSION_MISMATCH, "The vector store holds no vector yet");
@@ -13567,11 +13655,27 @@ function createVectorStore(options) {
13567
13655
  Atomics.store(store.present, ordinal, 0);
13568
13656
  if (store.partitions[ordinal] === UNKNOWN_PARTITION) unknownPartitions -= 1;
13569
13657
  }
13570
- function claimOrdinal(store, docId, cold, partitionId) {
13658
+ function keepReplacedOrdinal(store, ordinal) {
13659
+ ordToDoc[ordinal] = void 0;
13660
+ if (store.partitions[ordinal] === UNKNOWN_PARTITION) unknownPartitions -= 1;
13661
+ store.partitions[ordinal] = NO_PARTITION;
13662
+ replacedOrdinals.add(ordinal);
13663
+ }
13664
+ function restoreReplacedVector(store, ordinal, vector) {
13665
+ store.diskFile[ordinal] = IN_MEMORY;
13666
+ writeVector(store, ordinal, vector);
13667
+ Atomics.store(store.present, ordinal, 1);
13668
+ replacedOrdinals.add(ordinal);
13669
+ }
13670
+ function claimSlot(store, docId, cold) {
13571
13671
  const ordinal = Atomics.load(store.handles.header, STORE_SLOTS);
13572
13672
  ensureOrdinal(store, ordinal, cold);
13573
13673
  store.diskFile[ordinal] = IN_MEMORY;
13574
13674
  appendDocId(store.handles, ordinal, docId);
13675
+ return ordinal;
13676
+ }
13677
+ function claimOrdinal(store, docId, cold, partitionId) {
13678
+ const ordinal = claimSlot(store, docId, cold);
13575
13679
  recordPartition(store, ordinal, partitionId);
13576
13680
  ordToDoc[ordinal] = docId;
13577
13681
  docToOrd.set(docId, ordinal);
@@ -13581,25 +13685,30 @@ function createVectorStore(options) {
13581
13685
  Atomics.store(store.handles.header, STORE_SLOTS, ordinal + 1);
13582
13686
  Atomics.store(store.present, ordinal, 1);
13583
13687
  }
13688
+ function publishOnDisk(store, ordinal, vectorMagnitude, location) {
13689
+ store.magnitudes[ordinal] = vectorMagnitude;
13690
+ store.diskOffset[ordinal] = location.offset;
13691
+ store.diskFile[ordinal] = location.fileIndex;
13692
+ publishOrdinal(store, ordinal);
13693
+ }
13584
13694
  function appendOrdinal(store, docId, vector, partitionId) {
13585
13695
  const ordinal = claimOrdinal(store, docId, false, partitionId);
13586
13696
  writeVector(store, ordinal, vector);
13587
13697
  publishOrdinal(store, ordinal);
13588
13698
  return ordinal;
13589
13699
  }
13590
- function settleReplacement(store, ordinal, previous) {
13700
+ function settleReplacement(store, ordinal, previous, keepsReplacedVector) {
13591
13701
  if (previous !== void 0) {
13592
- retireOrdinal(store, previous);
13702
+ if (keepsReplacedVector) keepReplacedOrdinal(store, previous);
13703
+ else retireOrdinal(store, previous);
13593
13704
  return ordinal;
13594
13705
  }
13595
13706
  liveCount++;
13596
13707
  Atomics.add(store.handles.header, STORE_LIVE_COUNT, 1);
13597
13708
  return ordinal;
13598
13709
  }
13599
- function entryAt(store, ordinal) {
13600
- return { vector: store.view.vectorAt(ordinal), magnitude: store.magnitudes[ordinal] };
13601
- }
13602
13710
  return {
13711
+ ...createOrdinalReads(() => open, readable),
13603
13712
  get size() {
13604
13713
  return liveCount;
13605
13714
  },
@@ -13618,7 +13727,10 @@ function createVectorStore(options) {
13618
13727
  get view() {
13619
13728
  return requireOpen().view;
13620
13729
  },
13621
- insert(docId, vector, partitionId) {
13730
+ get replacedVectorCount() {
13731
+ return replacedOrdinals.size;
13732
+ },
13733
+ insert(docId, vector, partitionId, keepsReplacedVector = false) {
13622
13734
  if (open === null) open = openHandles(vector.length, codeBits, blockBytes2);
13623
13735
  const store = open;
13624
13736
  if (vector.length !== store.handles.dimension) {
@@ -13630,18 +13742,37 @@ function createVectorStore(options) {
13630
13742
  }
13631
13743
  const previous = docToOrd.get(docId);
13632
13744
  const ordinal = appendOrdinal(store, docId, vector, partitionId);
13633
- return settleReplacement(store, ordinal, previous);
13745
+ return settleReplacement(store, ordinal, previous, keepsReplacedVector);
13746
+ },
13747
+ releaseReplacedVectors() {
13748
+ const released = [...replacedOrdinals];
13749
+ replacedOrdinals.clear();
13750
+ if (open !== null) for (const ordinal of released) Atomics.store(open.present, ordinal, 0);
13751
+ return released;
13634
13752
  },
13635
13753
  insertCold(docId, vectorMagnitude, location, partitionId) {
13636
13754
  const store = requireOpen();
13637
13755
  const previous = docToOrd.get(docId);
13638
13756
  Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
13639
13757
  const ordinal = claimOrdinal(store, docId, true, partitionId);
13640
- store.magnitudes[ordinal] = vectorMagnitude;
13641
- store.diskOffset[ordinal] = location.offset;
13642
- store.diskFile[ordinal] = location.fileIndex;
13758
+ publishOnDisk(store, ordinal, vectorMagnitude, location);
13759
+ return settleReplacement(store, ordinal, previous, false);
13760
+ },
13761
+ insertReplaced(docId, vector, location) {
13762
+ if (open === null) open = openHandles(vector.length, codeBits, blockBytes2);
13763
+ const store = open;
13764
+ if (location !== void 0) Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
13765
+ const ordinal = claimSlot(store, docId, location !== void 0);
13766
+ store.partitions[ordinal] = NO_PARTITION;
13767
+ ordToDoc[ordinal] = void 0;
13768
+ replacedOrdinals.add(ordinal);
13769
+ if (location !== void 0) {
13770
+ publishOnDisk(store, ordinal, magnitude(vector), location);
13771
+ return ordinal;
13772
+ }
13773
+ writeVector(store, ordinal, vector);
13643
13774
  publishOrdinal(store, ordinal);
13644
- return settleReplacement(store, ordinal, previous);
13775
+ return ordinal;
13645
13776
  },
13646
13777
  addVectorFile(path) {
13647
13778
  const store = requireOpen();
@@ -13668,11 +13799,11 @@ function createVectorStore(options) {
13668
13799
  const store = requireOpen();
13669
13800
  let released = 0;
13670
13801
  for (let index = 0; index < store.handles.blocks.length; index++) {
13671
- if (store.handles.blocks[index] === null || !blockFullyCold(store, index, ordToDoc)) continue;
13802
+ if (store.handles.blocks[index] === null || !blockFullyCold(store, index, ordToDoc, replacedOrdinals)) continue;
13672
13803
  store.handles.blocks[index] = null;
13673
13804
  released += 1;
13674
13805
  }
13675
- if (released + forgetUnreferencedFiles(store, ordToDoc) > 0) relayout(store);
13806
+ if (released + forgetUnreferencedFiles(store, ordToDoc, replacedOrdinals) > 0) relayout(store);
13676
13807
  return released;
13677
13808
  },
13678
13809
  setPartition(docId, partitionId) {
@@ -13716,21 +13847,8 @@ function createVectorStore(options) {
13716
13847
  holdsOrdinal(ordinal) {
13717
13848
  return ordinal >= 0 && ordinal < ordToDoc.length && ordToDoc[ordinal] !== void 0;
13718
13849
  },
13719
- *entries() {
13720
- if (open === null) return;
13721
- for (let ordinal = 0; ordinal < ordToDoc.length; ordinal++) {
13722
- const docId = ordToDoc[ordinal];
13723
- if (docId === void 0) continue;
13724
- yield [docId, entryAt(open, ordinal)];
13725
- }
13726
- },
13727
- *docIds() {
13728
- if (open === null) return;
13729
- for (let ordinal = 0; ordinal < ordToDoc.length; ordinal++) {
13730
- const docId = ordToDoc[ordinal];
13731
- if (docId !== void 0) yield docId;
13732
- }
13733
- },
13850
+ entries: () => entriesOf(open, ordToDoc),
13851
+ docIds: () => docIdsOf(open, ordToDoc),
13734
13852
  clear() {
13735
13853
  forgetDocuments();
13736
13854
  if (open === null) return;
@@ -13762,59 +13880,16 @@ function createVectorStore(options) {
13762
13880
  docIdForOrdinal(ordinal) {
13763
13881
  return ordToDoc[ordinal];
13764
13882
  },
13765
- entryForOrdinal(ordinal) {
13766
- if (open === null || ordinal < 0 || ordinal >= ordToDoc.length || ordToDoc[ordinal] === void 0) {
13767
- return void 0;
13768
- }
13769
- return entryAt(open, ordinal);
13770
- },
13771
- distanceByOrdinal(ordA, ordB, metric) {
13772
- if (open === null || ordToDoc[ordA] === void 0 || ordToDoc[ordB] === void 0) {
13773
- return Number.POSITIVE_INFINITY;
13774
- }
13775
- return open.view.distanceByOrdinal(ordA, ordB, metric);
13776
- },
13777
13883
  prepareQueryArena(query) {
13778
13884
  return open === null ? null : open.view.prepareQueryArena(query);
13779
13885
  },
13780
- distanceFromArena(prepared, ordinal, metric) {
13781
- if (open === null || ordToDoc[ordinal] === void 0) return Number.POSITIVE_INFINITY;
13782
- return open.view.distanceFromArena(prepared, ordinal, metric);
13783
- },
13784
- queryDistance(prepared, metric) {
13785
- if (open === null) return () => Number.POSITIVE_INFINITY;
13786
- const inner = open.view.queryDistance(prepared, metric);
13787
- return (ordinal) => ordToDoc[ordinal] === void 0 ? Number.POSITIVE_INFINITY : inner(ordinal);
13788
- },
13789
- ordinalDistance(from, metric) {
13790
- const pair = this.pairDistance(metric);
13791
- return (ordinal) => pair(from, ordinal);
13792
- },
13793
- pairDistance(metric) {
13794
- if (open === null) return () => Number.POSITIVE_INFINITY;
13795
- const inner = open.view.pairDistance(metric);
13796
- return (ordA, ordB) => ordToDoc[ordA] === void 0 || ordToDoc[ordB] === void 0 ? Number.POSITIVE_INFINITY : inner(ordA, ordB);
13797
- },
13798
- exportSnapshot() {
13799
- const slots = ordToDoc.length;
13800
- const dimension = open === null ? 0 : open.handles.dimension;
13801
- const vectors = new Float32Array(slots * dimension);
13802
- const magnitudes = new Float64Array(slots);
13803
- const docIds = new Array(slots);
13804
- for (let ordinal = 0; ordinal < slots; ordinal++) {
13805
- const docId = ordToDoc[ordinal];
13806
- docIds[ordinal] = docId ?? null;
13807
- if (docId === void 0 || open === null) continue;
13808
- vectors.set(open.view.vectorAt(ordinal), ordinal * dimension);
13809
- magnitudes[ordinal] = open.magnitudes[ordinal];
13810
- }
13811
- return { dimension, slots, vectors, magnitudes, docIds };
13812
- },
13886
+ exportSnapshot: () => snapshotOf(open, ordToDoc, replacedOrdinals),
13813
13887
  restoreSnapshot(snapshot) {
13814
13888
  this.clear();
13815
13889
  if (snapshot.dimension === 0 || snapshot.slots === 0) return;
13816
13890
  if (open === null) open = openHandles(snapshot.dimension, codeBits, blockBytes2);
13817
13891
  const store = open;
13892
+ const replaced = new Set(snapshot.replacedOrdinals);
13818
13893
  for (let ordinal = 0; ordinal < snapshot.slots; ordinal++) {
13819
13894
  const docId = snapshot.docIds[ordinal];
13820
13895
  const vector = snapshot.vectors.subarray(ordinal * snapshot.dimension, (ordinal + 1) * snapshot.dimension);
@@ -13822,6 +13897,7 @@ function createVectorStore(options) {
13822
13897
  ensureOrdinal(store, ordinal, false);
13823
13898
  ordToDoc[ordinal] = void 0;
13824
13899
  store.partitions[ordinal] = NO_PARTITION;
13900
+ if (replaced.has(ordinal)) restoreReplacedVector(store, ordinal, vector);
13825
13901
  Atomics.store(store.handles.header, STORE_SLOTS, ordinal + 1);
13826
13902
  continue;
13827
13903
  }
@@ -14646,7 +14722,7 @@ function readNeighborsLocked(state, ord, layer) {
14646
14722
  if (nodeReadHeld(state.locks, ord, version)) return count;
14647
14723
  }
14648
14724
  }
14649
- function searchLayer(state, qVec, qMag, ef, layer, metric, skipTombstones, distFn, results) {
14725
+ function searchLayer(state, qVec, qMag, ef, layer, metric, excludeTombstones, distFn, results) {
14650
14726
  const getDistance = distFn ?? ((ord) => queryDistanceByOrd(state, qVec, qMag, ord, metric));
14651
14727
  const workspace = state.workspace;
14652
14728
  const frontier = workspace.frontier;
@@ -14662,10 +14738,10 @@ function searchLayer(state, qVec, qMag, ef, layer, metric, skipTombstones, distF
14662
14738
  if (state.visited[epOrd] === stamp) continue;
14663
14739
  state.visited[epOrd] = stamp;
14664
14740
  if (!nodeExists(state, epOrd)) continue;
14665
- if (skipTombstones && isTombstoned(state, epOrd)) continue;
14666
14741
  const dist = getDistance(epOrd);
14667
14742
  if (dist === Number.POSITIVE_INFINITY) continue;
14668
14743
  pushHeap(frontier, epOrd, dist);
14744
+ if (excludeTombstones && isTombstoned(state, epOrd)) continue;
14669
14745
  pushHeap(found, epOrd, dist);
14670
14746
  if (found.size > ef) {
14671
14747
  popHeap(found);
@@ -14673,19 +14749,19 @@ function searchLayer(state, qVec, qMag, ef, layer, metric, skipTombstones, distF
14673
14749
  }
14674
14750
  if (found.size > 0) furthestDist = found.distances[0];
14675
14751
  while (popHeap(frontier)) {
14676
- if (frontier.topDistance > furthestDist) break;
14752
+ if (frontier.topDistance > furthestDist && found.size >= ef) break;
14677
14753
  const count = readNeighborsLocked(state, frontier.topOrd, layer);
14678
14754
  for (let i = 0; i < count; i++) {
14679
14755
  const neighborOrd = scratch[i];
14680
14756
  ensureVisited(state, neighborOrd + 1);
14681
14757
  if (state.visited[neighborOrd] === stamp) continue;
14682
14758
  state.visited[neighborOrd] = stamp;
14683
- if (skipTombstones && isTombstoned(state, neighborOrd)) continue;
14684
14759
  if (!nodeExists(state, neighborOrd)) continue;
14685
14760
  const dist = getDistance(neighborOrd);
14686
14761
  if (dist === Number.POSITIVE_INFINITY) continue;
14687
14762
  if (dist < furthestDist || found.size < ef) {
14688
14763
  pushHeap(frontier, neighborOrd, dist);
14764
+ if (excludeTombstones && isTombstoned(state, neighborOrd)) continue;
14689
14765
  pushHeap(found, neighborOrd, dist);
14690
14766
  if (found.size > ef) {
14691
14767
  popHeap(found);
@@ -14961,13 +15037,6 @@ function markTombstone(state, ord) {
14961
15037
  if (Atomics.compareExchange(state.tombstones, ord, 0, 1) === 0) {
14962
15038
  Atomics.add(state.header, GRAPH_TOMBSTONE_COUNT, 1);
14963
15039
  }
14964
- if (entryPointOf(state) !== ord) return;
14965
- lockEntry(state.locks);
14966
- try {
14967
- if (entryPointOf(state) === ord) setEntryPoint(state, highestNode(state, false));
14968
- } finally {
14969
- unlockEntry(state.locks);
14970
- }
14971
15040
  }
14972
15041
  function compactTombstones(state) {
14973
15042
  if (Atomics.load(state.header, GRAPH_TOMBSTONE_COUNT) === 0) return;
@@ -15041,7 +15110,7 @@ function valuesOf(bytes) {
15041
15110
  return new Uint32Array(copy.buffer, 0, copy.byteLength / BYTES_PER_VALUE);
15042
15111
  }
15043
15112
  function savedTopLayer(state, ordinal) {
15044
- if (ordinal === NO_ORDINAL || isTombstoned(state, ordinal)) return NO_LAYER;
15113
+ if (ordinal === NO_ORDINAL) return NO_LAYER;
15045
15114
  return Math.min(nodeMaxLayer(state, ordinal), MAX_LAYER_CAP);
15046
15115
  }
15047
15116
  function neighbourValueCount(state, topLayers, start, end) {
@@ -15107,6 +15176,25 @@ function serializeNumberedGraphParts(state, numberOfOrdinal, ordinalOfNumber, pa
15107
15176
  }
15108
15177
  return parts;
15109
15178
  }
15179
+ function graphWithoutNodes(part, firstNumber, numbers) {
15180
+ const values = valuesOf(part.neighbours);
15181
+ const levels = part.levels.slice();
15182
+ const kept = new Uint32Array(values.length);
15183
+ let cursor = 0;
15184
+ let length = 0;
15185
+ for (let at = 0; at < part.levels.length; at++) {
15186
+ const start = cursor;
15187
+ for (let layer = 0; layer < part.levels[at]; layer++) cursor += values[cursor] + 1;
15188
+ if (numbers.has(firstNumber + at)) {
15189
+ levels[at] = 0;
15190
+ continue;
15191
+ }
15192
+ kept.set(values.subarray(start, cursor), length);
15193
+ length += cursor - start;
15194
+ }
15195
+ const entryPoint = part.entryPoint !== null && numbers.has(part.entryPoint) ? null : part.entryPoint;
15196
+ return { ...part, entryPoint, levels, neighbours: littleEndianBytesOf(kept.subarray(0, length)) };
15197
+ }
15110
15198
  function endsInsideAList() {
15111
15199
  throw new NarsilError(
15112
15200
  ErrorCodes.PERSISTENCE_LOAD_FAILED,
@@ -15373,7 +15461,7 @@ function traverse(state, query, k, searchMetric, options) {
15373
15461
  if (entryPoint === -1) return { candidates, depth: useQuantized ? depth : 0, arenaQuery, qMag, native: null };
15374
15462
  setSingleEntryPoint(workspace, entryPoint);
15375
15463
  for (let layer = topLayerOf(state); layer >= 1; layer--) {
15376
- searchLayer(state, query, qMag, 1, layer, searchMetric, true, distFn, candidates);
15464
+ searchLayer(state, query, qMag, 1, layer, searchMetric, false, distFn, candidates);
15377
15465
  if (candidates.size > 0) {
15378
15466
  setSingleEntryPoint(workspace, candidates.ords[0]);
15379
15467
  }
@@ -15681,6 +15769,8 @@ function createHNSWIndex(dimension, store, config, quantizer, handles) {
15681
15769
  if (ord !== void 0) markTombstone(state, ord);
15682
15770
  },
15683
15771
  markTombstoneOrdinal: (ordinal) => markTombstone(state, ordinal),
15772
+ holdsNode: (ordinal) => hasNode(state.adjacency, ordinal),
15773
+ holdsTombstone: (ordinal) => hasNode(state.adjacency, ordinal) && isTombstoned(state, ordinal),
15684
15774
  has: (docId) => {
15685
15775
  const ord = store.getOrdinal(docId);
15686
15776
  return ord !== void 0 && hasNode(state.adjacency, ord) && !isTombstoned(state, ord);
@@ -15750,6 +15840,9 @@ function emptyFieldBeforeRestore(state) {
15750
15840
  state.savedFiles = [];
15751
15841
  state.savedSignature = null;
15752
15842
  }
15843
+ function releaseReplacedVectors(state) {
15844
+ for (const ordinal of state.store.releaseReplacedVectors()) state.osq?.removeOrdinal(ordinal);
15845
+ }
15753
15846
  function adoptGraph(state, graph) {
15754
15847
  if (graph === null && state.hnsw !== null) {
15755
15848
  state.hnsw.clear();
@@ -16309,9 +16402,7 @@ function captureCloneCodes(state) {
16309
16402
  const records = new Uint8Array(slots * layout.slotStride);
16310
16403
  const present = new Uint8Array(slots);
16311
16404
  for (let ordinal = 0; ordinal < slots; ordinal++) {
16312
- const docId = state.store.docIdForOrdinal(ordinal);
16313
- if (docId === void 0) continue;
16314
- const record = quantizer.recordOf(docId);
16405
+ const record = quantizer.recordAtOrdinal(ordinal);
16315
16406
  if (record === void 0) continue;
16316
16407
  records.set(record, ordinal * layout.slotStride);
16317
16408
  present[ordinal] = 1;
@@ -16448,7 +16539,7 @@ async function dispatcherFor(state, graph) {
16448
16539
  workerCount: Math.max(1, host.workerCount)
16449
16540
  };
16450
16541
  }
16451
- if (liveSize(state) < WORKER_COPY_MIN_VECTORS) return null;
16542
+ if (liveSize(state) < WORKER_COPY_MIN_VECTORS || !sharedMemoryAvailable()) return null;
16452
16543
  const handle = await shareGraph(state, graph, graph === state.hnsw);
16453
16544
  const pool = state.workerCopyPool;
16454
16545
  if (handle === null || pool === null || state.sharedHandles.get(graph)?.mode !== "shared") return null;
@@ -16538,12 +16629,40 @@ async function insertLocally(state, graph, docIds, admit) {
16538
16629
  state.buffer.delete(docId);
16539
16630
  count += 1;
16540
16631
  if (count % BUILD_CHUNK_SIZE === 0) {
16632
+ noteWrite(state);
16541
16633
  await yieldToEventLoop2();
16542
16634
  }
16543
16635
  }
16636
+ if (count > 0) noteWrite(state);
16544
16637
  return !state.disposed;
16545
16638
  }
16546
16639
 
16640
+ // src/vector/vector-index/dead-bits.ts
16641
+ var NO_LIVE_VECTOR_AT_POSITION = -1;
16642
+ function deadBytesFor(count) {
16643
+ return Math.ceil(count / 8);
16644
+ }
16645
+ function isDead(dead, position) {
16646
+ return dead !== null && (dead[position >> 3] >> (position & 7) & 1) === 1;
16647
+ }
16648
+ function countDead(dead, count) {
16649
+ if (dead === null) return 0;
16650
+ let found = 0;
16651
+ for (let position = 0; position < count; position++) {
16652
+ if (isDead(dead, position)) found += 1;
16653
+ }
16654
+ return found;
16655
+ }
16656
+ function deadBitsWhere(count, isDeadAt) {
16657
+ let dead = null;
16658
+ for (let position = 0; position < count; position++) {
16659
+ if (!isDeadAt(position)) continue;
16660
+ dead ??= new Uint8Array(deadBytesFor(count));
16661
+ dead[position >> 3] |= 1 << (position & 7);
16662
+ }
16663
+ return dead;
16664
+ }
16665
+
16547
16666
  // src/vector/vector-index/disk.ts
16548
16667
  function readsFromDisk(state) {
16549
16668
  return state.storage === "disk";
@@ -16554,6 +16673,7 @@ function holdsVectorsOnDisk(state) {
16554
16673
  function* locationsOf(state, layout) {
16555
16674
  const stride = state.dimension * 4;
16556
16675
  for (let i = 0; i < layout.docIds.length; i++) {
16676
+ if (isDead(layout.dead, i)) continue;
16557
16677
  yield [layout.docIds[i], { path: layout.path, offset: layout.vectorsOffset + i * stride }];
16558
16678
  }
16559
16679
  }
@@ -16944,32 +17064,6 @@ function graphPartsOfEarlierPayload(payload) {
16944
17064
  };
16945
17065
  }
16946
17066
 
16947
- // src/vector/vector-index/dead-bits.ts
16948
- var NO_LIVE_VECTOR_AT_POSITION = -1;
16949
- function deadBytesFor(count) {
16950
- return Math.ceil(count / 8);
16951
- }
16952
- function isDead(dead, position) {
16953
- return dead !== null && (dead[position >> 3] >> (position & 7) & 1) === 1;
16954
- }
16955
- function countDead(dead, count) {
16956
- if (dead === null) return 0;
16957
- let found = 0;
16958
- for (let position = 0; position < count; position++) {
16959
- if (isDead(dead, position)) found += 1;
16960
- }
16961
- return found;
16962
- }
16963
- function deadBitsWhere(count, isDeadAt) {
16964
- let dead = null;
16965
- for (let position = 0; position < count; position++) {
16966
- if (!isDeadAt(position)) continue;
16967
- dead ??= new Uint8Array(deadBytesFor(count));
16968
- dead[position >> 3] |= 1 << (position & 7);
16969
- }
16970
- return dead;
16971
- }
16972
-
16973
17067
  // src/vector/vector-index/persistence.ts
16974
17068
  function recordFor(state, quantizer, docId) {
16975
17069
  const record = quantizer.recordOf(docId);
@@ -16991,7 +17085,7 @@ function sameCentroid(state, planned) {
16991
17085
  }
16992
17086
  return true;
16993
17087
  }
16994
- function partCodes(state, docIds, centroid) {
17088
+ function partCodes(state, docIds, centroid, tombstonedAt) {
16995
17089
  const quantizer = state.osq;
16996
17090
  const layout = state.store.handles.codeLayout;
16997
17091
  if (centroid === null || quantizer === null || layout === null) return null;
@@ -17004,7 +17098,8 @@ function partCodes(state, docIds, centroid) {
17004
17098
  }
17005
17099
  const records = new Uint8Array(docIds.length * layout.slotStride);
17006
17100
  for (let i = 0; i < docIds.length; i++) {
17007
- const record = recordFor(state, quantizer, docIds[i]);
17101
+ const tombstoned = tombstonedAt.get(i);
17102
+ const record = tombstoned === void 0 ? recordFor(state, quantizer, docIds[i]) : quantizer.recordAtOrdinal(tombstoned);
17008
17103
  if (record !== void 0) records.set(record, i * layout.slotStride);
17009
17104
  }
17010
17105
  return { bits: quantizer.bits, centroid, records };
@@ -17161,33 +17256,39 @@ function isLive(state, ordinal) {
17161
17256
  const docId = state.store.docIdForOrdinal(ordinal);
17162
17257
  return docId !== void 0 && !state.tombstones.has(docId);
17163
17258
  }
17164
- function liveCountOf(state, file) {
17165
- let live = 0;
17166
- for (let i = 0; i < file.ordinals.length; i++) if (isLive(state, file.ordinals[i])) live += 1;
17167
- return live;
17259
+ function isGraphTombstone(state, ordinal) {
17260
+ return ordinal !== NO_LIVE_VECTOR_AT_POSITION && state.hnsw !== null && state.hnsw.holdsTombstone(ordinal);
17261
+ }
17262
+ function isCarried(state, ordinal) {
17263
+ return isLive(state, ordinal) || isGraphTombstone(state, ordinal);
17264
+ }
17265
+ function carriedCountOf(state, file) {
17266
+ let carried = 0;
17267
+ for (let i = 0; i < file.ordinals.length; i++) if (isCarried(state, file.ordinals[i])) carried += 1;
17268
+ return carried;
17168
17269
  }
17169
17270
  function deadBitsOf(state, file) {
17170
17271
  return deadBitsWhere(file.ordinals.length, (position) => !isLive(state, file.ordinals[position]));
17171
17272
  }
17172
- function filesWorthKeeping(state, listed, calibration, liveCounts) {
17273
+ function filesWorthKeeping(state, listed, calibration, carriedCounts) {
17173
17274
  const kept = [];
17174
17275
  for (const file of listed) {
17175
- const live = liveCountOf(state, file);
17176
- const deadShare = file.ordinals.length === 0 ? 1 : 1 - live / file.ordinals.length;
17177
- if (file.calibration !== calibration || deadShare > VECTOR_FILE_REPLACED_ABOVE_DEAD_SHARE) continue;
17276
+ const carried = carriedCountOf(state, file);
17277
+ const deadWithoutNodeShare = file.ordinals.length === 0 ? 1 : 1 - carried / file.ordinals.length;
17278
+ if (file.calibration !== calibration || deadWithoutNodeShare > VECTOR_FILE_REPLACED_ABOVE_DEAD_SHARE) continue;
17178
17279
  kept.push(file);
17179
- liveCounts.push(live);
17280
+ carriedCounts.push(carried);
17180
17281
  }
17181
17282
  return kept;
17182
17283
  }
17183
- function absorbSmallerPartialFiles(kept, liveCounts, vectorsToWrite, capacity) {
17284
+ function absorbSmallerPartialFiles(kept, carriedCounts, vectorsToWrite, capacity) {
17184
17285
  let pending = vectorsToWrite;
17185
17286
  for (let index = kept.length - 1; index >= 0 && pending > 0; index--) {
17186
17287
  if (kept[index].ordinals.length >= capacity) continue;
17187
- if (liveCounts[index] > pending) return;
17188
- pending += liveCounts[index];
17288
+ if (carriedCounts[index] > pending) return;
17289
+ pending += carriedCounts[index];
17189
17290
  kept.splice(index, 1);
17190
- liveCounts.splice(index, 1);
17291
+ carriedCounts.splice(index, 1);
17191
17292
  }
17192
17293
  }
17193
17294
  function ordinalsOutsideOf(state, kept) {
@@ -17201,14 +17302,15 @@ function ordinalsOutsideOf(state, kept) {
17201
17302
  }
17202
17303
  const outside = [];
17203
17304
  for (let ordinal = 0; ordinal < slots; ordinal++) {
17204
- if (saved[ordinal] === 0 && isLive(state, ordinal)) outside.push(ordinal);
17305
+ if (saved[ordinal] === 0 && isCarried(state, ordinal)) outside.push(ordinal);
17205
17306
  }
17206
17307
  return outside;
17207
17308
  }
17208
- function countOutside(state, liveCounts) {
17309
+ function countOutside(state, carriedCounts) {
17209
17310
  let held = 0;
17210
- for (const live of liveCounts) held += live;
17211
- return Math.max(0, state.store.size - state.tombstones.size - held);
17311
+ for (const carried of carriedCounts) held += carried;
17312
+ const tombstonedNodes = state.hnsw?.tombstoneCount ?? 0;
17313
+ return Math.max(0, state.store.size - state.tombstones.size + tombstonedNodes - held);
17212
17314
  }
17213
17315
  function numberEveryVector(state, kept, written) {
17214
17316
  let total = written.length;
@@ -17219,7 +17321,7 @@ function numberEveryVector(state, kept, written) {
17219
17321
  for (const file of kept) {
17220
17322
  for (let i = 0; i < file.ordinals.length; i++, number++) {
17221
17323
  const ordinal = file.ordinals[i];
17222
- if (!isLive(state, ordinal)) continue;
17324
+ if (!isCarried(state, ordinal)) continue;
17223
17325
  numberOfOrdinal[ordinal] = number;
17224
17326
  ordinalOfNumber[number] = ordinal;
17225
17327
  }
@@ -17240,16 +17342,30 @@ function graphPartsOf(state, numbers) {
17240
17342
  );
17241
17343
  return parts.map((graph) => ({ v: VECTOR_GRAPH_PART_PAYLOAD_VERSION, fieldName: state.fieldName, graphs: [graph] }));
17242
17344
  }
17243
- function readFile(state, planned, docIds, centroid) {
17345
+ function graphPartsWithoutNodes(parts, numbers) {
17346
+ return parts.map((part, index) => ({
17347
+ ...part,
17348
+ graphs: part.graphs.map((graph) => graphWithoutNodes(graph, index * VECTOR_GRAPH_PART_VECTORS, numbers))
17349
+ }));
17350
+ }
17351
+ function plannedVectorOf(state, ordinal) {
17352
+ const docId = state.store.docIdForOrdinal(ordinal) ?? state.store.view.docIdAt(ordinal) ?? "";
17353
+ return { ordinal, docId, tombstoned: !isLive(state, ordinal) };
17354
+ }
17355
+ function readFile(state, planned, centroid) {
17244
17356
  const dimension = state.dimension;
17245
17357
  const vectors = new Float32Array(planned.length * dimension);
17246
17358
  const ordinals = new Int32Array(planned.length).fill(NO_LIVE_VECTOR_AT_POSITION);
17359
+ const tombstonedAt = /* @__PURE__ */ new Map();
17360
+ const docIds = planned.map((vector) => vector.docId);
17247
17361
  for (let i = 0; i < planned.length; i++) {
17248
- const held = state.store.holdsOrdinal(planned[i]) ? planned[i] : state.store.getOrdinal(docIds[i]);
17362
+ const { ordinal, docId, tombstoned } = planned[i];
17363
+ const held = tombstoned || state.store.holdsOrdinal(ordinal) ? ordinal : state.store.getOrdinal(docId);
17249
17364
  const entry = held === void 0 ? void 0 : state.store.entryForOrdinal(held);
17250
17365
  if (held === void 0 || entry === void 0) continue;
17251
17366
  vectors.set(entry.vector, i * dimension);
17252
17367
  ordinals[i] = held;
17368
+ if (tombstoned) tombstonedAt.set(i, held);
17253
17369
  }
17254
17370
  return {
17255
17371
  payload: {
@@ -17257,10 +17373,11 @@ function readFile(state, planned, docIds, centroid) {
17257
17373
  fieldName: state.fieldName,
17258
17374
  dimension,
17259
17375
  docIds,
17260
- codes: partCodes(state, docIds, centroid),
17376
+ codes: partCodes(state, docIds, centroid, tombstonedAt),
17261
17377
  vectors: vectorsToBytes(vectors)
17262
17378
  },
17263
- ordinals
17379
+ ordinals,
17380
+ dead: deadBitsWhere(planned.length, (i) => planned[i].tombstoned || ordinals[i] === NO_LIVE_VECTOR_AT_POSITION)
17264
17381
  };
17265
17382
  }
17266
17383
  function planCheckpoint(state, listedKeys, rewritesGraph = false) {
@@ -17268,14 +17385,20 @@ function planCheckpoint(state, listedKeys, rewritesGraph = false) {
17268
17385
  const calibration = signature.calibration;
17269
17386
  const trusted = registryMatches(state, listedKeys);
17270
17387
  const capacity = fileCapacityOf(state);
17271
- const liveCounts = [];
17272
- const keptFiles = filesWorthKeeping(state, trusted ? state.savedFiles : [], calibration, liveCounts);
17273
- absorbSmallerPartialFiles(keptFiles, liveCounts, countOutside(state, liveCounts), capacity);
17388
+ const carriedCounts = [];
17389
+ const keptFiles = filesWorthKeeping(state, trusted ? state.savedFiles : [], calibration, carriedCounts);
17390
+ absorbSmallerPartialFiles(keptFiles, carriedCounts, countOutside(state, carriedCounts), capacity);
17274
17391
  const written = ordinalsOutsideOf(state, keptFiles);
17275
- const writtenDocIds = written.map((ordinal) => state.store.docIdForOrdinal(ordinal) ?? "");
17392
+ const planned = written.map((ordinal) => plannedVectorOf(state, ordinal));
17276
17393
  const centroid = codedCentroid(state);
17277
17394
  const listedBefore = trusted && listedKeys !== null;
17278
17395
  const unchanged = !rewritesGraph && listedBefore && sameSignature(state.savedSignature, signature) && written.length === 0;
17396
+ let firstWrittenNumber = 0;
17397
+ for (const file of keptFiles) firstWrittenNumber += file.ordinals.length;
17398
+ const graphParts = unchanged ? null : graphPartsOf(state, numberEveryVector(state, keptFiles, written));
17399
+ const vanished = /* @__PURE__ */ new Set();
17400
+ let vanishedWhenStripped = 0;
17401
+ let graphPartsToWrite = graphParts;
17279
17402
  return {
17280
17403
  unchanged,
17281
17404
  keptFiles,
@@ -17286,12 +17409,22 @@ function planCheckpoint(state, listedKeys, rewritesGraph = false) {
17286
17409
  dead: deadBitsOf(state, file)
17287
17410
  })),
17288
17411
  newFiles: Math.ceil(written.length / capacity),
17289
- graphParts: unchanged ? null : graphPartsOf(state, numberEveryVector(state, keptFiles, written)),
17412
+ get graphParts() {
17413
+ if (graphParts !== null && vanished.size !== vanishedWhenStripped) {
17414
+ graphPartsToWrite = graphPartsWithoutNodes(graphParts, vanished);
17415
+ vanishedWhenStripped = vanished.size;
17416
+ }
17417
+ return graphPartsToWrite;
17418
+ },
17290
17419
  signature,
17291
17420
  readNewFile(index) {
17292
17421
  const start = index * capacity;
17293
17422
  const end = start + capacity;
17294
- return readFile(state, written.slice(start, end), writtenDocIds.slice(start, end), centroid);
17423
+ const file = readFile(state, planned.slice(start, end), centroid);
17424
+ for (let i = 0; i < file.ordinals.length; i++) {
17425
+ if (file.ordinals[i] === NO_LIVE_VECTOR_AT_POSITION) vanished.add(firstWrittenNumber + start + i);
17426
+ }
17427
+ return file;
17295
17428
  }
17296
17429
  };
17297
17430
  }
@@ -17324,9 +17457,10 @@ function insertNodesOfLaterGraph(state, graph, later, ordinals) {
17324
17457
  if (docId !== void 0 && !graph.has(docId)) graph.insertOrdinal(ordinal);
17325
17458
  }
17326
17459
  }
17327
- function restoreGraph(state, payload, ordinalOfNumber) {
17460
+ function restoreGraph(state, payload, ordinalOfNumber, tombstoned) {
17328
17461
  const first = payload?.graphs[0]?.parts[0];
17329
17462
  if (payload === null || first === void 0) {
17463
+ releaseReplacedVectors(state);
17330
17464
  bufferEveryVectorOutsideTheGraph(state);
17331
17465
  return;
17332
17466
  }
@@ -17337,11 +17471,13 @@ function restoreGraph(state, payload, ordinalOfNumber) {
17337
17471
  state.osq ?? void 0
17338
17472
  );
17339
17473
  graph.deserializeNumbered(payload.graphs[0], ordinalOfNumber);
17474
+ for (const ordinal of tombstoned) graph.markTombstoneOrdinal(ordinal);
17340
17475
  adoptGraph(state, graph);
17341
17476
  for (const later of payload.graphs.slice(1)) insertNodesOfLaterGraph(state, graph, later, ordinalOfNumber);
17342
17477
  bufferEveryVectorOutsideTheGraph(state);
17343
17478
  if (state.buffer.size <= graph.size) return;
17344
17479
  adoptGraph(state, null);
17480
+ releaseReplacedVectors(state);
17345
17481
  state.buffer.clear();
17346
17482
  bufferEveryVectorOutsideTheGraph(state);
17347
17483
  }
@@ -17350,6 +17486,9 @@ function beginCheckpointRestore(state, shape) {
17350
17486
  const cold = readsFromDisk(state) && (shape.holdsGraph || shape.liveVectors >= state.promotionThreshold);
17351
17487
  const restoresCodes = state.osq !== null && osqBitsOf(state.quantizationMode) !== null && shape.holdsGraph;
17352
17488
  const files = [];
17489
+ const graphNodes = shape.graph?.graphs[0] ?? null;
17490
+ const tombstoned = /* @__PURE__ */ new Set();
17491
+ let nextNumber = 0;
17353
17492
  let sharedCodes = null;
17354
17493
  let codesAgree = true;
17355
17494
  function restoreRecords(payload, ordinals) {
@@ -17371,9 +17510,14 @@ function beginCheckpointRestore(state, shape) {
17371
17510
  const stride = layout.slotStride;
17372
17511
  for (let i = 0; i < ordinals.length; i++) {
17373
17512
  if (ordinals[i] === NO_LIVE_VECTOR_AT_POSITION) continue;
17374
- quantizer.restoreRecord(payload.docIds[i], codes.records.subarray(i * stride, (i + 1) * stride));
17513
+ const record = codes.records.subarray(i * stride, (i + 1) * stride);
17514
+ if (tombstoned.has(ordinals[i])) quantizer.restoreRecordAtOrdinal(ordinals[i], record);
17515
+ else quantizer.restoreRecord(payload.docIds[i], record);
17375
17516
  }
17376
17517
  }
17518
+ function graphHoldsNodeFor(number) {
17519
+ return graphNodes !== null && topLayerInParts(graphNodes, number) >= 0;
17520
+ }
17377
17521
  function addFile(file) {
17378
17522
  const { payload, location } = file;
17379
17523
  if (payload.dimension !== state.dimension) {
@@ -17388,13 +17532,21 @@ function beginCheckpointRestore(state, shape) {
17388
17532
  const vectors = bytesToVectors(payload.vectors);
17389
17533
  const fileIndex = onDisk && cold ? state.store.addVectorFile(location.path) : null;
17390
17534
  const ordinals = new Int32Array(payload.docIds.length).fill(NO_LIVE_VECTOR_AT_POSITION);
17535
+ const firstNumber = nextNumber;
17536
+ nextNumber += payload.docIds.length;
17391
17537
  for (let i = 0; i < payload.docIds.length; i++) {
17392
- if (isDead(file.dead, i)) continue;
17538
+ const dead = isDead(file.dead, i);
17539
+ if (dead && !graphHoldsNodeFor(firstNumber + i)) continue;
17393
17540
  const docId = payload.docIds[i];
17394
- const replaced = state.store.getOrdinal(docId);
17395
- if (replaced !== void 0) state.osq?.removeOrdinal(replaced);
17396
17541
  const vector = vectors.subarray(i * dimension, (i + 1) * dimension);
17397
17542
  const offset = onDisk ? location.vectorsOffset + i * dimension * 4 : 0;
17543
+ if (dead) {
17544
+ ordinals[i] = state.store.insertReplaced(docId, vector, fileIndex === null ? void 0 : { fileIndex, offset });
17545
+ tombstoned.add(ordinals[i]);
17546
+ continue;
17547
+ }
17548
+ const replaced = state.store.getOrdinal(docId);
17549
+ if (replaced !== void 0) state.osq?.removeOrdinal(replaced);
17398
17550
  if (fileIndex !== null) {
17399
17551
  ordinals[i] = state.store.insertCold(docId, magnitude(vector), { fileIndex, offset });
17400
17552
  continue;
@@ -17405,21 +17557,27 @@ function beginCheckpointRestore(state, shape) {
17405
17557
  restoreRecords(payload, ordinals);
17406
17558
  files.push({ id: file.id, key: file.key, ordinals, coded: payload.codes !== null });
17407
17559
  }
17408
- function finish(graph) {
17560
+ function finish() {
17409
17561
  const codesRestored = restoresCodes && codesAgree && sharedCodes !== null;
17410
- if (restoresCodes && !codesRestored) recalibrateFromStore(state);
17562
+ if (restoresCodes && !codesRestored) {
17563
+ recalibrateFromStore(state);
17564
+ for (const ordinal of tombstoned) state.osq?.writeCodes(ordinal);
17565
+ }
17411
17566
  let total = 0;
17412
17567
  for (const file of files) total += file.ordinals.length;
17413
17568
  const ordinalOfNumber = new Int32Array(total);
17414
17569
  let number = 0;
17415
17570
  for (const file of files) {
17416
17571
  for (let i = 0; i < file.ordinals.length; i++) {
17417
- if (!state.store.holdsOrdinal(file.ordinals[i])) file.ordinals[i] = NO_LIVE_VECTOR_AT_POSITION;
17572
+ const ordinal = file.ordinals[i];
17573
+ if (!state.store.holdsOrdinal(ordinal) && !tombstoned.has(ordinal)) {
17574
+ file.ordinals[i] = NO_LIVE_VECTOR_AT_POSITION;
17575
+ }
17418
17576
  }
17419
17577
  ordinalOfNumber.set(file.ordinals, number);
17420
17578
  number += file.ordinals.length;
17421
17579
  }
17422
- restoreGraph(state, graph, ordinalOfNumber);
17580
+ restoreGraph(state, shape.graph, ordinalOfNumber, tombstoned);
17423
17581
  const current = signatureOf(state).calibration;
17424
17582
  const codedUnder = codesRestored && current !== null ? current : CALIBRATION_NEVER_CURRENT;
17425
17583
  const saved = files.map((file) => ({
@@ -17436,11 +17594,12 @@ function beginCheckpointRestore(state, shape) {
17436
17594
 
17437
17595
  // src/vector/vector-index/maintenance.ts
17438
17596
  function compact(state) {
17439
- if (state.tombstones.size === 0) return;
17597
+ if (state.tombstones.size === 0 && state.store.replacedVectorCount === 0) return;
17440
17598
  if (state.hnsw) {
17441
17599
  state.compactedNodeCount += state.hnsw.tombstoneCount;
17442
17600
  state.hnsw.compactTombstones();
17443
17601
  }
17602
+ state.freshGraph?.compactTombstones();
17444
17603
  for (const docId of state.tombstones) {
17445
17604
  state.store.remove(docId);
17446
17605
  state.buffer.delete(docId);
@@ -17448,6 +17607,7 @@ function compact(state) {
17448
17607
  state.osq.remove(docId);
17449
17608
  }
17450
17609
  }
17610
+ releaseReplacedVectors(state);
17451
17611
  state.tombstones.clear();
17452
17612
  }
17453
17613
  function recalibrateWhileNoThreadSearches(state, graph) {
@@ -17479,6 +17639,7 @@ async function foldIntoGraph(state) {
17479
17639
  const graph = state.hnsw;
17480
17640
  if (graph === null || rebuildNeeded) {
17481
17641
  if (state.osq?.isCalibrated()) recalibrateWhileNoThreadSearches(state, graph);
17642
+ else calibrateQuantizer(state);
17482
17643
  await buildGraphFromStore(state);
17483
17644
  } else {
17484
17645
  await insertMissing(state, graph);
@@ -17525,8 +17686,9 @@ function maintenanceStatus(state) {
17525
17686
  const storeSize = state.store.size;
17526
17687
  const graphCount = state.hnsw ? 1 : 0;
17527
17688
  const graphNodes = state.hnsw === null ? storeSize : state.hnsw.size + state.hnsw.tombstoneCount;
17528
- const tombstoneRatio = graphNodes > 0 ? state.tombstones.size / graphNodes : 0;
17529
- const estimatedCompactMs = Math.round(state.tombstones.size * ESTIMATED_MS_PER_TOMBSTONE * state.dimensionScale);
17689
+ const tombstoned = state.tombstones.size + state.store.replacedVectorCount;
17690
+ const tombstoneRatio = graphNodes > 0 ? tombstoned / graphNodes : 0;
17691
+ const estimatedCompactMs = Math.round(tombstoned * ESTIMATED_MS_PER_TOMBSTONE * state.dimensionScale);
17530
17692
  const optimizeVectorCount = state.hnsw === null || graphNeedsRebuild(state) ? storeSize : state.buffer.size;
17531
17693
  const estimatedOptimizeMs = Math.round(optimizeVectorCount * ESTIMATED_MS_PER_VECTOR_REBUILD * state.dimensionScale);
17532
17694
  return {
@@ -17786,19 +17948,19 @@ function searchWithFilter(state, query, k, options, filter) {
17786
17948
  return scannedOutcome(scan2);
17787
17949
  }
17788
17950
  }
17789
- const graphOptions = { filter, efSearch, oversample };
17790
- if (state.buffer.size === 0) {
17791
- const hnswResults2 = state.hnsw.search(query, k, metric, minSimilarity, graphOptions);
17792
- return fetchedOutcome(
17793
- state,
17794
- hnswResults2.map((r) => ({ docId: r.docId, score: r.score })),
17795
- minSimilarity,
17796
- filter
17797
- );
17798
- }
17799
- const hnswResults = state.hnsw.search(query, k, metric, minSimilarity, graphOptions).map((r) => ({ docId: r.docId, score: r.score }));
17800
- const buffered = bruteForceSearch(state, query, k, metric, minSimilarity, bufferCandidates(state, filter));
17801
- return fetchedOutcome(state, mergeResults(hnswResults, buffered.results, k), minSimilarity, filter);
17951
+ const hnswResults = state.hnsw.search(query, k, metric, minSimilarity, { filter, efSearch, oversample }).map((r) => ({ docId: r.docId, score: r.score }));
17952
+ return fetchedOutcome(
17953
+ state,
17954
+ withBufferedVectors(state, query, k, options, filter, hnswResults),
17955
+ minSimilarity,
17956
+ filter
17957
+ );
17958
+ }
17959
+ function withBufferedVectors(state, query, k, options, filter, graphResults) {
17960
+ if (state.buffer.size === 0) return graphResults;
17961
+ const candidates = bufferCandidates(state, filter);
17962
+ const buffered = bruteForceSearch(state, query, k, options.metric, options.minSimilarity, candidates);
17963
+ return mergeResults(graphResults, buffered.results, k);
17802
17964
  }
17803
17965
 
17804
17966
  // src/vector/vector-index/index.ts
@@ -17899,11 +18061,12 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
17899
18061
  }
17900
18062
  noteWrite(state);
17901
18063
  state.tombstones.delete(docId);
17902
- state.store.insert(docId, vector, partitionId);
18064
+ const graphHoldsPrevious = previous !== void 0 && ((state.hnsw?.holdsNode(previous) ?? false) || (state.freshGraph?.holdsNode(previous) ?? false));
18065
+ state.store.insert(docId, vector, partitionId, graphHoldsPrevious);
17903
18066
  if (previous !== void 0) {
17904
18067
  state.hnsw?.markTombstoneOrdinal(previous);
17905
18068
  state.freshGraph?.markTombstoneOrdinal(previous);
17906
- state.osq?.removeOrdinal(previous);
18069
+ if (!graphHoldsPrevious) state.osq?.removeOrdinal(previous);
17907
18070
  }
17908
18071
  state.buffer.add(docId);
17909
18072
  state.pendingLocations.delete(docId);
@@ -17973,10 +18136,13 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
17973
18136
  efSearch: options.efSearch,
17974
18137
  oversample: options.oversample
17975
18138
  });
17976
- if (viaWorkerCopy !== null) return fetchedOutcome(state, viaWorkerCopy, options.minSimilarity, filter);
17977
18139
  if (confined && state.revision !== filterRevision) {
17978
18140
  filter = filterForOptions(state, options);
17979
18141
  }
18142
+ if (viaWorkerCopy !== null) {
18143
+ const results = withBufferedVectors(state, query, k, options, filter, viaWorkerCopy);
18144
+ return fetchedOutcome(state, results, options.minSimilarity, filter);
18145
+ }
17980
18146
  return searchWithFilter(state, query, k, options, filter);
17981
18147
  }
17982
18148
  return {
@@ -19379,10 +19545,10 @@ function indexByFieldPath(entries) {
19379
19545
  for (const entry of entries) byFieldPath.set(entry.fieldPath, entry);
19380
19546
  return byFieldPath;
19381
19547
  }
19382
- function rowsByFieldPath(inputs, entriesOf, valuesOf2) {
19548
+ function rowsByFieldPath(inputs, entriesOf2, valuesOf2) {
19383
19549
  const byFieldPath = /* @__PURE__ */ new Map();
19384
19550
  for (const input of inputs) {
19385
- for (const entry of entriesOf(input)) {
19551
+ for (const entry of entriesOf2(input)) {
19386
19552
  let rows = byFieldPath.get(entry.fieldPath);
19387
19553
  if (rows === void 0) {
19388
19554
  rows = [];
@@ -21088,7 +21254,6 @@ var DEFAULT_ASYNC_FLUSH_INTERVAL_MS = 1e3;
21088
21254
  var DIRECTORY_IDENTITY_CHECK_INTERVAL_MS = 1e3;
21089
21255
  var DIRECTORY_LOCK_ATTEMPTS = 3;
21090
21256
  var DIRECTORY_LOCK_SETTLE_MS = 50;
21091
- var DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS = 1e3;
21092
21257
  var DEFAULT_SEGMENT_MAX_BYTES = 67108864;
21093
21258
  var DEFAULT_COMPACTION_THRESHOLD = 12;
21094
21259
  var COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT = 2e5;
@@ -21116,6 +21281,16 @@ async function getPath() {
21116
21281
  return filesystemUnavailable2();
21117
21282
  }
21118
21283
 
21284
+ // src/persistence/durability/lock-holder.browser.ts
21285
+ async function holderIsLive(_holder, _self, _lockPath) {
21286
+ return true;
21287
+ }
21288
+ function holdDirectoryOpen(_directory) {
21289
+ return () => void 0;
21290
+ }
21291
+ async function removeStaleLock(_lockPath, _asidePath, _isStale) {
21292
+ }
21293
+
21119
21294
  // src/persistence/durability/directory-lock.ts
21120
21295
  var LOCK_FILE_NAME = ".narsil.lock";
21121
21296
  var HARD_LINK_UNSUPPORTED_CODES = /* @__PURE__ */ new Set(["EPERM", "ENOTSUP", "EOPNOTSUPP", "ENOSYS"]);
@@ -21123,14 +21298,6 @@ var lockedDirectories = /* @__PURE__ */ new Set();
21123
21298
  function errnoOf(err) {
21124
21299
  return err?.code;
21125
21300
  }
21126
- function processIsRunning(pid) {
21127
- try {
21128
- process.kill(pid, 0);
21129
- return true;
21130
- } catch (err) {
21131
- return errnoOf(err) === "EPERM";
21132
- }
21133
- }
21134
21301
  function integerOrNull(text) {
21135
21302
  const value = Number.parseInt(text ?? "", 10);
21136
21303
  return Number.isSafeInteger(value) ? value : null;
@@ -21168,11 +21335,6 @@ async function settledHolder(fs, lockPath) {
21168
21335
  await new Promise((resolve) => setTimeout(resolve, DIRECTORY_LOCK_SETTLE_MS));
21169
21336
  return readHolder(fs, lockPath);
21170
21337
  }
21171
- function holderIsLive(holder, self) {
21172
- if (holder.pid !== self.pid) return processIsRunning(holder.pid);
21173
- if (holder.processStartMs === null || self.processStartMs === null) return false;
21174
- return Math.abs(holder.processStartMs - self.processStartMs) <= DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS;
21175
- }
21176
21338
  function heldElsewhere(root, holder, lockPath) {
21177
21339
  return new NarsilError(
21178
21340
  ErrorCodes.CONFIG_INVALID,
@@ -21215,12 +21377,10 @@ async function acquire(fs, root, lockPath, self) {
21215
21377
  for (let attempt = 0; attempt < DIRECTORY_LOCK_ATTEMPTS; attempt++) {
21216
21378
  if (await createLockFile(fs, lockPath, draftPath, content)) return;
21217
21379
  const holder = await settledHolder(fs, lockPath);
21218
- if (typeof holder === "object" && holderIsLive(holder, self)) {
21380
+ if (typeof holder === "object" && await holderIsLive()) {
21219
21381
  throw heldElsewhere(root, holderLabel(holder, self), lockPath);
21220
21382
  }
21221
- await fs.unlink(lockPath).catch((err) => {
21222
- if (errnoOf(err) !== "ENOENT") throw err;
21223
- });
21383
+ await removeStaleLock();
21224
21384
  }
21225
21385
  throw heldElsewhere(root, "another process", lockPath);
21226
21386
  }
@@ -21238,7 +21398,15 @@ async function lockDirectory(root) {
21238
21398
  const resolvedRoot = pathMod.resolve(root);
21239
21399
  const lockPath = pathMod.join(resolvedRoot, LOCK_FILE_NAME);
21240
21400
  const self = await describeSelf();
21401
+ async function holds() {
21402
+ const holder = await readHolder(fs, lockPath);
21403
+ return typeof holder === "object" && holder.nonce === self.nonce;
21404
+ }
21405
+ async function removeOwnLockFile() {
21406
+ if (await holds().catch(() => false)) await fs.unlink(lockPath).catch(() => void 0);
21407
+ }
21241
21408
  let directoryKey = null;
21409
+ let closeDirectory = () => void 0;
21242
21410
  try {
21243
21411
  await fs.mkdir(resolvedRoot, { recursive: true });
21244
21412
  const stat = await fs.stat(resolvedRoot, { bigint: true });
@@ -21247,8 +21415,10 @@ async function lockDirectory(root) {
21247
21415
  lockedDirectories.add(key);
21248
21416
  directoryKey = key;
21249
21417
  await acquire(fs, root, lockPath, self);
21418
+ closeDirectory = holdDirectoryOpen(resolvedRoot);
21250
21419
  } catch (err) {
21251
21420
  if (directoryKey !== null) lockedDirectories.delete(directoryKey);
21421
+ await removeOwnLockFile();
21252
21422
  if (err instanceof NarsilError) throw err;
21253
21423
  throw new NarsilError(
21254
21424
  ErrorCodes.PERSISTENCE_SAVE_FAILED,
@@ -21258,17 +21428,14 @@ async function lockDirectory(root) {
21258
21428
  }
21259
21429
  const heldKey = directoryKey;
21260
21430
  let released = false;
21261
- async function holds() {
21262
- const holder = await readHolder(fs, lockPath);
21263
- return typeof holder === "object" && holder.nonce === self.nonce;
21264
- }
21265
21431
  return {
21266
21432
  holds,
21267
21433
  async release() {
21268
21434
  if (released) return;
21269
21435
  released = true;
21270
21436
  lockedDirectories.delete(heldKey);
21271
- if (await holds().catch(() => false)) await fs.unlink(lockPath).catch(() => void 0);
21437
+ await removeOwnLockFile();
21438
+ closeDirectory();
21272
21439
  }
21273
21440
  };
21274
21441
  }
@@ -23780,7 +23947,8 @@ async function readVectorFile(store, file, fieldPath) {
23780
23947
  async function loadVectorField(store, ref, vectorIndex) {
23781
23948
  const restore = vectorIndex.restoreCheckpoint({
23782
23949
  liveVectors: liveVectorsOf(ref),
23783
- holdsGraph: ref.graphParts.length > 0
23950
+ holdsGraph: ref.graphParts.length > 0,
23951
+ graph: await readGraphParts(store, ref)
23784
23952
  });
23785
23953
  for (const file of ref.files) {
23786
23954
  const { payload, payloadLength } = await readVectorFile(store, file, ref.fieldPath);
@@ -23795,7 +23963,7 @@ async function loadVectorField(store, ref, vectorIndex) {
23795
23963
  }
23796
23964
  });
23797
23965
  }
23798
- restore.finish(await readGraphParts(store, ref));
23966
+ restore.finish();
23799
23967
  }
23800
23968
  var EMPTY_BIN = new Uint8Array(0);
23801
23969
  var EMPTY_BIN_BYTES = 2;
@@ -23859,7 +24027,7 @@ async function writeField(input, fieldPath, plan, outcome) {
23859
24027
  const files = plan.kept.map((kept) => keptFileRef(input, prior, kept));
23860
24028
  const written = [];
23861
24029
  for (let index = 0; index < plan.newFiles; index++) {
23862
- const { payload, ordinals } = plan.readNewFile(index);
24030
+ const { payload, ordinals, dead } = plan.readNewFile(index);
23863
24031
  const id = nextFileId;
23864
24032
  nextFileId += 1;
23865
24033
  const key = vectorFileKey(input.indexName, fieldPath, id);
@@ -23869,13 +24037,13 @@ async function writeField(input, fieldPath, plan, outcome) {
23869
24037
  `vector file "${key}"`
23870
24038
  );
23871
24039
  await input.store.write(key, packed.chunks);
23872
- const dead = deadBitsWhere(ordinals.length, (position) => ordinals[position] === NO_LIVE_VECTOR_AT_POSITION);
23873
24040
  files.push({ id, key, count: payload.docIds.length, dead, bytes: packed.byteLength, checksum: packed.checksum });
23874
24041
  written.push({ id, key, ordinals });
23875
24042
  outcome.layouts.push({
23876
24043
  fieldPath,
23877
24044
  key,
23878
24045
  docIds: payload.docIds,
24046
+ dead,
23879
24047
  vectorsOffset: packed.byteLength - payload.vectors.byteLength
23880
24048
  });
23881
24049
  }
@@ -25746,7 +25914,12 @@ async function adoptVectorLayouts(store, vectorIndexes, layouts) {
25746
25914
  const vecIndex = vectorIndexes.get(layout.fieldPath);
25747
25915
  if (vecIndex === void 0 || vecIndex.storage !== "disk") continue;
25748
25916
  const path = await pathOf.call(store, layout.key);
25749
- await vecIndex.adoptDiskLayout({ path, docIds: layout.docIds, vectorsOffset: layout.vectorsOffset });
25917
+ await vecIndex.adoptDiskLayout({
25918
+ path,
25919
+ docIds: layout.docIds,
25920
+ dead: layout.dead,
25921
+ vectorsOffset: layout.vectorsOffset
25922
+ });
25750
25923
  }
25751
25924
  }
25752
25925
  async function garbageThatNoFieldStillReads(store, vectorIndexes, keys) {
@@ -26701,7 +26874,11 @@ async function readGraph(store, ref) {
26701
26874
  async function loadVectorField2(store, ref, vectorIndex, fileRefs) {
26702
26875
  let liveVectors = 0;
26703
26876
  for (const file of ref.files) liveVectors += file.count - countDead(file.dead, file.count);
26704
- const restore = vectorIndex.restoreCheckpoint({ liveVectors, holdsGraph: ref.graphKey !== null });
26877
+ const restore = vectorIndex.restoreCheckpoint({
26878
+ liveVectors,
26879
+ holdsGraph: ref.graphKey !== null,
26880
+ graph: await readGraph(store, ref)
26881
+ });
26705
26882
  for (const file of ref.files) {
26706
26883
  const bytes = await store.read(file.key);
26707
26884
  if (bytes === null) {
@@ -26734,7 +26911,7 @@ async function loadVectorField2(store, ref, vectorIndex, fileRefs) {
26734
26911
  }
26735
26912
  });
26736
26913
  }
26737
- restore.finish(await readGraph(store, ref));
26914
+ restore.finish();
26738
26915
  }
26739
26916
  async function loadEarlierLayout(store, indexName, manifest, deps) {
26740
26917
  let highestPartitionId = manifest.partitions.length - 1;
@@ -34395,7 +34572,7 @@ function partitionVectorPlan(vectorIndex, docIds) {
34395
34572
  codes: null,
34396
34573
  vectors: vectorsToBytes(packed)
34397
34574
  };
34398
- files.push({ payload, ordinals: new Int32Array(slice.length) });
34575
+ files.push({ payload, ordinals: new Int32Array(slice.length), dead: null });
34399
34576
  }
34400
34577
  return {
34401
34578
  unchanged: false,