@delali/narsil 0.3.1-next.82 → 0.3.1-next.84

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7612,65 +7612,6 @@ function mergeSegmentState(target, source) {
7612
7612
  target.scoreBuffer = null;
7613
7613
  }
7614
7614
 
7615
- // src/core/partition/rebuild.ts
7616
- var EMPTY_STATS = {
7617
- totalDocuments: 0,
7618
- totalFieldLengths: {},
7619
- averageFieldLengths: {},
7620
- docFrequencies: {}
7621
- };
7622
- function textFieldsOf(fields) {
7623
- return fields.filter((field) => isWordIndexedBase(field.base));
7624
- }
7625
- function rebuildTextIndex(state, schema, language, options) {
7626
- const textFields = textFieldsOf(schemaFieldsOf(schema));
7627
- state.invertedIdx.clear();
7628
- state.surfaceRegistry.clear();
7629
- state.stats.deserialize(EMPTY_STATS);
7630
- state.invertedIdx.beginBatch();
7631
- try {
7632
- for (const [docId, stored] of state.docStore.all()) {
7633
- const internalId = state.docStore.getInternalId(docId);
7634
- if (internalId === void 0) continue;
7635
- const fields = stored.fields;
7636
- const fieldLengths = {};
7637
- const tokensByField = {};
7638
- for (const { path: fieldPath, base } of textFields) {
7639
- const value = getNestedValue(fields, fieldPath);
7640
- if (value === void 0 || value === null) continue;
7641
- if (base === "string") {
7642
- indexStringField(
7643
- state,
7644
- internalId,
7645
- fieldPath,
7646
- value,
7647
- language,
7648
- options,
7649
- fieldLengths,
7650
- tokensByField
7651
- );
7652
- } else {
7653
- indexStringArrayField(
7654
- state,
7655
- internalId,
7656
- fieldPath,
7657
- value,
7658
- language,
7659
- options,
7660
- fieldLengths,
7661
- tokensByField
7662
- );
7663
- }
7664
- }
7665
- state.docStore.storeRef(docId, stored.fields, fieldLengths);
7666
- state.stats.addDocument(fieldLengths, tokensByField);
7667
- }
7668
- } finally {
7669
- state.invertedIdx.endBatch();
7670
- state.partialWords.forgetSpellings();
7671
- }
7672
- }
7673
-
7674
7615
  // src/core/partition/block-bounds.ts
7675
7616
  var ABSENT_FIELD_LENGTH_BOUND = 0;
7676
7617
  var boundsByList = /* @__PURE__ */ new WeakMap();
@@ -9163,9 +9104,6 @@ function createPartitionIndex(partitionId, trackPositions = true) {
9163
9104
  if (internalId !== void 0) recordSortValues(state, internalId, document2);
9164
9105
  }
9165
9106
  },
9166
- rebuildTextIndex(schema, language, options) {
9167
- rebuildTextIndex(state, schema, language, options);
9168
- },
9169
9107
  get(docId, projection) {
9170
9108
  const stored = state.docStore.get(docId);
9171
9109
  if (!stored) return void 0;
@@ -9259,6 +9197,8 @@ function yieldToEventLoop() {
9259
9197
  // src/partitioning/constants.ts
9260
9198
  var REBALANCE_CHUNK_SIZE = 1e3;
9261
9199
  var DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE = 1e4;
9200
+ var TEXT_REBUILD_CHUNK_DOCUMENTS = 1e3;
9201
+ var TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS = 1e3;
9262
9202
 
9263
9203
  // src/partitioning/rebalancer.ts
9264
9204
  function createRebalancer() {
@@ -11162,18 +11102,6 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
11162
11102
  if (frozen.length === 0) return live.encodeSegment();
11163
11103
  return composedScratch().encodeSegment();
11164
11104
  },
11165
- rebuildTextIndex(schema, language, options) {
11166
- invalidateDocFrequencies();
11167
- if (frozen.length > 0) {
11168
- for (const segment of frozen) {
11169
- const payload = encodeSegmentState(segment);
11170
- if (payload.documentCount === 0) continue;
11171
- live.mergeSegmentPayload(payload, survivorDocumentsOf(segment, payload));
11172
- }
11173
- frozen.length = 0;
11174
- }
11175
- live.rebuildTextIndex(schema, language, options);
11176
- },
11177
11105
  serialize(indexName, totalPartitions, language, schema) {
11178
11106
  if (frozen.length === 0) return live.serialize(indexName, totalPartitions, language, schema);
11179
11107
  return composedScratch().serialize(indexName, totalPartitions, language, schema);
@@ -11417,6 +11345,81 @@ function setNestedValue(obj, path, value) {
11417
11345
  current[segments[segments.length - 1]] = value;
11418
11346
  }
11419
11347
 
11348
+ // src/partitioning/manager/stats.ts
11349
+ function aggregateStatsOf(partitions) {
11350
+ let totalDocuments = 0;
11351
+ const docFrequencies = /* @__PURE__ */ Object.create(null);
11352
+ const totalFieldLengths = /* @__PURE__ */ Object.create(null);
11353
+ for (let i = 0; i < partitions.length; i++) {
11354
+ const stats = partitions[i].stats;
11355
+ totalDocuments += stats.totalDocuments;
11356
+ for (const [term, freq] of Object.entries(stats.docFrequencies)) {
11357
+ docFrequencies[term] = (docFrequencies[term] ?? 0) + freq;
11358
+ }
11359
+ for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
11360
+ totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
11361
+ }
11362
+ }
11363
+ return { totalDocuments, docFrequencies, totalFieldLengths };
11364
+ }
11365
+ function estimateBytesOf(partitions, vecIndexes) {
11366
+ let total = 0;
11367
+ for (let i = 0; i < partitions.length; i++) {
11368
+ total += partitions[i].estimateMemoryBytes();
11369
+ }
11370
+ for (const [, vecIndex] of vecIndexes) {
11371
+ total += vecIndex.estimateMemoryBytes();
11372
+ }
11373
+ return total;
11374
+ }
11375
+ function partitionStatsOf(partitions) {
11376
+ return partitions.map((p, i) => ({
11377
+ partitionId: i,
11378
+ documentCount: p.count(),
11379
+ estimatedMemoryBytes: p.estimateMemoryBytes()
11380
+ }));
11381
+ }
11382
+
11383
+ // src/partitioning/manager/write-log.ts
11384
+ function createWriteLogRecorder() {
11385
+ let current = null;
11386
+ return {
11387
+ open() {
11388
+ const log = { docIds: /* @__PURE__ */ new Set(), replaced: false };
11389
+ current = log;
11390
+ return {
11391
+ get partitionsReplaced() {
11392
+ return log.replaced;
11393
+ },
11394
+ get pendingDocuments() {
11395
+ return log.docIds.size;
11396
+ },
11397
+ takeWrittenDocIds() {
11398
+ const taken = log.docIds;
11399
+ log.docIds = /* @__PURE__ */ new Set();
11400
+ return taken;
11401
+ },
11402
+ close() {
11403
+ if (current === log) current = null;
11404
+ }
11405
+ };
11406
+ },
11407
+ documentWritten(docId) {
11408
+ if (current !== null) current.docIds.add(docId);
11409
+ },
11410
+ segmentWritten(segment) {
11411
+ if (current === null) return;
11412
+ for (const internalId of segment.docStore.allInternalIds()) {
11413
+ const docId = segment.docStore.getExternalId(internalId);
11414
+ if (docId !== void 0) current.docIds.add(docId);
11415
+ }
11416
+ },
11417
+ partitionsReplaced() {
11418
+ if (current !== null) current.replaced = true;
11419
+ }
11420
+ };
11421
+ }
11422
+
11420
11423
  // src/partitioning/manager/index.ts
11421
11424
  function createPartitionManager(indexName, config, language, router, initialPartitionCount, vectorIndexes) {
11422
11425
  requirePatternSearchFor(config.schema);
@@ -11428,6 +11431,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
11428
11431
  const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
11429
11432
  let partitions = [];
11430
11433
  const docPartitionMap = /* @__PURE__ */ new Map();
11434
+ const writeLog = createWriteLogRecorder();
11431
11435
  for (let i = 0; i < count; i++) {
11432
11436
  partitions.push(createPartitionIndex(i, trackPositions));
11433
11437
  }
@@ -11515,9 +11519,23 @@ function createPartitionManager(indexName, config, language, router, initialPart
11515
11519
  return [...partitions];
11516
11520
  },
11517
11521
  setPartitions(newPartitions) {
11522
+ writeLog.partitionsReplaced();
11518
11523
  partitions = [...newPartitions];
11519
11524
  rebuildDocPartitionMap();
11520
11525
  },
11526
+ openWriteLog() {
11527
+ return writeLog.open();
11528
+ },
11529
+ adoptRebuiltPartitions(rebuilt) {
11530
+ if (rebuilt.length !== partitions.length) {
11531
+ throw new NarsilError(
11532
+ ErrorCodes.PARTITION_CORRUPTED,
11533
+ `Index "${indexName}" holds ${partitions.length} partitions, but the rebuild produced ${rebuilt.length}`,
11534
+ { indexName, partitionCount: partitions.length, rebuiltCount: rebuilt.length }
11535
+ );
11536
+ }
11537
+ partitions = [...rebuilt];
11538
+ },
11521
11539
  addPartition() {
11522
11540
  const currentMaxPartitions = config.partitions?.maxPartitions;
11523
11541
  if (currentMaxPartitions !== void 0 && partitions.length >= currentMaxPartitions) {
@@ -11528,11 +11546,13 @@ function createPartitionManager(indexName, config, language, router, initialPart
11528
11546
  );
11529
11547
  }
11530
11548
  const partition = createPartitionIndex(partitions.length, trackPositions);
11549
+ writeLog.partitionsReplaced();
11531
11550
  partitions.push(partition);
11532
11551
  return partition;
11533
11552
  },
11534
11553
  removePartition(partitionId) {
11535
11554
  validatePartitionId(partitionId);
11555
+ writeLog.partitionsReplaced();
11536
11556
  partitions.splice(partitionId, 1);
11537
11557
  rebuildDocPartitionMap();
11538
11558
  },
@@ -11546,6 +11566,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
11546
11566
  if (newCount >= partitions.length) {
11547
11567
  return;
11548
11568
  }
11569
+ writeLog.partitionsReplaced();
11549
11570
  for (const [docId, pid] of docPartitionMap) {
11550
11571
  if (pid >= newCount) {
11551
11572
  docPartitionMap.delete(docId);
@@ -11568,6 +11589,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
11568
11589
  const insertOpts = resolveInsertOptions(options);
11569
11590
  partitions[pid].insert(docId, document2, config.schema, language, insertOpts);
11570
11591
  docPartitionMap.set(docId, pid);
11592
+ writeLog.documentWritten(docId);
11571
11593
  },
11572
11594
  remove(docId) {
11573
11595
  const pid = locateDocument(docId);
@@ -11576,11 +11598,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
11576
11598
  }
11577
11599
  partitions[pid].remove(docId, config.schema, language, resolveInsertOptions());
11578
11600
  docPartitionMap.delete(docId);
11579
- },
11580
- rebuildTextIndex(partitionId) {
11581
- validatePartitionId(partitionId);
11582
- partitions[partitionId].rebuildTextIndex(config.schema, language, resolveInsertOptions());
11583
- preparePartialWords(partitionId);
11601
+ writeLog.documentWritten(docId);
11584
11602
  },
11585
11603
  beginBatchRemove() {
11586
11604
  for (let i = 0; i < partitions.length; i++) {
@@ -11600,6 +11618,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
11600
11618
  const updateOpts = resolveInsertOptions(options);
11601
11619
  partitions[pid].update(docId, document2, config.schema, language, updateOpts);
11602
11620
  docPartitionMap.set(docId, pid);
11621
+ writeLog.documentWritten(docId);
11603
11622
  },
11604
11623
  get(docId, projection) {
11605
11624
  const pid = locateDocument(docId);
@@ -11650,6 +11669,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
11650
11669
  partition.endBatch();
11651
11670
  for (const docId of payload.docIds) {
11652
11671
  docPartitionMap.set(docId, partitionId);
11672
+ writeLog.documentWritten(docId);
11653
11673
  }
11654
11674
  },
11655
11675
  attachFrozenSegment(partitionId, segment) {
@@ -11664,6 +11684,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
11664
11684
  }
11665
11685
  asCompositePartition(partitionId).attachFrozenSegment(segment);
11666
11686
  registerSegmentDocuments(partitionId, segment);
11687
+ writeLog.segmentWritten(segment);
11667
11688
  },
11668
11689
  swapFrozenSegments(partitionId, dropSegmentIds, replacement) {
11669
11690
  validatePartitionId(partitionId);
@@ -11681,42 +11702,19 @@ function createPartitionManager(indexName, config, language, router, initialPart
11681
11702
  },
11682
11703
  deserializePartition(partitionId, data) {
11683
11704
  validatePartitionId(partitionId);
11705
+ writeLog.partitionsReplaced();
11684
11706
  partitions[partitionId].deserialize(data, config.schema);
11685
11707
  preparePartialWords(partitionId);
11686
11708
  rebuildDocPartitionMapForPartition(partitionId);
11687
11709
  },
11688
11710
  getAggregateStats() {
11689
- let totalDocuments = 0;
11690
- const docFrequencies = /* @__PURE__ */ Object.create(null);
11691
- const totalFieldLengths = /* @__PURE__ */ Object.create(null);
11692
- for (let i = 0; i < partitions.length; i++) {
11693
- const stats = partitions[i].stats;
11694
- totalDocuments += stats.totalDocuments;
11695
- for (const [term, freq] of Object.entries(stats.docFrequencies)) {
11696
- docFrequencies[term] = (docFrequencies[term] ?? 0) + freq;
11697
- }
11698
- for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
11699
- totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
11700
- }
11701
- }
11702
- return { totalDocuments, docFrequencies, totalFieldLengths };
11711
+ return aggregateStatsOf(partitions);
11703
11712
  },
11704
11713
  estimateMemoryBytes() {
11705
- let total = 0;
11706
- for (let i = 0; i < partitions.length; i++) {
11707
- total += partitions[i].estimateMemoryBytes();
11708
- }
11709
- for (const [, vecIndex] of vecIndexes) {
11710
- total += vecIndex.estimateMemoryBytes();
11711
- }
11712
- return total;
11714
+ return estimateBytesOf(partitions, vecIndexes);
11713
11715
  },
11714
11716
  getPartitionStats() {
11715
- return partitions.map((p, i) => ({
11716
- partitionId: i,
11717
- documentCount: p.count(),
11718
- estimatedMemoryBytes: p.estimateMemoryBytes()
11719
- }));
11717
+ return partitionStatsOf(partitions);
11720
11718
  },
11721
11719
  getVectorIndexes() {
11722
11720
  return vecIndexes;
@@ -12920,9 +12918,17 @@ function createOsqQuantizer(dimension, bits, metric, store) {
12920
12918
  if (view === null || ordinal === void 0 || !view.holdsOrdinal(ordinal)) return void 0;
12921
12919
  return view.recordAt(ordinal);
12922
12920
  },
12921
+ recordAtOrdinal(ordinal) {
12922
+ const view = shared2();
12923
+ if (view === null || !view.holdsOrdinal(ordinal)) return void 0;
12924
+ return view.recordAt(ordinal);
12925
+ },
12923
12926
  restoreRecord(docId, record) {
12924
12927
  requireShared().restoreRecord(ordinalOf(docId), record);
12925
12928
  },
12929
+ restoreRecordAtOrdinal(ordinal, record) {
12930
+ requireShared().restoreRecord(ordinal, record);
12931
+ },
12926
12932
  restoreCentroid(centroid) {
12927
12933
  requireShared().writeCentroid(centroid);
12928
12934
  },
@@ -12955,6 +12961,43 @@ function createOsqQuantizer(dimension, bits, metric, store) {
12955
12961
  };
12956
12962
  }
12957
12963
 
12964
+ // src/vector/similarity.ts
12965
+ function magnitude(v) {
12966
+ const result = simdMagnitude(v);
12967
+ if (result !== null) return result;
12968
+ let sum = 0;
12969
+ for (let i = 0; i < v.length; i++) {
12970
+ sum += v[i] * v[i];
12971
+ }
12972
+ return Math.sqrt(sum);
12973
+ }
12974
+ function cosineSimilarityWithMagnitudes(a, b, magA, magB) {
12975
+ if (magA === 0 || magB === 0) return 0;
12976
+ return dotProduct(a, b) / (magA * magB);
12977
+ }
12978
+ function dotProduct(a, b) {
12979
+ const result = simdDotProduct(a, b);
12980
+ if (result !== null) return result;
12981
+ let sum = 0;
12982
+ for (let i = 0; i < a.length; i++) {
12983
+ sum += a[i] * b[i];
12984
+ }
12985
+ return sum;
12986
+ }
12987
+ function euclideanDistance(a, b) {
12988
+ return Math.sqrt(squaredEuclideanDistance(a, b));
12989
+ }
12990
+ function squaredEuclideanDistance(a, b) {
12991
+ const result = simdSquaredEuclideanDistance(a, b);
12992
+ if (result !== null) return result;
12993
+ let sum = 0;
12994
+ for (let i = 0; i < a.length; i++) {
12995
+ const diff = a[i] - b[i];
12996
+ sum += diff * diff;
12997
+ }
12998
+ return sum;
12999
+ }
13000
+
12958
13001
  // src/vector/vector-store/doc-ids.ts
12959
13002
  var encoder2 = new TextEncoder();
12960
13003
  function appendDocId(handles, ordinal, docId) {
@@ -12974,6 +13017,78 @@ function resetDocIds(handles) {
12974
13017
  Atomics.store(handles.header, STORE_DOC_ID_BYTES, 0);
12975
13018
  }
12976
13019
 
13020
+ // src/vector/vector-store/ordinal-reads.ts
13021
+ function entryAt(store, ordinal) {
13022
+ return { vector: store.view.vectorAt(ordinal), magnitude: store.magnitudes[ordinal] };
13023
+ }
13024
+ function createOrdinalReads(openStore, readable) {
13025
+ function pairDistance(metric) {
13026
+ const open = openStore();
13027
+ if (open === null) return () => Number.POSITIVE_INFINITY;
13028
+ const inner = open.view.pairDistance(metric);
13029
+ return (ordA, ordB) => readable(ordA) && readable(ordB) ? inner(ordA, ordB) : Number.POSITIVE_INFINITY;
13030
+ }
13031
+ return {
13032
+ entryForOrdinal(ordinal) {
13033
+ const open = openStore();
13034
+ if (open === null || !readable(ordinal)) return void 0;
13035
+ return entryAt(open, ordinal);
13036
+ },
13037
+ distanceByOrdinal(ordA, ordB, metric) {
13038
+ const open = openStore();
13039
+ if (open === null || !readable(ordA) || !readable(ordB)) return Number.POSITIVE_INFINITY;
13040
+ return open.view.distanceByOrdinal(ordA, ordB, metric);
13041
+ },
13042
+ distanceFromArena(prepared, ordinal, metric) {
13043
+ const open = openStore();
13044
+ if (open === null || !readable(ordinal)) return Number.POSITIVE_INFINITY;
13045
+ return open.view.distanceFromArena(prepared, ordinal, metric);
13046
+ },
13047
+ queryDistance(prepared, metric) {
13048
+ const open = openStore();
13049
+ if (open === null) return () => Number.POSITIVE_INFINITY;
13050
+ const inner = open.view.queryDistance(prepared, metric);
13051
+ return (ordinal) => readable(ordinal) ? inner(ordinal) : Number.POSITIVE_INFINITY;
13052
+ },
13053
+ ordinalDistance(from, metric) {
13054
+ const pair = pairDistance(metric);
13055
+ return (ordinal) => pair(from, ordinal);
13056
+ },
13057
+ pairDistance
13058
+ };
13059
+ }
13060
+
13061
+ // src/vector/vector-store/listing.ts
13062
+ function* entriesOf(open, documents) {
13063
+ if (open === null) return;
13064
+ for (let ordinal = 0; ordinal < documents.length; ordinal++) {
13065
+ const docId = documents[ordinal];
13066
+ if (docId === void 0) continue;
13067
+ yield [docId, entryAt(open, ordinal)];
13068
+ }
13069
+ }
13070
+ function* docIdsOf(open, documents) {
13071
+ if (open === null) return;
13072
+ for (let ordinal = 0; ordinal < documents.length; ordinal++) {
13073
+ const docId = documents[ordinal];
13074
+ if (docId !== void 0) yield docId;
13075
+ }
13076
+ }
13077
+ function snapshotOf(open, documents, replacedOrdinals) {
13078
+ const slots = documents.length;
13079
+ const dimension = open === null ? 0 : open.handles.dimension;
13080
+ const vectors = new Float32Array(slots * dimension);
13081
+ const magnitudes = new Float64Array(slots);
13082
+ const docIds = new Array(slots);
13083
+ for (let ordinal = 0; ordinal < slots; ordinal++) {
13084
+ docIds[ordinal] = documents[ordinal] ?? null;
13085
+ if (open === null || documents[ordinal] === void 0 && !replacedOrdinals.has(ordinal)) continue;
13086
+ vectors.set(open.view.vectorAt(ordinal), ordinal * dimension);
13087
+ magnitudes[ordinal] = open.magnitudes[ordinal];
13088
+ }
13089
+ return { dimension, slots, vectors, magnitudes, docIds, replacedOrdinals: [...replacedOrdinals] };
13090
+ }
13091
+
12977
13092
  // src/vector/ordinal-filter.ts
12978
13093
  function createOrdinalFilter(slots) {
12979
13094
  return { bits: new Uint8Array(slots + 7 >>> 3), count: 0 };
@@ -13010,43 +13125,6 @@ function* ordinalFilterValues(filter) {
13010
13125
  }
13011
13126
  }
13012
13127
 
13013
- // src/vector/similarity.ts
13014
- function magnitude(v) {
13015
- const result = simdMagnitude(v);
13016
- if (result !== null) return result;
13017
- let sum = 0;
13018
- for (let i = 0; i < v.length; i++) {
13019
- sum += v[i] * v[i];
13020
- }
13021
- return Math.sqrt(sum);
13022
- }
13023
- function cosineSimilarityWithMagnitudes(a, b, magA, magB) {
13024
- if (magA === 0 || magB === 0) return 0;
13025
- return dotProduct(a, b) / (magA * magB);
13026
- }
13027
- function dotProduct(a, b) {
13028
- const result = simdDotProduct(a, b);
13029
- if (result !== null) return result;
13030
- let sum = 0;
13031
- for (let i = 0; i < a.length; i++) {
13032
- sum += a[i] * b[i];
13033
- }
13034
- return sum;
13035
- }
13036
- function euclideanDistance(a, b) {
13037
- return Math.sqrt(squaredEuclideanDistance(a, b));
13038
- }
13039
- function squaredEuclideanDistance(a, b) {
13040
- const result = simdSquaredEuclideanDistance(a, b);
13041
- if (result !== null) return result;
13042
- let sum = 0;
13043
- for (let i = 0; i < a.length; i++) {
13044
- const diff = a[i] - b[i];
13045
- sum += diff * diff;
13046
- }
13047
- return sum;
13048
- }
13049
-
13050
13128
  // src/vector/vector-store/vector-file.browser.ts
13051
13129
  function filesystemUnavailable() {
13052
13130
  throw new NarsilError(
@@ -13480,13 +13558,14 @@ function writeVector(store, ordinal, vector) {
13480
13558
  block.float32(base + vector.length).set(vector, base);
13481
13559
  store.magnitudes[ordinal] = magnitude(vector);
13482
13560
  }
13483
- function blockFullyCold(store, blockIndex, documents) {
13561
+ function blockFullyCold(store, blockIndex, documents, replacedOrdinals) {
13484
13562
  const capacity = store.handles.layout.capacity;
13485
13563
  const slots = Atomics.load(store.handles.header, STORE_SLOTS);
13486
13564
  const end = Math.min(slots, (blockIndex + 1) * capacity);
13487
13565
  if (end < (blockIndex + 1) * capacity) return false;
13488
13566
  for (let ordinal = blockIndex * capacity; ordinal < end; ordinal++) {
13489
- if (documents[ordinal] !== void 0 && store.diskFile[ordinal] === IN_MEMORY) return false;
13567
+ const held = documents[ordinal] !== void 0 || replacedOrdinals.has(ordinal);
13568
+ if (held && store.diskFile[ordinal] === IN_MEMORY) return false;
13490
13569
  }
13491
13570
  return true;
13492
13571
  }
@@ -13499,10 +13578,11 @@ function fileHoldsSameBytes(store, ordinal, location) {
13499
13578
  for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
13500
13579
  return true;
13501
13580
  }
13502
- function forgetUnreferencedFiles(store, documents) {
13581
+ function forgetUnreferencedFiles(store, documents, replacedOrdinals) {
13503
13582
  const referenced = /* @__PURE__ */ new Set();
13504
13583
  for (let ordinal = 0; ordinal < documents.length; ordinal++) {
13505
- if (documents[ordinal] !== void 0 && store.diskFile[ordinal] !== IN_MEMORY) {
13584
+ const held = documents[ordinal] !== void 0 || replacedOrdinals.has(ordinal);
13585
+ if (held && store.diskFile[ordinal] !== IN_MEMORY) {
13506
13586
  referenced.add(store.diskFile[ordinal]);
13507
13587
  }
13508
13588
  }
@@ -13543,15 +13623,20 @@ function createVectorStore(options) {
13543
13623
  const blockBytes2 = options?.blockBytes ?? BLOCK_MAX_BYTES;
13544
13624
  const docToOrd = /* @__PURE__ */ new Map();
13545
13625
  const ordToDoc = [];
13626
+ const replacedOrdinals = /* @__PURE__ */ new Set();
13546
13627
  let unknownPartitions = 0;
13547
13628
  let liveCount = 0;
13548
13629
  let open = options?.dimension === void 0 ? null : openHandles(options.dimension, codeBits, blockBytes2);
13549
13630
  function forgetDocuments() {
13550
13631
  docToOrd.clear();
13551
13632
  ordToDoc.length = 0;
13633
+ replacedOrdinals.clear();
13552
13634
  unknownPartitions = 0;
13553
13635
  liveCount = 0;
13554
13636
  }
13637
+ function readable(ordinal) {
13638
+ return ordToDoc[ordinal] !== void 0 || replacedOrdinals.has(ordinal);
13639
+ }
13555
13640
  function requireOpen() {
13556
13641
  if (open === null) {
13557
13642
  throw new NarsilError(ErrorCodes.VECTOR_DIMENSION_MISMATCH, "The vector store holds no vector yet");
@@ -13568,11 +13653,27 @@ function createVectorStore(options) {
13568
13653
  Atomics.store(store.present, ordinal, 0);
13569
13654
  if (store.partitions[ordinal] === UNKNOWN_PARTITION) unknownPartitions -= 1;
13570
13655
  }
13571
- function claimOrdinal(store, docId, cold, partitionId) {
13656
+ function keepReplacedOrdinal(store, ordinal) {
13657
+ ordToDoc[ordinal] = void 0;
13658
+ if (store.partitions[ordinal] === UNKNOWN_PARTITION) unknownPartitions -= 1;
13659
+ store.partitions[ordinal] = NO_PARTITION;
13660
+ replacedOrdinals.add(ordinal);
13661
+ }
13662
+ function restoreReplacedVector(store, ordinal, vector) {
13663
+ store.diskFile[ordinal] = IN_MEMORY;
13664
+ writeVector(store, ordinal, vector);
13665
+ Atomics.store(store.present, ordinal, 1);
13666
+ replacedOrdinals.add(ordinal);
13667
+ }
13668
+ function claimSlot(store, docId, cold) {
13572
13669
  const ordinal = Atomics.load(store.handles.header, STORE_SLOTS);
13573
13670
  ensureOrdinal(store, ordinal, cold);
13574
13671
  store.diskFile[ordinal] = IN_MEMORY;
13575
13672
  appendDocId(store.handles, ordinal, docId);
13673
+ return ordinal;
13674
+ }
13675
+ function claimOrdinal(store, docId, cold, partitionId) {
13676
+ const ordinal = claimSlot(store, docId, cold);
13576
13677
  recordPartition(store, ordinal, partitionId);
13577
13678
  ordToDoc[ordinal] = docId;
13578
13679
  docToOrd.set(docId, ordinal);
@@ -13582,25 +13683,30 @@ function createVectorStore(options) {
13582
13683
  Atomics.store(store.handles.header, STORE_SLOTS, ordinal + 1);
13583
13684
  Atomics.store(store.present, ordinal, 1);
13584
13685
  }
13686
+ function publishOnDisk(store, ordinal, vectorMagnitude, location) {
13687
+ store.magnitudes[ordinal] = vectorMagnitude;
13688
+ store.diskOffset[ordinal] = location.offset;
13689
+ store.diskFile[ordinal] = location.fileIndex;
13690
+ publishOrdinal(store, ordinal);
13691
+ }
13585
13692
  function appendOrdinal(store, docId, vector, partitionId) {
13586
13693
  const ordinal = claimOrdinal(store, docId, false, partitionId);
13587
13694
  writeVector(store, ordinal, vector);
13588
13695
  publishOrdinal(store, ordinal);
13589
13696
  return ordinal;
13590
13697
  }
13591
- function settleReplacement(store, ordinal, previous) {
13698
+ function settleReplacement(store, ordinal, previous, keepsReplacedVector) {
13592
13699
  if (previous !== void 0) {
13593
- retireOrdinal(store, previous);
13700
+ if (keepsReplacedVector) keepReplacedOrdinal(store, previous);
13701
+ else retireOrdinal(store, previous);
13594
13702
  return ordinal;
13595
13703
  }
13596
13704
  liveCount++;
13597
13705
  Atomics.add(store.handles.header, STORE_LIVE_COUNT, 1);
13598
13706
  return ordinal;
13599
13707
  }
13600
- function entryAt(store, ordinal) {
13601
- return { vector: store.view.vectorAt(ordinal), magnitude: store.magnitudes[ordinal] };
13602
- }
13603
13708
  return {
13709
+ ...createOrdinalReads(() => open, readable),
13604
13710
  get size() {
13605
13711
  return liveCount;
13606
13712
  },
@@ -13619,7 +13725,10 @@ function createVectorStore(options) {
13619
13725
  get view() {
13620
13726
  return requireOpen().view;
13621
13727
  },
13622
- insert(docId, vector, partitionId) {
13728
+ get replacedVectorCount() {
13729
+ return replacedOrdinals.size;
13730
+ },
13731
+ insert(docId, vector, partitionId, keepsReplacedVector = false) {
13623
13732
  if (open === null) open = openHandles(vector.length, codeBits, blockBytes2);
13624
13733
  const store = open;
13625
13734
  if (vector.length !== store.handles.dimension) {
@@ -13631,18 +13740,37 @@ function createVectorStore(options) {
13631
13740
  }
13632
13741
  const previous = docToOrd.get(docId);
13633
13742
  const ordinal = appendOrdinal(store, docId, vector, partitionId);
13634
- return settleReplacement(store, ordinal, previous);
13743
+ return settleReplacement(store, ordinal, previous, keepsReplacedVector);
13744
+ },
13745
+ releaseReplacedVectors() {
13746
+ const released = [...replacedOrdinals];
13747
+ replacedOrdinals.clear();
13748
+ if (open !== null) for (const ordinal of released) Atomics.store(open.present, ordinal, 0);
13749
+ return released;
13635
13750
  },
13636
13751
  insertCold(docId, vectorMagnitude, location, partitionId) {
13637
13752
  const store = requireOpen();
13638
13753
  const previous = docToOrd.get(docId);
13639
13754
  Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
13640
13755
  const ordinal = claimOrdinal(store, docId, true, partitionId);
13641
- store.magnitudes[ordinal] = vectorMagnitude;
13642
- store.diskOffset[ordinal] = location.offset;
13643
- store.diskFile[ordinal] = location.fileIndex;
13756
+ publishOnDisk(store, ordinal, vectorMagnitude, location);
13757
+ return settleReplacement(store, ordinal, previous, false);
13758
+ },
13759
+ insertReplaced(docId, vector, location) {
13760
+ if (open === null) open = openHandles(vector.length, codeBits, blockBytes2);
13761
+ const store = open;
13762
+ if (location !== void 0) Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
13763
+ const ordinal = claimSlot(store, docId, location !== void 0);
13764
+ store.partitions[ordinal] = NO_PARTITION;
13765
+ ordToDoc[ordinal] = void 0;
13766
+ replacedOrdinals.add(ordinal);
13767
+ if (location !== void 0) {
13768
+ publishOnDisk(store, ordinal, magnitude(vector), location);
13769
+ return ordinal;
13770
+ }
13771
+ writeVector(store, ordinal, vector);
13644
13772
  publishOrdinal(store, ordinal);
13645
- return settleReplacement(store, ordinal, previous);
13773
+ return ordinal;
13646
13774
  },
13647
13775
  addVectorFile(path) {
13648
13776
  const store = requireOpen();
@@ -13669,11 +13797,11 @@ function createVectorStore(options) {
13669
13797
  const store = requireOpen();
13670
13798
  let released = 0;
13671
13799
  for (let index = 0; index < store.handles.blocks.length; index++) {
13672
- if (store.handles.blocks[index] === null || !blockFullyCold(store, index, ordToDoc)) continue;
13800
+ if (store.handles.blocks[index] === null || !blockFullyCold(store, index, ordToDoc, replacedOrdinals)) continue;
13673
13801
  store.handles.blocks[index] = null;
13674
13802
  released += 1;
13675
13803
  }
13676
- if (released + forgetUnreferencedFiles(store, ordToDoc) > 0) relayout(store);
13804
+ if (released + forgetUnreferencedFiles(store, ordToDoc, replacedOrdinals) > 0) relayout(store);
13677
13805
  return released;
13678
13806
  },
13679
13807
  setPartition(docId, partitionId) {
@@ -13717,21 +13845,8 @@ function createVectorStore(options) {
13717
13845
  holdsOrdinal(ordinal) {
13718
13846
  return ordinal >= 0 && ordinal < ordToDoc.length && ordToDoc[ordinal] !== void 0;
13719
13847
  },
13720
- *entries() {
13721
- if (open === null) return;
13722
- for (let ordinal = 0; ordinal < ordToDoc.length; ordinal++) {
13723
- const docId = ordToDoc[ordinal];
13724
- if (docId === void 0) continue;
13725
- yield [docId, entryAt(open, ordinal)];
13726
- }
13727
- },
13728
- *docIds() {
13729
- if (open === null) return;
13730
- for (let ordinal = 0; ordinal < ordToDoc.length; ordinal++) {
13731
- const docId = ordToDoc[ordinal];
13732
- if (docId !== void 0) yield docId;
13733
- }
13734
- },
13848
+ entries: () => entriesOf(open, ordToDoc),
13849
+ docIds: () => docIdsOf(open, ordToDoc),
13735
13850
  clear() {
13736
13851
  forgetDocuments();
13737
13852
  if (open === null) return;
@@ -13763,59 +13878,16 @@ function createVectorStore(options) {
13763
13878
  docIdForOrdinal(ordinal) {
13764
13879
  return ordToDoc[ordinal];
13765
13880
  },
13766
- entryForOrdinal(ordinal) {
13767
- if (open === null || ordinal < 0 || ordinal >= ordToDoc.length || ordToDoc[ordinal] === void 0) {
13768
- return void 0;
13769
- }
13770
- return entryAt(open, ordinal);
13771
- },
13772
- distanceByOrdinal(ordA, ordB, metric) {
13773
- if (open === null || ordToDoc[ordA] === void 0 || ordToDoc[ordB] === void 0) {
13774
- return Number.POSITIVE_INFINITY;
13775
- }
13776
- return open.view.distanceByOrdinal(ordA, ordB, metric);
13777
- },
13778
13881
  prepareQueryArena(query) {
13779
13882
  return open === null ? null : open.view.prepareQueryArena(query);
13780
13883
  },
13781
- distanceFromArena(prepared, ordinal, metric) {
13782
- if (open === null || ordToDoc[ordinal] === void 0) return Number.POSITIVE_INFINITY;
13783
- return open.view.distanceFromArena(prepared, ordinal, metric);
13784
- },
13785
- queryDistance(prepared, metric) {
13786
- if (open === null) return () => Number.POSITIVE_INFINITY;
13787
- const inner = open.view.queryDistance(prepared, metric);
13788
- return (ordinal) => ordToDoc[ordinal] === void 0 ? Number.POSITIVE_INFINITY : inner(ordinal);
13789
- },
13790
- ordinalDistance(from, metric) {
13791
- const pair = this.pairDistance(metric);
13792
- return (ordinal) => pair(from, ordinal);
13793
- },
13794
- pairDistance(metric) {
13795
- if (open === null) return () => Number.POSITIVE_INFINITY;
13796
- const inner = open.view.pairDistance(metric);
13797
- return (ordA, ordB) => ordToDoc[ordA] === void 0 || ordToDoc[ordB] === void 0 ? Number.POSITIVE_INFINITY : inner(ordA, ordB);
13798
- },
13799
- exportSnapshot() {
13800
- const slots = ordToDoc.length;
13801
- const dimension = open === null ? 0 : open.handles.dimension;
13802
- const vectors = new Float32Array(slots * dimension);
13803
- const magnitudes = new Float64Array(slots);
13804
- const docIds = new Array(slots);
13805
- for (let ordinal = 0; ordinal < slots; ordinal++) {
13806
- const docId = ordToDoc[ordinal];
13807
- docIds[ordinal] = docId ?? null;
13808
- if (docId === void 0 || open === null) continue;
13809
- vectors.set(open.view.vectorAt(ordinal), ordinal * dimension);
13810
- magnitudes[ordinal] = open.magnitudes[ordinal];
13811
- }
13812
- return { dimension, slots, vectors, magnitudes, docIds };
13813
- },
13884
+ exportSnapshot: () => snapshotOf(open, ordToDoc, replacedOrdinals),
13814
13885
  restoreSnapshot(snapshot) {
13815
13886
  this.clear();
13816
13887
  if (snapshot.dimension === 0 || snapshot.slots === 0) return;
13817
13888
  if (open === null) open = openHandles(snapshot.dimension, codeBits, blockBytes2);
13818
13889
  const store = open;
13890
+ const replaced = new Set(snapshot.replacedOrdinals);
13819
13891
  for (let ordinal = 0; ordinal < snapshot.slots; ordinal++) {
13820
13892
  const docId = snapshot.docIds[ordinal];
13821
13893
  const vector = snapshot.vectors.subarray(ordinal * snapshot.dimension, (ordinal + 1) * snapshot.dimension);
@@ -13823,6 +13895,7 @@ function createVectorStore(options) {
13823
13895
  ensureOrdinal(store, ordinal, false);
13824
13896
  ordToDoc[ordinal] = void 0;
13825
13897
  store.partitions[ordinal] = NO_PARTITION;
13898
+ if (replaced.has(ordinal)) restoreReplacedVector(store, ordinal, vector);
13826
13899
  Atomics.store(store.handles.header, STORE_SLOTS, ordinal + 1);
13827
13900
  continue;
13828
13901
  }
@@ -14647,7 +14720,7 @@ function readNeighborsLocked(state, ord, layer) {
14647
14720
  if (nodeReadHeld(state.locks, ord, version)) return count;
14648
14721
  }
14649
14722
  }
14650
- function searchLayer(state, qVec, qMag, ef, layer, metric, skipTombstones, distFn, results) {
14723
+ function searchLayer(state, qVec, qMag, ef, layer, metric, excludeTombstones, distFn, results) {
14651
14724
  const getDistance = distFn ?? ((ord) => queryDistanceByOrd(state, qVec, qMag, ord, metric));
14652
14725
  const workspace = state.workspace;
14653
14726
  const frontier = workspace.frontier;
@@ -14663,10 +14736,10 @@ function searchLayer(state, qVec, qMag, ef, layer, metric, skipTombstones, distF
14663
14736
  if (state.visited[epOrd] === stamp) continue;
14664
14737
  state.visited[epOrd] = stamp;
14665
14738
  if (!nodeExists(state, epOrd)) continue;
14666
- if (skipTombstones && isTombstoned(state, epOrd)) continue;
14667
14739
  const dist = getDistance(epOrd);
14668
14740
  if (dist === Number.POSITIVE_INFINITY) continue;
14669
14741
  pushHeap(frontier, epOrd, dist);
14742
+ if (excludeTombstones && isTombstoned(state, epOrd)) continue;
14670
14743
  pushHeap(found, epOrd, dist);
14671
14744
  if (found.size > ef) {
14672
14745
  popHeap(found);
@@ -14674,19 +14747,19 @@ function searchLayer(state, qVec, qMag, ef, layer, metric, skipTombstones, distF
14674
14747
  }
14675
14748
  if (found.size > 0) furthestDist = found.distances[0];
14676
14749
  while (popHeap(frontier)) {
14677
- if (frontier.topDistance > furthestDist) break;
14750
+ if (frontier.topDistance > furthestDist && found.size >= ef) break;
14678
14751
  const count = readNeighborsLocked(state, frontier.topOrd, layer);
14679
14752
  for (let i = 0; i < count; i++) {
14680
14753
  const neighborOrd = scratch[i];
14681
14754
  ensureVisited(state, neighborOrd + 1);
14682
14755
  if (state.visited[neighborOrd] === stamp) continue;
14683
14756
  state.visited[neighborOrd] = stamp;
14684
- if (skipTombstones && isTombstoned(state, neighborOrd)) continue;
14685
14757
  if (!nodeExists(state, neighborOrd)) continue;
14686
14758
  const dist = getDistance(neighborOrd);
14687
14759
  if (dist === Number.POSITIVE_INFINITY) continue;
14688
14760
  if (dist < furthestDist || found.size < ef) {
14689
14761
  pushHeap(frontier, neighborOrd, dist);
14762
+ if (excludeTombstones && isTombstoned(state, neighborOrd)) continue;
14690
14763
  pushHeap(found, neighborOrd, dist);
14691
14764
  if (found.size > ef) {
14692
14765
  popHeap(found);
@@ -14962,13 +15035,6 @@ function markTombstone(state, ord) {
14962
15035
  if (Atomics.compareExchange(state.tombstones, ord, 0, 1) === 0) {
14963
15036
  Atomics.add(state.header, GRAPH_TOMBSTONE_COUNT, 1);
14964
15037
  }
14965
- if (entryPointOf(state) !== ord) return;
14966
- lockEntry(state.locks);
14967
- try {
14968
- if (entryPointOf(state) === ord) setEntryPoint(state, highestNode(state, false));
14969
- } finally {
14970
- unlockEntry(state.locks);
14971
- }
14972
15038
  }
14973
15039
  function compactTombstones(state) {
14974
15040
  if (Atomics.load(state.header, GRAPH_TOMBSTONE_COUNT) === 0) return;
@@ -15042,7 +15108,7 @@ function valuesOf(bytes) {
15042
15108
  return new Uint32Array(copy.buffer, 0, copy.byteLength / BYTES_PER_VALUE);
15043
15109
  }
15044
15110
  function savedTopLayer(state, ordinal) {
15045
- if (ordinal === NO_ORDINAL || isTombstoned(state, ordinal)) return NO_LAYER;
15111
+ if (ordinal === NO_ORDINAL) return NO_LAYER;
15046
15112
  return Math.min(nodeMaxLayer(state, ordinal), MAX_LAYER_CAP);
15047
15113
  }
15048
15114
  function neighbourValueCount(state, topLayers, start, end) {
@@ -15108,6 +15174,25 @@ function serializeNumberedGraphParts(state, numberOfOrdinal, ordinalOfNumber, pa
15108
15174
  }
15109
15175
  return parts;
15110
15176
  }
15177
+ function graphWithoutNodes(part, firstNumber, numbers) {
15178
+ const values = valuesOf(part.neighbours);
15179
+ const levels = part.levels.slice();
15180
+ const kept = new Uint32Array(values.length);
15181
+ let cursor = 0;
15182
+ let length = 0;
15183
+ for (let at = 0; at < part.levels.length; at++) {
15184
+ const start = cursor;
15185
+ for (let layer = 0; layer < part.levels[at]; layer++) cursor += values[cursor] + 1;
15186
+ if (numbers.has(firstNumber + at)) {
15187
+ levels[at] = 0;
15188
+ continue;
15189
+ }
15190
+ kept.set(values.subarray(start, cursor), length);
15191
+ length += cursor - start;
15192
+ }
15193
+ const entryPoint = part.entryPoint !== null && numbers.has(part.entryPoint) ? null : part.entryPoint;
15194
+ return { ...part, entryPoint, levels, neighbours: littleEndianBytesOf(kept.subarray(0, length)) };
15195
+ }
15111
15196
  function endsInsideAList() {
15112
15197
  throw new NarsilError(
15113
15198
  ErrorCodes.PERSISTENCE_LOAD_FAILED,
@@ -15374,7 +15459,7 @@ function traverse(state, query, k, searchMetric, options) {
15374
15459
  if (entryPoint === -1) return { candidates, depth: useQuantized ? depth : 0, arenaQuery, qMag, native: null };
15375
15460
  setSingleEntryPoint(workspace, entryPoint);
15376
15461
  for (let layer = topLayerOf(state); layer >= 1; layer--) {
15377
- searchLayer(state, query, qMag, 1, layer, searchMetric, true, distFn, candidates);
15462
+ searchLayer(state, query, qMag, 1, layer, searchMetric, false, distFn, candidates);
15378
15463
  if (candidates.size > 0) {
15379
15464
  setSingleEntryPoint(workspace, candidates.ords[0]);
15380
15465
  }
@@ -15682,6 +15767,8 @@ function createHNSWIndex(dimension, store, config, quantizer, handles) {
15682
15767
  if (ord !== void 0) markTombstone(state, ord);
15683
15768
  },
15684
15769
  markTombstoneOrdinal: (ordinal) => markTombstone(state, ordinal),
15770
+ holdsNode: (ordinal) => hasNode(state.adjacency, ordinal),
15771
+ holdsTombstone: (ordinal) => hasNode(state.adjacency, ordinal) && isTombstoned(state, ordinal),
15685
15772
  has: (docId) => {
15686
15773
  const ord = store.getOrdinal(docId);
15687
15774
  return ord !== void 0 && hasNode(state.adjacency, ord) && !isTombstoned(state, ord);
@@ -15751,6 +15838,9 @@ function emptyFieldBeforeRestore(state) {
15751
15838
  state.savedFiles = [];
15752
15839
  state.savedSignature = null;
15753
15840
  }
15841
+ function releaseReplacedVectors(state) {
15842
+ for (const ordinal of state.store.releaseReplacedVectors()) state.osq?.removeOrdinal(ordinal);
15843
+ }
15754
15844
  function adoptGraph(state, graph) {
15755
15845
  if (graph === null && state.hnsw !== null) {
15756
15846
  state.hnsw.clear();
@@ -16310,9 +16400,7 @@ function captureCloneCodes(state) {
16310
16400
  const records = new Uint8Array(slots * layout.slotStride);
16311
16401
  const present = new Uint8Array(slots);
16312
16402
  for (let ordinal = 0; ordinal < slots; ordinal++) {
16313
- const docId = state.store.docIdForOrdinal(ordinal);
16314
- if (docId === void 0) continue;
16315
- const record = quantizer.recordOf(docId);
16403
+ const record = quantizer.recordAtOrdinal(ordinal);
16316
16404
  if (record === void 0) continue;
16317
16405
  records.set(record, ordinal * layout.slotStride);
16318
16406
  present[ordinal] = 1;
@@ -16449,7 +16537,7 @@ async function dispatcherFor(state, graph) {
16449
16537
  workerCount: Math.max(1, host.workerCount)
16450
16538
  };
16451
16539
  }
16452
- if (liveSize(state) < WORKER_COPY_MIN_VECTORS) return null;
16540
+ if (liveSize(state) < WORKER_COPY_MIN_VECTORS || !sharedMemoryAvailable()) return null;
16453
16541
  const handle = await shareGraph(state, graph, graph === state.hnsw);
16454
16542
  const pool = state.workerCopyPool;
16455
16543
  if (handle === null || pool === null || state.sharedHandles.get(graph)?.mode !== "shared") return null;
@@ -16539,12 +16627,40 @@ async function insertLocally(state, graph, docIds, admit) {
16539
16627
  state.buffer.delete(docId);
16540
16628
  count += 1;
16541
16629
  if (count % BUILD_CHUNK_SIZE === 0) {
16630
+ noteWrite(state);
16542
16631
  await yieldToEventLoop2();
16543
16632
  }
16544
16633
  }
16634
+ if (count > 0) noteWrite(state);
16545
16635
  return !state.disposed;
16546
16636
  }
16547
16637
 
16638
+ // src/vector/vector-index/dead-bits.ts
16639
+ var NO_LIVE_VECTOR_AT_POSITION = -1;
16640
+ function deadBytesFor(count) {
16641
+ return Math.ceil(count / 8);
16642
+ }
16643
+ function isDead(dead, position) {
16644
+ return dead !== null && (dead[position >> 3] >> (position & 7) & 1) === 1;
16645
+ }
16646
+ function countDead(dead, count) {
16647
+ if (dead === null) return 0;
16648
+ let found = 0;
16649
+ for (let position = 0; position < count; position++) {
16650
+ if (isDead(dead, position)) found += 1;
16651
+ }
16652
+ return found;
16653
+ }
16654
+ function deadBitsWhere(count, isDeadAt) {
16655
+ let dead = null;
16656
+ for (let position = 0; position < count; position++) {
16657
+ if (!isDeadAt(position)) continue;
16658
+ dead ??= new Uint8Array(deadBytesFor(count));
16659
+ dead[position >> 3] |= 1 << (position & 7);
16660
+ }
16661
+ return dead;
16662
+ }
16663
+
16548
16664
  // src/vector/vector-index/disk.ts
16549
16665
  function readsFromDisk(state) {
16550
16666
  return state.storage === "disk";
@@ -16555,6 +16671,7 @@ function holdsVectorsOnDisk(state) {
16555
16671
  function* locationsOf(state, layout) {
16556
16672
  const stride = state.dimension * 4;
16557
16673
  for (let i = 0; i < layout.docIds.length; i++) {
16674
+ if (isDead(layout.dead, i)) continue;
16558
16675
  yield [layout.docIds[i], { path: layout.path, offset: layout.vectorsOffset + i * stride }];
16559
16676
  }
16560
16677
  }
@@ -16945,32 +17062,6 @@ function graphPartsOfEarlierPayload(payload) {
16945
17062
  };
16946
17063
  }
16947
17064
 
16948
- // src/vector/vector-index/dead-bits.ts
16949
- var NO_LIVE_VECTOR_AT_POSITION = -1;
16950
- function deadBytesFor(count) {
16951
- return Math.ceil(count / 8);
16952
- }
16953
- function isDead(dead, position) {
16954
- return dead !== null && (dead[position >> 3] >> (position & 7) & 1) === 1;
16955
- }
16956
- function countDead(dead, count) {
16957
- if (dead === null) return 0;
16958
- let found = 0;
16959
- for (let position = 0; position < count; position++) {
16960
- if (isDead(dead, position)) found += 1;
16961
- }
16962
- return found;
16963
- }
16964
- function deadBitsWhere(count, isDeadAt) {
16965
- let dead = null;
16966
- for (let position = 0; position < count; position++) {
16967
- if (!isDeadAt(position)) continue;
16968
- dead ??= new Uint8Array(deadBytesFor(count));
16969
- dead[position >> 3] |= 1 << (position & 7);
16970
- }
16971
- return dead;
16972
- }
16973
-
16974
17065
  // src/vector/vector-index/persistence.ts
16975
17066
  function recordFor(state, quantizer, docId) {
16976
17067
  const record = quantizer.recordOf(docId);
@@ -16992,7 +17083,7 @@ function sameCentroid(state, planned) {
16992
17083
  }
16993
17084
  return true;
16994
17085
  }
16995
- function partCodes(state, docIds, centroid) {
17086
+ function partCodes(state, docIds, centroid, tombstonedAt) {
16996
17087
  const quantizer = state.osq;
16997
17088
  const layout = state.store.handles.codeLayout;
16998
17089
  if (centroid === null || quantizer === null || layout === null) return null;
@@ -17005,7 +17096,8 @@ function partCodes(state, docIds, centroid) {
17005
17096
  }
17006
17097
  const records = new Uint8Array(docIds.length * layout.slotStride);
17007
17098
  for (let i = 0; i < docIds.length; i++) {
17008
- const record = recordFor(state, quantizer, docIds[i]);
17099
+ const tombstoned = tombstonedAt.get(i);
17100
+ const record = tombstoned === void 0 ? recordFor(state, quantizer, docIds[i]) : quantizer.recordAtOrdinal(tombstoned);
17009
17101
  if (record !== void 0) records.set(record, i * layout.slotStride);
17010
17102
  }
17011
17103
  return { bits: quantizer.bits, centroid, records };
@@ -17162,33 +17254,39 @@ function isLive(state, ordinal) {
17162
17254
  const docId = state.store.docIdForOrdinal(ordinal);
17163
17255
  return docId !== void 0 && !state.tombstones.has(docId);
17164
17256
  }
17165
- function liveCountOf(state, file) {
17166
- let live = 0;
17167
- for (let i = 0; i < file.ordinals.length; i++) if (isLive(state, file.ordinals[i])) live += 1;
17168
- return live;
17257
+ function isGraphTombstone(state, ordinal) {
17258
+ return ordinal !== NO_LIVE_VECTOR_AT_POSITION && state.hnsw !== null && state.hnsw.holdsTombstone(ordinal);
17259
+ }
17260
+ function isCarried(state, ordinal) {
17261
+ return isLive(state, ordinal) || isGraphTombstone(state, ordinal);
17262
+ }
17263
+ function carriedCountOf(state, file) {
17264
+ let carried = 0;
17265
+ for (let i = 0; i < file.ordinals.length; i++) if (isCarried(state, file.ordinals[i])) carried += 1;
17266
+ return carried;
17169
17267
  }
17170
17268
  function deadBitsOf(state, file) {
17171
17269
  return deadBitsWhere(file.ordinals.length, (position) => !isLive(state, file.ordinals[position]));
17172
17270
  }
17173
- function filesWorthKeeping(state, listed, calibration, liveCounts) {
17271
+ function filesWorthKeeping(state, listed, calibration, carriedCounts) {
17174
17272
  const kept = [];
17175
17273
  for (const file of listed) {
17176
- const live = liveCountOf(state, file);
17177
- const deadShare = file.ordinals.length === 0 ? 1 : 1 - live / file.ordinals.length;
17178
- if (file.calibration !== calibration || deadShare > VECTOR_FILE_REPLACED_ABOVE_DEAD_SHARE) continue;
17274
+ const carried = carriedCountOf(state, file);
17275
+ const deadWithoutNodeShare = file.ordinals.length === 0 ? 1 : 1 - carried / file.ordinals.length;
17276
+ if (file.calibration !== calibration || deadWithoutNodeShare > VECTOR_FILE_REPLACED_ABOVE_DEAD_SHARE) continue;
17179
17277
  kept.push(file);
17180
- liveCounts.push(live);
17278
+ carriedCounts.push(carried);
17181
17279
  }
17182
17280
  return kept;
17183
17281
  }
17184
- function absorbSmallerPartialFiles(kept, liveCounts, vectorsToWrite, capacity) {
17282
+ function absorbSmallerPartialFiles(kept, carriedCounts, vectorsToWrite, capacity) {
17185
17283
  let pending = vectorsToWrite;
17186
17284
  for (let index = kept.length - 1; index >= 0 && pending > 0; index--) {
17187
17285
  if (kept[index].ordinals.length >= capacity) continue;
17188
- if (liveCounts[index] > pending) return;
17189
- pending += liveCounts[index];
17286
+ if (carriedCounts[index] > pending) return;
17287
+ pending += carriedCounts[index];
17190
17288
  kept.splice(index, 1);
17191
- liveCounts.splice(index, 1);
17289
+ carriedCounts.splice(index, 1);
17192
17290
  }
17193
17291
  }
17194
17292
  function ordinalsOutsideOf(state, kept) {
@@ -17202,14 +17300,15 @@ function ordinalsOutsideOf(state, kept) {
17202
17300
  }
17203
17301
  const outside = [];
17204
17302
  for (let ordinal = 0; ordinal < slots; ordinal++) {
17205
- if (saved[ordinal] === 0 && isLive(state, ordinal)) outside.push(ordinal);
17303
+ if (saved[ordinal] === 0 && isCarried(state, ordinal)) outside.push(ordinal);
17206
17304
  }
17207
17305
  return outside;
17208
17306
  }
17209
- function countOutside(state, liveCounts) {
17307
+ function countOutside(state, carriedCounts) {
17210
17308
  let held = 0;
17211
- for (const live of liveCounts) held += live;
17212
- return Math.max(0, state.store.size - state.tombstones.size - held);
17309
+ for (const carried of carriedCounts) held += carried;
17310
+ const tombstonedNodes = state.hnsw?.tombstoneCount ?? 0;
17311
+ return Math.max(0, state.store.size - state.tombstones.size + tombstonedNodes - held);
17213
17312
  }
17214
17313
  function numberEveryVector(state, kept, written) {
17215
17314
  let total = written.length;
@@ -17220,7 +17319,7 @@ function numberEveryVector(state, kept, written) {
17220
17319
  for (const file of kept) {
17221
17320
  for (let i = 0; i < file.ordinals.length; i++, number++) {
17222
17321
  const ordinal = file.ordinals[i];
17223
- if (!isLive(state, ordinal)) continue;
17322
+ if (!isCarried(state, ordinal)) continue;
17224
17323
  numberOfOrdinal[ordinal] = number;
17225
17324
  ordinalOfNumber[number] = ordinal;
17226
17325
  }
@@ -17241,16 +17340,30 @@ function graphPartsOf(state, numbers) {
17241
17340
  );
17242
17341
  return parts.map((graph) => ({ v: VECTOR_GRAPH_PART_PAYLOAD_VERSION, fieldName: state.fieldName, graphs: [graph] }));
17243
17342
  }
17244
- function readFile(state, planned, docIds, centroid) {
17343
+ function graphPartsWithoutNodes(parts, numbers) {
17344
+ return parts.map((part, index) => ({
17345
+ ...part,
17346
+ graphs: part.graphs.map((graph) => graphWithoutNodes(graph, index * VECTOR_GRAPH_PART_VECTORS, numbers))
17347
+ }));
17348
+ }
17349
+ function plannedVectorOf(state, ordinal) {
17350
+ const docId = state.store.docIdForOrdinal(ordinal) ?? state.store.view.docIdAt(ordinal) ?? "";
17351
+ return { ordinal, docId, tombstoned: !isLive(state, ordinal) };
17352
+ }
17353
+ function readFile(state, planned, centroid) {
17245
17354
  const dimension = state.dimension;
17246
17355
  const vectors = new Float32Array(planned.length * dimension);
17247
17356
  const ordinals = new Int32Array(planned.length).fill(NO_LIVE_VECTOR_AT_POSITION);
17357
+ const tombstonedAt = /* @__PURE__ */ new Map();
17358
+ const docIds = planned.map((vector) => vector.docId);
17248
17359
  for (let i = 0; i < planned.length; i++) {
17249
- const held = state.store.holdsOrdinal(planned[i]) ? planned[i] : state.store.getOrdinal(docIds[i]);
17360
+ const { ordinal, docId, tombstoned } = planned[i];
17361
+ const held = tombstoned || state.store.holdsOrdinal(ordinal) ? ordinal : state.store.getOrdinal(docId);
17250
17362
  const entry = held === void 0 ? void 0 : state.store.entryForOrdinal(held);
17251
17363
  if (held === void 0 || entry === void 0) continue;
17252
17364
  vectors.set(entry.vector, i * dimension);
17253
17365
  ordinals[i] = held;
17366
+ if (tombstoned) tombstonedAt.set(i, held);
17254
17367
  }
17255
17368
  return {
17256
17369
  payload: {
@@ -17258,10 +17371,11 @@ function readFile(state, planned, docIds, centroid) {
17258
17371
  fieldName: state.fieldName,
17259
17372
  dimension,
17260
17373
  docIds,
17261
- codes: partCodes(state, docIds, centroid),
17374
+ codes: partCodes(state, docIds, centroid, tombstonedAt),
17262
17375
  vectors: vectorsToBytes(vectors)
17263
17376
  },
17264
- ordinals
17377
+ ordinals,
17378
+ dead: deadBitsWhere(planned.length, (i) => planned[i].tombstoned || ordinals[i] === NO_LIVE_VECTOR_AT_POSITION)
17265
17379
  };
17266
17380
  }
17267
17381
  function planCheckpoint(state, listedKeys, rewritesGraph = false) {
@@ -17269,14 +17383,20 @@ function planCheckpoint(state, listedKeys, rewritesGraph = false) {
17269
17383
  const calibration = signature.calibration;
17270
17384
  const trusted = registryMatches(state, listedKeys);
17271
17385
  const capacity = fileCapacityOf(state);
17272
- const liveCounts = [];
17273
- const keptFiles = filesWorthKeeping(state, trusted ? state.savedFiles : [], calibration, liveCounts);
17274
- absorbSmallerPartialFiles(keptFiles, liveCounts, countOutside(state, liveCounts), capacity);
17386
+ const carriedCounts = [];
17387
+ const keptFiles = filesWorthKeeping(state, trusted ? state.savedFiles : [], calibration, carriedCounts);
17388
+ absorbSmallerPartialFiles(keptFiles, carriedCounts, countOutside(state, carriedCounts), capacity);
17275
17389
  const written = ordinalsOutsideOf(state, keptFiles);
17276
- const writtenDocIds = written.map((ordinal) => state.store.docIdForOrdinal(ordinal) ?? "");
17390
+ const planned = written.map((ordinal) => plannedVectorOf(state, ordinal));
17277
17391
  const centroid = codedCentroid(state);
17278
17392
  const listedBefore = trusted && listedKeys !== null;
17279
17393
  const unchanged = !rewritesGraph && listedBefore && sameSignature(state.savedSignature, signature) && written.length === 0;
17394
+ let firstWrittenNumber = 0;
17395
+ for (const file of keptFiles) firstWrittenNumber += file.ordinals.length;
17396
+ const graphParts = unchanged ? null : graphPartsOf(state, numberEveryVector(state, keptFiles, written));
17397
+ const vanished = /* @__PURE__ */ new Set();
17398
+ let vanishedWhenStripped = 0;
17399
+ let graphPartsToWrite = graphParts;
17280
17400
  return {
17281
17401
  unchanged,
17282
17402
  keptFiles,
@@ -17287,12 +17407,22 @@ function planCheckpoint(state, listedKeys, rewritesGraph = false) {
17287
17407
  dead: deadBitsOf(state, file)
17288
17408
  })),
17289
17409
  newFiles: Math.ceil(written.length / capacity),
17290
- graphParts: unchanged ? null : graphPartsOf(state, numberEveryVector(state, keptFiles, written)),
17410
+ get graphParts() {
17411
+ if (graphParts !== null && vanished.size !== vanishedWhenStripped) {
17412
+ graphPartsToWrite = graphPartsWithoutNodes(graphParts, vanished);
17413
+ vanishedWhenStripped = vanished.size;
17414
+ }
17415
+ return graphPartsToWrite;
17416
+ },
17291
17417
  signature,
17292
17418
  readNewFile(index) {
17293
17419
  const start = index * capacity;
17294
17420
  const end = start + capacity;
17295
- return readFile(state, written.slice(start, end), writtenDocIds.slice(start, end), centroid);
17421
+ const file = readFile(state, planned.slice(start, end), centroid);
17422
+ for (let i = 0; i < file.ordinals.length; i++) {
17423
+ if (file.ordinals[i] === NO_LIVE_VECTOR_AT_POSITION) vanished.add(firstWrittenNumber + start + i);
17424
+ }
17425
+ return file;
17296
17426
  }
17297
17427
  };
17298
17428
  }
@@ -17325,9 +17455,10 @@ function insertNodesOfLaterGraph(state, graph, later, ordinals) {
17325
17455
  if (docId !== void 0 && !graph.has(docId)) graph.insertOrdinal(ordinal);
17326
17456
  }
17327
17457
  }
17328
- function restoreGraph(state, payload, ordinalOfNumber) {
17458
+ function restoreGraph(state, payload, ordinalOfNumber, tombstoned) {
17329
17459
  const first = payload?.graphs[0]?.parts[0];
17330
17460
  if (payload === null || first === void 0) {
17461
+ releaseReplacedVectors(state);
17331
17462
  bufferEveryVectorOutsideTheGraph(state);
17332
17463
  return;
17333
17464
  }
@@ -17338,11 +17469,13 @@ function restoreGraph(state, payload, ordinalOfNumber) {
17338
17469
  state.osq ?? void 0
17339
17470
  );
17340
17471
  graph.deserializeNumbered(payload.graphs[0], ordinalOfNumber);
17472
+ for (const ordinal of tombstoned) graph.markTombstoneOrdinal(ordinal);
17341
17473
  adoptGraph(state, graph);
17342
17474
  for (const later of payload.graphs.slice(1)) insertNodesOfLaterGraph(state, graph, later, ordinalOfNumber);
17343
17475
  bufferEveryVectorOutsideTheGraph(state);
17344
17476
  if (state.buffer.size <= graph.size) return;
17345
17477
  adoptGraph(state, null);
17478
+ releaseReplacedVectors(state);
17346
17479
  state.buffer.clear();
17347
17480
  bufferEveryVectorOutsideTheGraph(state);
17348
17481
  }
@@ -17351,6 +17484,9 @@ function beginCheckpointRestore(state, shape) {
17351
17484
  const cold = readsFromDisk(state) && (shape.holdsGraph || shape.liveVectors >= state.promotionThreshold);
17352
17485
  const restoresCodes = state.osq !== null && osqBitsOf(state.quantizationMode) !== null && shape.holdsGraph;
17353
17486
  const files = [];
17487
+ const graphNodes = shape.graph?.graphs[0] ?? null;
17488
+ const tombstoned = /* @__PURE__ */ new Set();
17489
+ let nextNumber = 0;
17354
17490
  let sharedCodes = null;
17355
17491
  let codesAgree = true;
17356
17492
  function restoreRecords(payload, ordinals) {
@@ -17372,9 +17508,14 @@ function beginCheckpointRestore(state, shape) {
17372
17508
  const stride = layout.slotStride;
17373
17509
  for (let i = 0; i < ordinals.length; i++) {
17374
17510
  if (ordinals[i] === NO_LIVE_VECTOR_AT_POSITION) continue;
17375
- quantizer.restoreRecord(payload.docIds[i], codes.records.subarray(i * stride, (i + 1) * stride));
17511
+ const record = codes.records.subarray(i * stride, (i + 1) * stride);
17512
+ if (tombstoned.has(ordinals[i])) quantizer.restoreRecordAtOrdinal(ordinals[i], record);
17513
+ else quantizer.restoreRecord(payload.docIds[i], record);
17376
17514
  }
17377
17515
  }
17516
+ function graphHoldsNodeFor(number) {
17517
+ return graphNodes !== null && topLayerInParts(graphNodes, number) >= 0;
17518
+ }
17378
17519
  function addFile(file) {
17379
17520
  const { payload, location } = file;
17380
17521
  if (payload.dimension !== state.dimension) {
@@ -17389,13 +17530,21 @@ function beginCheckpointRestore(state, shape) {
17389
17530
  const vectors = bytesToVectors(payload.vectors);
17390
17531
  const fileIndex = onDisk && cold ? state.store.addVectorFile(location.path) : null;
17391
17532
  const ordinals = new Int32Array(payload.docIds.length).fill(NO_LIVE_VECTOR_AT_POSITION);
17533
+ const firstNumber = nextNumber;
17534
+ nextNumber += payload.docIds.length;
17392
17535
  for (let i = 0; i < payload.docIds.length; i++) {
17393
- if (isDead(file.dead, i)) continue;
17536
+ const dead = isDead(file.dead, i);
17537
+ if (dead && !graphHoldsNodeFor(firstNumber + i)) continue;
17394
17538
  const docId = payload.docIds[i];
17395
- const replaced = state.store.getOrdinal(docId);
17396
- if (replaced !== void 0) state.osq?.removeOrdinal(replaced);
17397
17539
  const vector = vectors.subarray(i * dimension, (i + 1) * dimension);
17398
17540
  const offset = onDisk ? location.vectorsOffset + i * dimension * 4 : 0;
17541
+ if (dead) {
17542
+ ordinals[i] = state.store.insertReplaced(docId, vector, fileIndex === null ? void 0 : { fileIndex, offset });
17543
+ tombstoned.add(ordinals[i]);
17544
+ continue;
17545
+ }
17546
+ const replaced = state.store.getOrdinal(docId);
17547
+ if (replaced !== void 0) state.osq?.removeOrdinal(replaced);
17399
17548
  if (fileIndex !== null) {
17400
17549
  ordinals[i] = state.store.insertCold(docId, magnitude(vector), { fileIndex, offset });
17401
17550
  continue;
@@ -17406,21 +17555,27 @@ function beginCheckpointRestore(state, shape) {
17406
17555
  restoreRecords(payload, ordinals);
17407
17556
  files.push({ id: file.id, key: file.key, ordinals, coded: payload.codes !== null });
17408
17557
  }
17409
- function finish(graph) {
17558
+ function finish() {
17410
17559
  const codesRestored = restoresCodes && codesAgree && sharedCodes !== null;
17411
- if (restoresCodes && !codesRestored) recalibrateFromStore(state);
17560
+ if (restoresCodes && !codesRestored) {
17561
+ recalibrateFromStore(state);
17562
+ for (const ordinal of tombstoned) state.osq?.writeCodes(ordinal);
17563
+ }
17412
17564
  let total = 0;
17413
17565
  for (const file of files) total += file.ordinals.length;
17414
17566
  const ordinalOfNumber = new Int32Array(total);
17415
17567
  let number = 0;
17416
17568
  for (const file of files) {
17417
17569
  for (let i = 0; i < file.ordinals.length; i++) {
17418
- if (!state.store.holdsOrdinal(file.ordinals[i])) file.ordinals[i] = NO_LIVE_VECTOR_AT_POSITION;
17570
+ const ordinal = file.ordinals[i];
17571
+ if (!state.store.holdsOrdinal(ordinal) && !tombstoned.has(ordinal)) {
17572
+ file.ordinals[i] = NO_LIVE_VECTOR_AT_POSITION;
17573
+ }
17419
17574
  }
17420
17575
  ordinalOfNumber.set(file.ordinals, number);
17421
17576
  number += file.ordinals.length;
17422
17577
  }
17423
- restoreGraph(state, graph, ordinalOfNumber);
17578
+ restoreGraph(state, shape.graph, ordinalOfNumber, tombstoned);
17424
17579
  const current = signatureOf(state).calibration;
17425
17580
  const codedUnder = codesRestored && current !== null ? current : CALIBRATION_NEVER_CURRENT;
17426
17581
  const saved = files.map((file) => ({
@@ -17437,11 +17592,12 @@ function beginCheckpointRestore(state, shape) {
17437
17592
 
17438
17593
  // src/vector/vector-index/maintenance.ts
17439
17594
  function compact(state) {
17440
- if (state.tombstones.size === 0) return;
17595
+ if (state.tombstones.size === 0 && state.store.replacedVectorCount === 0) return;
17441
17596
  if (state.hnsw) {
17442
17597
  state.compactedNodeCount += state.hnsw.tombstoneCount;
17443
17598
  state.hnsw.compactTombstones();
17444
17599
  }
17600
+ state.freshGraph?.compactTombstones();
17445
17601
  for (const docId of state.tombstones) {
17446
17602
  state.store.remove(docId);
17447
17603
  state.buffer.delete(docId);
@@ -17449,6 +17605,7 @@ function compact(state) {
17449
17605
  state.osq.remove(docId);
17450
17606
  }
17451
17607
  }
17608
+ releaseReplacedVectors(state);
17452
17609
  state.tombstones.clear();
17453
17610
  }
17454
17611
  function recalibrateWhileNoThreadSearches(state, graph) {
@@ -17480,6 +17637,7 @@ async function foldIntoGraph(state) {
17480
17637
  const graph = state.hnsw;
17481
17638
  if (graph === null || rebuildNeeded) {
17482
17639
  if (state.osq?.isCalibrated()) recalibrateWhileNoThreadSearches(state, graph);
17640
+ else calibrateQuantizer(state);
17483
17641
  await buildGraphFromStore(state);
17484
17642
  } else {
17485
17643
  await insertMissing(state, graph);
@@ -17526,8 +17684,9 @@ function maintenanceStatus(state) {
17526
17684
  const storeSize = state.store.size;
17527
17685
  const graphCount = state.hnsw ? 1 : 0;
17528
17686
  const graphNodes = state.hnsw === null ? storeSize : state.hnsw.size + state.hnsw.tombstoneCount;
17529
- const tombstoneRatio = graphNodes > 0 ? state.tombstones.size / graphNodes : 0;
17530
- const estimatedCompactMs = Math.round(state.tombstones.size * ESTIMATED_MS_PER_TOMBSTONE * state.dimensionScale);
17687
+ const tombstoned = state.tombstones.size + state.store.replacedVectorCount;
17688
+ const tombstoneRatio = graphNodes > 0 ? tombstoned / graphNodes : 0;
17689
+ const estimatedCompactMs = Math.round(tombstoned * ESTIMATED_MS_PER_TOMBSTONE * state.dimensionScale);
17531
17690
  const optimizeVectorCount = state.hnsw === null || graphNeedsRebuild(state) ? storeSize : state.buffer.size;
17532
17691
  const estimatedOptimizeMs = Math.round(optimizeVectorCount * ESTIMATED_MS_PER_VECTOR_REBUILD * state.dimensionScale);
17533
17692
  return {
@@ -17787,19 +17946,19 @@ function searchWithFilter(state, query, k, options, filter) {
17787
17946
  return scannedOutcome(scan2);
17788
17947
  }
17789
17948
  }
17790
- const graphOptions = { filter, efSearch, oversample };
17791
- if (state.buffer.size === 0) {
17792
- const hnswResults2 = state.hnsw.search(query, k, metric, minSimilarity, graphOptions);
17793
- return fetchedOutcome(
17794
- state,
17795
- hnswResults2.map((r) => ({ docId: r.docId, score: r.score })),
17796
- minSimilarity,
17797
- filter
17798
- );
17799
- }
17800
- const hnswResults = state.hnsw.search(query, k, metric, minSimilarity, graphOptions).map((r) => ({ docId: r.docId, score: r.score }));
17801
- const buffered = bruteForceSearch(state, query, k, metric, minSimilarity, bufferCandidates(state, filter));
17802
- return fetchedOutcome(state, mergeResults(hnswResults, buffered.results, k), minSimilarity, filter);
17949
+ const hnswResults = state.hnsw.search(query, k, metric, minSimilarity, { filter, efSearch, oversample }).map((r) => ({ docId: r.docId, score: r.score }));
17950
+ return fetchedOutcome(
17951
+ state,
17952
+ withBufferedVectors(state, query, k, options, filter, hnswResults),
17953
+ minSimilarity,
17954
+ filter
17955
+ );
17956
+ }
17957
+ function withBufferedVectors(state, query, k, options, filter, graphResults) {
17958
+ if (state.buffer.size === 0) return graphResults;
17959
+ const candidates = bufferCandidates(state, filter);
17960
+ const buffered = bruteForceSearch(state, query, k, options.metric, options.minSimilarity, candidates);
17961
+ return mergeResults(graphResults, buffered.results, k);
17803
17962
  }
17804
17963
 
17805
17964
  // src/vector/vector-index/index.ts
@@ -17900,11 +18059,12 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
17900
18059
  }
17901
18060
  noteWrite(state);
17902
18061
  state.tombstones.delete(docId);
17903
- state.store.insert(docId, vector, partitionId);
18062
+ const graphHoldsPrevious = previous !== void 0 && ((state.hnsw?.holdsNode(previous) ?? false) || (state.freshGraph?.holdsNode(previous) ?? false));
18063
+ state.store.insert(docId, vector, partitionId, graphHoldsPrevious);
17904
18064
  if (previous !== void 0) {
17905
18065
  state.hnsw?.markTombstoneOrdinal(previous);
17906
18066
  state.freshGraph?.markTombstoneOrdinal(previous);
17907
- state.osq?.removeOrdinal(previous);
18067
+ if (!graphHoldsPrevious) state.osq?.removeOrdinal(previous);
17908
18068
  }
17909
18069
  state.buffer.add(docId);
17910
18070
  state.pendingLocations.delete(docId);
@@ -17974,10 +18134,13 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
17974
18134
  efSearch: options.efSearch,
17975
18135
  oversample: options.oversample
17976
18136
  });
17977
- if (viaWorkerCopy !== null) return fetchedOutcome(state, viaWorkerCopy, options.minSimilarity, filter);
17978
18137
  if (confined && state.revision !== filterRevision) {
17979
18138
  filter = filterForOptions(state, options);
17980
18139
  }
18140
+ if (viaWorkerCopy !== null) {
18141
+ const results = withBufferedVectors(state, query, k, options, filter, viaWorkerCopy);
18142
+ return fetchedOutcome(state, results, options.minSimilarity, filter);
18143
+ }
17981
18144
  return searchWithFilter(state, query, k, options, filter);
17982
18145
  }
17983
18146
  return {
@@ -19380,10 +19543,10 @@ function indexByFieldPath(entries) {
19380
19543
  for (const entry of entries) byFieldPath.set(entry.fieldPath, entry);
19381
19544
  return byFieldPath;
19382
19545
  }
19383
- function rowsByFieldPath(inputs, entriesOf, valuesOf2) {
19546
+ function rowsByFieldPath(inputs, entriesOf2, valuesOf2) {
19384
19547
  const byFieldPath = /* @__PURE__ */ new Map();
19385
19548
  for (const input of inputs) {
19386
- for (const entry of entriesOf(input)) {
19549
+ for (const entry of entriesOf2(input)) {
19387
19550
  let rows = byFieldPath.get(entry.fieldPath);
19388
19551
  if (rows === void 0) {
19389
19552
  rows = [];
@@ -20839,6 +21002,74 @@ function createDirectExecutor(options) {
20839
21002
  };
20840
21003
  }
20841
21004
 
21005
+ // src/partitioning/text-rebuild.ts
21006
+ function interrupted(pass) {
21007
+ return pass.log.partitionsReplaced || !pass.hooks.isCurrent();
21008
+ }
21009
+ async function buildPartition(pass, source, partitionId) {
21010
+ const { manager, options } = pass;
21011
+ const target = createPartitionIndex(partitionId, manager.config.trackPositions ?? true);
21012
+ const docIds = [...source.docIds()];
21013
+ target.beginBatch();
21014
+ for (let start = 0; start < docIds.length; start += TEXT_REBUILD_CHUNK_DOCUMENTS) {
21015
+ const end = Math.min(start + TEXT_REBUILD_CHUNK_DOCUMENTS, docIds.length);
21016
+ for (let i = start; i < end; i++) {
21017
+ const document2 = manager.getRef(docIds[i]);
21018
+ if (document2 !== void 0) target.insert(docIds[i], document2, manager.schema, manager.language, options);
21019
+ }
21020
+ await yieldToEventLoop();
21021
+ if (interrupted(pass)) return null;
21022
+ }
21023
+ target.endBatch();
21024
+ if (manager.partialWordAnalysis !== null) target.preparePartialWords(manager.partialWordAnalysis);
21025
+ return target;
21026
+ }
21027
+ function applyWrittenDocuments(pass, built) {
21028
+ const { manager, options } = pass;
21029
+ for (const docId of pass.log.takeWrittenDocIds()) {
21030
+ for (const partition of built) {
21031
+ if (!partition.has(docId)) continue;
21032
+ partition.remove(docId, manager.schema, manager.language, options);
21033
+ break;
21034
+ }
21035
+ const partitionId = manager.partitionIdOf(docId);
21036
+ const document2 = manager.getRef(docId);
21037
+ if (partitionId === void 0 || document2 === void 0) continue;
21038
+ built[partitionId].insert(docId, document2, manager.schema, manager.language, options);
21039
+ }
21040
+ }
21041
+ async function rebuildOnce(pass) {
21042
+ const partitionCount = pass.manager.partitionCount;
21043
+ const built = [];
21044
+ for (let partitionId = 0; partitionId < partitionCount; partitionId++) {
21045
+ const partition = await buildPartition(pass, pass.manager.getPartition(partitionId), partitionId);
21046
+ if (partition === null) return false;
21047
+ built.push(partition);
21048
+ pass.hooks.onPartitionBuilt(built.length);
21049
+ }
21050
+ while (pass.log.pendingDocuments > TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS) {
21051
+ applyWrittenDocuments(pass, built);
21052
+ await yieldToEventLoop();
21053
+ if (interrupted(pass)) return false;
21054
+ }
21055
+ applyWrittenDocuments(pass, built);
21056
+ pass.manager.adoptRebuiltPartitions(built);
21057
+ pass.hooks.onSwitch();
21058
+ return true;
21059
+ }
21060
+ async function rebuildTextIndexes(manager, hooks) {
21061
+ const options = resolvePartitionInsertOptions(manager.config, manager.analysis, { validate: false, skipClone: true });
21062
+ while (hooks.isCurrent()) {
21063
+ const log = manager.openWriteLog();
21064
+ try {
21065
+ if (await rebuildOnce({ manager, log, hooks, options })) return true;
21066
+ } finally {
21067
+ log.close();
21068
+ }
21069
+ }
21070
+ return false;
21071
+ }
21072
+
20842
21073
  // src/engine/events.ts
20843
21074
  function eventHandlersOf(events) {
20844
21075
  const handlers = /* @__PURE__ */ new Map();
@@ -20861,11 +21092,6 @@ function emitEngineEvent(eventHandlers, event, payload) {
20861
21092
  }
20862
21093
 
20863
21094
  // src/engine/analysis-rebuild.ts
20864
- function yieldToEventLoop4() {
20865
- return new Promise((resolve) => {
20866
- setTimeout(resolve, 0);
20867
- });
20868
- }
20869
21095
  function toError(value) {
20870
21096
  return value instanceof Error ? value : new Error(String(value));
20871
21097
  }
@@ -20876,9 +21102,13 @@ function wireAnalysisRebuild(wiring) {
20876
21102
  resyncIndex: wiring.resyncIndex,
20877
21103
  async persistAnalysisRevision(indexName) {
20878
21104
  const durability = wiring.durabilityManager;
20879
- if (durability === null) return;
20880
- await durability.persistMetadata(indexName);
21105
+ if (durability === null) {
21106
+ wiring.recordCurrentAnalysis(indexName);
21107
+ return;
21108
+ }
20881
21109
  await (durability.checkpointFromMemory?.(indexName) ?? durability.checkpoint(indexName));
21110
+ wiring.recordCurrentAnalysis(indexName);
21111
+ await durability.persistMetadata(indexName);
20882
21112
  },
20883
21113
  emit(payload) {
20884
21114
  emitEngineEvent(wiring.eventHandlers, "analysisRebuild", payload);
@@ -20889,9 +21119,10 @@ function createAnalysisRebuildCoordinator(config, deps) {
20889
21119
  const stale = /* @__PURE__ */ new Map();
20890
21120
  const rebuildMode = config?.rebuild ?? "auto";
20891
21121
  let queue = Promise.resolve();
21122
+ let active = null;
20892
21123
  async function rebuildNow(indexName, entry) {
20893
21124
  const manager = deps.getManager(indexName);
20894
- if (manager === void 0) {
21125
+ if (manager === void 0 || entry.abandoned) {
20895
21126
  entry.run = null;
20896
21127
  return;
20897
21128
  }
@@ -20899,12 +21130,20 @@ function createAnalysisRebuildCoordinator(config, deps) {
20899
21130
  deps.emit({ indexName, status: "started", partitionsRebuilt: 0, partitionCount });
20900
21131
  const wasPromoted = deps.desyncIndex(indexName);
20901
21132
  try {
20902
- for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
20903
- manager.rebuildTextIndex(partitionId);
20904
- entry.partitionsRebuilt = partitionId + 1;
20905
- await yieldToEventLoop4();
21133
+ const rebuilt = await rebuildTextIndexes(manager, {
21134
+ isCurrent: () => deps.getManager(indexName) === manager && stale.get(indexName) === entry && !entry.abandoned,
21135
+ onPartitionBuilt(partitionsBuilt) {
21136
+ entry.partitionsRebuilt = partitionsBuilt;
21137
+ },
21138
+ onSwitch() {
21139
+ stale.delete(indexName);
21140
+ }
21141
+ });
21142
+ if (!rebuilt) {
21143
+ entry.run = null;
21144
+ if (deps.getManager(indexName) === manager) await deps.resyncIndex(indexName, wasPromoted);
21145
+ return;
20906
21146
  }
20907
- stale.delete(indexName);
20908
21147
  await deps.persistAnalysisRevision(indexName);
20909
21148
  await deps.resyncIndex(indexName, wasPromoted);
20910
21149
  deps.emit({
@@ -20915,7 +21154,6 @@ function createAnalysisRebuildCoordinator(config, deps) {
20915
21154
  });
20916
21155
  } catch (err) {
20917
21156
  const error = toError(err);
20918
- stale.set(indexName, entry);
20919
21157
  entry.run = null;
20920
21158
  await deps.resyncIndex(indexName, wasPromoted).catch(() => void 0);
20921
21159
  deps.emit({
@@ -20928,11 +21166,20 @@ function createAnalysisRebuildCoordinator(config, deps) {
20928
21166
  throw error;
20929
21167
  }
20930
21168
  }
21169
+ function runTracked(indexName, entry) {
21170
+ const done = rebuildNow(indexName, entry);
21171
+ const tracked = { indexName, entry, done };
21172
+ active = tracked;
21173
+ return done.finally(() => {
21174
+ if (active === tracked) active = null;
21175
+ });
21176
+ }
20931
21177
  function enqueue(indexName, entry) {
20932
21178
  if (entry.run !== null) {
20933
21179
  return entry.run;
20934
21180
  }
20935
- const run = queue.then(() => rebuildNow(indexName, entry));
21181
+ entry.abandoned = false;
21182
+ const run = queue.then(() => runTracked(indexName, entry));
20936
21183
  entry.run = run;
20937
21184
  queue = run.then(
20938
21185
  () => void 0,
@@ -20967,7 +21214,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
20967
21214
  if (existing !== void 0 && existing.run !== null) {
20968
21215
  return;
20969
21216
  }
20970
- stale.set(index.indexName, { index, partitionsRebuilt: 0, run: null });
21217
+ stale.set(index.indexName, { index, partitionsRebuilt: 0, run: null, abandoned: false });
20971
21218
  },
20972
21219
  clearStale(indexName) {
20973
21220
  stale.delete(indexName);
@@ -20978,6 +21225,14 @@ function createAnalysisRebuildCoordinator(config, deps) {
20978
21225
  isRunning(indexName) {
20979
21226
  return stale.get(indexName)?.run !== null && stale.get(indexName)?.run !== void 0;
20980
21227
  },
21228
+ async abandon(indexName) {
21229
+ const entry = stale.get(indexName);
21230
+ if (entry !== void 0 && entry.run !== null) entry.abandoned = true;
21231
+ const running = active;
21232
+ if (running === null || running.indexName !== indexName) return;
21233
+ running.entry.abandoned = true;
21234
+ await running.done.catch(() => void 0);
21235
+ },
20981
21236
  review(indexName) {
20982
21237
  const entry = stale.get(indexName);
20983
21238
  return entry === void 0 ? Promise.resolve() : review(indexName, entry);
@@ -21644,7 +21899,7 @@ function buildEntry(fields) {
21644
21899
  }
21645
21900
 
21646
21901
  // src/serialization/chunk-checksum.ts
21647
- function yieldToEventLoop5() {
21902
+ function yieldToEventLoop4() {
21648
21903
  if (typeof setImmediate === "function") {
21649
21904
  return new Promise((resolve) => setImmediate(resolve));
21650
21905
  }
@@ -21656,7 +21911,7 @@ async function advanceChecksum(start, payload) {
21656
21911
  const end = Math.min(offset + CHECKSUM_YIELD_CHUNK_BYTES, payload.length);
21657
21912
  state = crc32Update(state, payload.subarray(offset, end));
21658
21913
  if (end < payload.length) {
21659
- await yieldToEventLoop5();
21914
+ await yieldToEventLoop4();
21660
21915
  }
21661
21916
  }
21662
21917
  return state;
@@ -23782,7 +24037,8 @@ async function readVectorFile(store, file, fieldPath) {
23782
24037
  async function loadVectorField(store, ref, vectorIndex) {
23783
24038
  const restore = vectorIndex.restoreCheckpoint({
23784
24039
  liveVectors: liveVectorsOf(ref),
23785
- holdsGraph: ref.graphParts.length > 0
24040
+ holdsGraph: ref.graphParts.length > 0,
24041
+ graph: await readGraphParts(store, ref)
23786
24042
  });
23787
24043
  for (const file of ref.files) {
23788
24044
  const { payload, payloadLength } = await readVectorFile(store, file, ref.fieldPath);
@@ -23797,7 +24053,7 @@ async function loadVectorField(store, ref, vectorIndex) {
23797
24053
  }
23798
24054
  });
23799
24055
  }
23800
- restore.finish(await readGraphParts(store, ref));
24056
+ restore.finish();
23801
24057
  }
23802
24058
  var EMPTY_BIN = new Uint8Array(0);
23803
24059
  var EMPTY_BIN_BYTES = 2;
@@ -23861,7 +24117,7 @@ async function writeField(input, fieldPath, plan, outcome) {
23861
24117
  const files = plan.kept.map((kept) => keptFileRef(input, prior, kept));
23862
24118
  const written = [];
23863
24119
  for (let index = 0; index < plan.newFiles; index++) {
23864
- const { payload, ordinals } = plan.readNewFile(index);
24120
+ const { payload, ordinals, dead } = plan.readNewFile(index);
23865
24121
  const id = nextFileId;
23866
24122
  nextFileId += 1;
23867
24123
  const key = vectorFileKey(input.indexName, fieldPath, id);
@@ -23871,13 +24127,13 @@ async function writeField(input, fieldPath, plan, outcome) {
23871
24127
  `vector file "${key}"`
23872
24128
  );
23873
24129
  await input.store.write(key, packed.chunks);
23874
- const dead = deadBitsWhere(ordinals.length, (position) => ordinals[position] === NO_LIVE_VECTOR_AT_POSITION);
23875
24130
  files.push({ id, key, count: payload.docIds.length, dead, bytes: packed.byteLength, checksum: packed.checksum });
23876
24131
  written.push({ id, key, ordinals });
23877
24132
  outcome.layouts.push({
23878
24133
  fieldPath,
23879
24134
  key,
23880
24135
  docIds: payload.docIds,
24136
+ dead,
23881
24137
  vectorsOffset: packed.byteLength - payload.vectors.byteLength
23882
24138
  });
23883
24139
  }
@@ -25664,7 +25920,7 @@ function removeWhatTheSegmentReplaces(manager, segment, contents) {
25664
25920
  manager.endBatchRemove();
25665
25921
  }
25666
25922
  }
25667
- function yieldToEventLoop6() {
25923
+ function yieldToEventLoop5() {
25668
25924
  return new Promise((resolve) => setTimeout(resolve, 0));
25669
25925
  }
25670
25926
  async function loadPartitionSegments(store, partition, manager, options) {
@@ -25675,7 +25931,7 @@ async function loadPartitionSegments(store, partition, manager, options) {
25675
25931
  const segment = createSharedFrozenSegment(shareSegmentArrays(contents.arrays, manager.partialWordAnalysis));
25676
25932
  removeWhatTheSegmentReplaces(manager, segment, contents);
25677
25933
  if (contents.arrays.documentCount > 0) manager.attachFrozenSegment(partition.partitionId, segment);
25678
- await yieldToEventLoop6();
25934
+ await yieldToEventLoop5();
25679
25935
  }
25680
25936
  }
25681
25937
 
@@ -25748,7 +26004,12 @@ async function adoptVectorLayouts(store, vectorIndexes, layouts) {
25748
26004
  const vecIndex = vectorIndexes.get(layout.fieldPath);
25749
26005
  if (vecIndex === void 0 || vecIndex.storage !== "disk") continue;
25750
26006
  const path = await pathOf.call(store, layout.key);
25751
- await vecIndex.adoptDiskLayout({ path, docIds: layout.docIds, vectorsOffset: layout.vectorsOffset });
26007
+ await vecIndex.adoptDiskLayout({
26008
+ path,
26009
+ docIds: layout.docIds,
26010
+ dead: layout.dead,
26011
+ vectorsOffset: layout.vectorsOffset
26012
+ });
25752
26013
  }
25753
26014
  }
25754
26015
  async function garbageThatNoFieldStillReads(store, vectorIndexes, keys) {
@@ -26703,7 +26964,11 @@ async function readGraph(store, ref) {
26703
26964
  async function loadVectorField2(store, ref, vectorIndex, fileRefs) {
26704
26965
  let liveVectors = 0;
26705
26966
  for (const file of ref.files) liveVectors += file.count - countDead(file.dead, file.count);
26706
- const restore = vectorIndex.restoreCheckpoint({ liveVectors, holdsGraph: ref.graphKey !== null });
26967
+ const restore = vectorIndex.restoreCheckpoint({
26968
+ liveVectors,
26969
+ holdsGraph: ref.graphKey !== null,
26970
+ graph: await readGraph(store, ref)
26971
+ });
26707
26972
  for (const file of ref.files) {
26708
26973
  const bytes = await store.read(file.key);
26709
26974
  if (bytes === null) {
@@ -26736,7 +27001,7 @@ async function loadVectorField2(store, ref, vectorIndex, fileRefs) {
26736
27001
  }
26737
27002
  });
26738
27003
  }
26739
- restore.finish(await readGraph(store, ref));
27004
+ restore.finish();
26740
27005
  }
26741
27006
  async function loadEarlierLayout(store, indexName, manifest, deps) {
26742
27007
  let highestPartitionId = manifest.partitions.length - 1;
@@ -28094,7 +28359,7 @@ function metadataConfigOf(entry) {
28094
28359
  b: entry.config.bm25?.b ?? 0.75,
28095
28360
  ...embedding !== void 0 ? { embedding } : {},
28096
28361
  surfaceForms: entry.config.surfaceForms !== false,
28097
- analysisRevision: entry.language.revision,
28362
+ ...entry.analysisRevision !== null ? { analysisRevision: entry.analysisRevision } : {},
28098
28363
  ...typeof entry.config.tokenizer === "string" ? { tokenizer: entry.config.tokenizer } : {},
28099
28364
  ...typeof entry.config.stopWords === "string" ? { stopWords: entry.config.stopWords } : {},
28100
28365
  ...entry.config.stopWords instanceof Set ? { stopWordList: [...entry.config.stopWords].sort(compareCodePoints) } : {},
@@ -28335,7 +28600,8 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
28335
28600
  indexUuid: metadata.indexUuid ?? null,
28336
28601
  heldPartitions: metadata.heldPartitions ?? null,
28337
28602
  documentCount: metadata.documentCount ?? 0,
28338
- partitionCount: metadata.partitionCount
28603
+ partitionCount: metadata.partitionCount,
28604
+ analysisRevision: metadata.analysisRevision ?? null
28339
28605
  });
28340
28606
  if (loadData) await deps.indexState().registerOpen(metadata.indexName);
28341
28607
  else deps.indexState().registerClosed(metadata.indexName);
@@ -28458,8 +28724,14 @@ function createIndexStateCoordinator(config, callbacks) {
28458
28724
  return entry;
28459
28725
  }
28460
28726
  }
28727
+ function transitionPending(indexName, entry) {
28728
+ if (entries.get(indexName) !== entry) return true;
28729
+ if (entry.state === "dropping" && entry.dropPromise !== null) return true;
28730
+ return entry.state === "closing" && entry.closePromise !== null;
28731
+ }
28461
28732
  async function reopen(indexName, explicit) {
28462
- const entry = await awaitPendingTransitions(indexName);
28733
+ let entry = await awaitPendingTransitions(indexName);
28734
+ while (transitionPending(indexName, entry)) entry = await awaitPendingTransitions(indexName);
28463
28735
  if (entry.state === "open") return;
28464
28736
  if (explicit) {
28465
28737
  entry.reopenFailures = 0;
@@ -30878,7 +31150,13 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
30878
31150
  idleSweep: null
30879
31151
  };
30880
31152
  startIdleSweep(state);
31153
+ function settlingCopyWork(indexName) {
31154
+ return state.copyTransitions.get(indexName)?.done ?? state.compactionsInFlight.get(indexName);
31155
+ }
30881
31156
  async function resyncIndex(indexName, wasScaledOut) {
31157
+ for (let pending = settlingCopyWork(indexName); pending !== void 0; pending = settlingCopyWork(indexName)) {
31158
+ await pending;
31159
+ }
30882
31160
  state.desyncedIndexes.delete(indexName);
30883
31161
  if (!wasScaledOut) return;
30884
31162
  await scaleOutIndex(state, indexName, copyThresholdReason(state, indexName));
@@ -31230,6 +31508,10 @@ function createEngineCore(config, hooks, capacityScope = "index") {
31230
31508
  getManager: (indexName) => executor.getManager(indexName),
31231
31509
  desyncIndex: (indexName) => orchestrator.desyncIndex(indexName),
31232
31510
  resyncIndex: (indexName, wasPromoted) => orchestrator.resyncIndex(indexName, wasPromoted),
31511
+ recordCurrentAnalysis: (indexName) => {
31512
+ const entry = indexRegistry.get(indexName);
31513
+ if (entry !== void 0 && !analysisRebuild.isStale(indexName)) entry.analysisRevision = entry.language.revision;
31514
+ },
31233
31515
  durabilityManager: durability?.manager ?? null
31234
31516
  });
31235
31517
  const indexState = wireIndexState({
@@ -31403,11 +31685,16 @@ function refuseCode(entry) {
31403
31685
  }
31404
31686
  function indexConfigurationOf(entry) {
31405
31687
  refuseCode(entry);
31406
- return configurationPayloadOf(entry.config, entry.language, entry.embeddingAdapterName);
31688
+ return configurationPayloadOf(entry.config, entry.language, entry.embeddingAdapterName, entry.analysisRevision);
31407
31689
  }
31408
31690
  function requireConfigurationWithinSizeLimit(config, language) {
31409
31691
  const adapter = config.embedding?.adapter;
31410
- const payload = configurationPayloadOf(config, language, typeof adapter === "string" ? adapter : null);
31692
+ const payload = configurationPayloadOf(
31693
+ config,
31694
+ language,
31695
+ typeof adapter === "string" ? adapter : null,
31696
+ language.revision
31697
+ );
31411
31698
  if (messagePackUpperBoundBytes(payload) <= MAX_INDEX_CONFIGURATION_BYTES) return;
31412
31699
  const bytes = encode(payload).byteLength;
31413
31700
  if (bytes <= MAX_INDEX_CONFIGURATION_BYTES) return;
@@ -31417,12 +31704,12 @@ function requireConfigurationWithinSizeLimit(config, language) {
31417
31704
  { bytes, maximum: MAX_INDEX_CONFIGURATION_BYTES }
31418
31705
  );
31419
31706
  }
31420
- function configurationPayloadOf(config, language, adapterName) {
31707
+ function configurationPayloadOf(config, language, adapterName, analysisRevision) {
31421
31708
  const bm25 = config.bm25;
31422
31709
  return {
31423
31710
  schema: config.schema,
31424
31711
  language: language.name,
31425
- analysisRevision: language.revision,
31712
+ ...analysisRevision !== null ? { analysisRevision } : {},
31426
31713
  ...typeof config.tokenizer === "string" ? { tokenizer: config.tokenizer } : {},
31427
31714
  ...typeof config.stopWords === "string" ? { stopWords: config.stopWords } : {},
31428
31715
  ...config.stopWords instanceof Set ? { stopWordList: [...config.stopWords].sort(compareCodePoints) } : {},
@@ -31581,7 +31868,8 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
31581
31868
  indexUuid: indexUuid ?? null,
31582
31869
  heldPartitions: null,
31583
31870
  documentCount: 0,
31584
- partitionCount: indexConfig.partitions?.maxPartitions ?? 1
31871
+ partitionCount: indexConfig.partitions?.maxPartitions ?? 1,
31872
+ analysisRevision: language.revision
31585
31873
  });
31586
31874
  await core.indexState.registerOpen(name);
31587
31875
  if (core.durability) {
@@ -31593,6 +31881,7 @@ async function dropEngineIndex(core, name) {
31593
31881
  core.guardShutdown();
31594
31882
  const entry = core.requireIndex(name);
31595
31883
  await core.indexState.drop(name, async () => {
31884
+ await core.analysisRebuild.abandon(name);
31596
31885
  await core.orchestrator.closeIndex(name);
31597
31886
  if (core.durability) {
31598
31887
  try {
@@ -34397,7 +34686,7 @@ function partitionVectorPlan(vectorIndex, docIds) {
34397
34686
  codes: null,
34398
34687
  vectors: vectorsToBytes(packed)
34399
34688
  };
34400
- files.push({ payload, ordinals: new Int32Array(slice.length) });
34689
+ files.push({ payload, ordinals: new Int32Array(slice.length), dead: null });
34401
34690
  }
34402
34691
  return {
34403
34692
  unchanged: false,
@@ -34893,7 +35182,8 @@ async function createRestoredIndex(indexName, prepared, partitionCount, deps) {
34893
35182
  indexUuid: null,
34894
35183
  heldPartitions: null,
34895
35184
  documentCount: 0,
34896
- partitionCount
35185
+ partitionCount,
35186
+ analysisRevision: prepared.storedRevision
34897
35187
  });
34898
35188
  await deps.indexState.registerOpen(indexName);
34899
35189
  const release = await deps.indexState.acquire(indexName, false);
@@ -36790,6 +37080,15 @@ async function enterRead(core, indexName) {
36790
37080
  core.guardShutdown();
36791
37081
  if (core.rebalancingIndexes.has(indexName)) await yieldToEventLoop();
36792
37082
  }
37083
+ async function readUnderOneAnalysis(core, indexName, read) {
37084
+ let analysisStale = core.analysisRebuild.isStale(indexName);
37085
+ let result = await read();
37086
+ while (core.analysisRebuild.isStale(indexName) !== analysisStale) {
37087
+ analysisStale = core.analysisRebuild.isStale(indexName);
37088
+ result = await read();
37089
+ }
37090
+ return { result, analysisStale };
37091
+ }
36793
37092
  function scopedParams(params, options) {
36794
37093
  {
36795
37094
  return params;
@@ -36817,7 +37116,7 @@ async function runEngineQuery(core, indexName, params, options) {
36817
37116
  entry.embeddingAdapterName
36818
37117
  );
36819
37118
  const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
36820
- const result = await executeQuery(resolvedParams, {
37119
+ const context = {
36821
37120
  manager,
36822
37121
  language: entry.language,
36823
37122
  config: entry.config,
@@ -36827,7 +37126,15 @@ async function runEngineQuery(core, indexName, params, options) {
36827
37126
  partitionIds: options?.partitionIds,
36828
37127
  cursorBinding: queryBindingOf(params),
36829
37128
  patternWorkCap: core.patternWorkCap
36830
- });
37129
+ };
37130
+ const { result, analysisStale } = await readUnderOneAnalysis(
37131
+ core,
37132
+ indexName,
37133
+ () => executeQuery(resolvedParams, context)
37134
+ );
37135
+ if (analysisStale) {
37136
+ result.analysisStale = true;
37137
+ }
36831
37138
  if (core.pluginRegistry.hasHooks("afterSearch")) {
36832
37139
  const observed = { ...result, hits: [...result.hits] };
36833
37140
  try {
@@ -36836,9 +37143,6 @@ async function runEngineQuery(core, indexName, params, options) {
36836
37143
  console.warn("afterSearch plugin hook error:", err);
36837
37144
  }
36838
37145
  }
36839
- if (core.analysisRebuild.isStale(indexName)) {
36840
- result.analysisStale = true;
36841
- }
36842
37146
  return result;
36843
37147
  } finally {
36844
37148
  release();
@@ -36859,7 +37163,7 @@ async function runEnginePreflight(core, indexName, params, options) {
36859
37163
  entry.embeddingAdapterName
36860
37164
  );
36861
37165
  const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
36862
- const result = await executePreflight(resolvedParams, {
37166
+ const context = {
36863
37167
  manager,
36864
37168
  language: entry.language,
36865
37169
  config: entry.config,
@@ -36869,8 +37173,13 @@ async function runEnginePreflight(core, indexName, params, options) {
36869
37173
  partitionIds: options?.partitionIds,
36870
37174
  cursorBinding: queryBindingOf(params),
36871
37175
  patternWorkCap: core.patternWorkCap
36872
- });
36873
- if (core.analysisRebuild.isStale(indexName)) {
37176
+ };
37177
+ const { result, analysisStale } = await readUnderOneAnalysis(
37178
+ core,
37179
+ indexName,
37180
+ () => executePreflight(resolvedParams, context)
37181
+ );
37182
+ if (analysisStale) {
36874
37183
  result.analysisStale = true;
36875
37184
  }
36876
37185
  return result;