@delali/narsil 0.3.1-next.82 → 0.3.1-next.84

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,7 +1,7 @@
1
1
  import { createGrowingPartialWords, requirePatternSearchFor, requirePartialWordSearchFor, buildPartialWordTableFrom, patternIndexOf, encodePartialSpellingsOf, countPartialSpelling, joinPatternWorkMeter, countsWithoutScores, fanOutMatchCount, fanOutQuery, sanitizeGlobalStats, expandPartialWords, computeFacets, cloneProjected, projectionKeepsField, setNestedValue, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, PATTERN_HOST, validatedFilterOf, everyValueFacetConfig, mergeFacets, kWayMerge, polygonContainment, polygonBoxes, radiusBoxes, locationCode, undeclaredFilterField, coveringRanges } from './chunk-XYDP7CCJ.mjs';
2
2
  import { getLanguage, detectRuntime } from './chunk-OLXM65YH.mjs';
3
3
  import { MAX_DOC_ID_LENGTH } from './chunk-ZCKPRCYI.mjs';
4
- import { SCRATCH_SLOTS_PER_THREAD_POOL, osqRecordBytes, openSharedVectorField, osqBitsOf, createVectorStore, DISK_STORAGE_BLOCK_BYTES, createOsqQuantizer, magnitude, createHNSWIndex, nativeStoreScratchBytes, createOrdinalFilter, isTombstoned, addToOrdinalFilter, ordinalFilterValues, nativeScoresOf, toScore, removeFromOrdinalFilter, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, ordinalFilterHas, STORE_CALIBRATION_GENERATION, entryForOrd, toDistance, sharedMemoryAvailable, requireWholeNeighbourLists, topLayerInParts, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, releaseLocksHeldBy, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS } from './chunk-IN5CH4P2.mjs';
4
+ import { SCRATCH_SLOTS_PER_THREAD_POOL, osqRecordBytes, openSharedVectorField, osqBitsOf, createVectorStore, DISK_STORAGE_BLOCK_BYTES, createOsqQuantizer, magnitude, graphWithoutNodes, createHNSWIndex, nativeStoreScratchBytes, createOrdinalFilter, isTombstoned, addToOrdinalFilter, ordinalFilterValues, nativeScoresOf, toScore, removeFromOrdinalFilter, ordinalFilterHas, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, topLayerInParts, STORE_CALIBRATION_GENERATION, entryForOrd, toDistance, sharedMemoryAvailable, requireWholeNeighbourLists, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, releaseLocksHeldBy, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS } from './chunk-RZXNWQSM.mjs';
5
5
  import { noteFallback } from './chunk-B3UOQJD3.mjs';
6
6
  import { GrowableUint32, bitsetHas, createBitSet, bitsetSet, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, bitsetFromSet, bitsetIsEmpty } from './chunk-SREPGC6O.mjs';
7
7
  import { GEO_UNIT_METRES } from './chunk-WQBCBKY3.mjs';
@@ -3394,65 +3394,6 @@ function mergeSegmentState(target, source) {
3394
3394
  target.scoreBuffer = null;
3395
3395
  }
3396
3396
 
3397
- // src/core/partition/rebuild.ts
3398
- var EMPTY_STATS = {
3399
- totalDocuments: 0,
3400
- totalFieldLengths: {},
3401
- averageFieldLengths: {},
3402
- docFrequencies: {}
3403
- };
3404
- function textFieldsOf(fields) {
3405
- return fields.filter((field) => isWordIndexedBase(field.base));
3406
- }
3407
- function rebuildTextIndex(state, schema, language, options) {
3408
- const textFields = textFieldsOf(schemaFieldsOf(schema));
3409
- state.invertedIdx.clear();
3410
- state.surfaceRegistry.clear();
3411
- state.stats.deserialize(EMPTY_STATS);
3412
- state.invertedIdx.beginBatch();
3413
- try {
3414
- for (const [docId, stored] of state.docStore.all()) {
3415
- const internalId = state.docStore.getInternalId(docId);
3416
- if (internalId === void 0) continue;
3417
- const fields = stored.fields;
3418
- const fieldLengths = {};
3419
- const tokensByField = {};
3420
- for (const { path: fieldPath, base } of textFields) {
3421
- const value = getNestedValue(fields, fieldPath);
3422
- if (value === void 0 || value === null) continue;
3423
- if (base === "string") {
3424
- indexStringField(
3425
- state,
3426
- internalId,
3427
- fieldPath,
3428
- value,
3429
- language,
3430
- options,
3431
- fieldLengths,
3432
- tokensByField
3433
- );
3434
- } else {
3435
- indexStringArrayField(
3436
- state,
3437
- internalId,
3438
- fieldPath,
3439
- value,
3440
- language,
3441
- options,
3442
- fieldLengths,
3443
- tokensByField
3444
- );
3445
- }
3446
- }
3447
- state.docStore.storeRef(docId, stored.fields, fieldLengths);
3448
- state.stats.addDocument(fieldLengths, tokensByField);
3449
- }
3450
- } finally {
3451
- state.invertedIdx.endBatch();
3452
- state.partialWords.forgetSpellings();
3453
- }
3454
- }
3455
-
3456
3397
  // src/core/partition/block-bounds.ts
3457
3398
  var ABSENT_FIELD_LENGTH_BOUND = 0;
3458
3399
  var boundsByList = /* @__PURE__ */ new WeakMap();
@@ -4873,9 +4814,6 @@ function createPartitionIndex(partitionId, trackPositions = true) {
4873
4814
  if (internalId !== void 0) recordSortValues(state, internalId, document);
4874
4815
  }
4875
4816
  },
4876
- rebuildTextIndex(schema, language, options) {
4877
- rebuildTextIndex(state, schema, language, options);
4878
- },
4879
4817
  get(docId, projection) {
4880
4818
  const stored = state.docStore.get(docId);
4881
4819
  if (!stored) return void 0;
@@ -5442,18 +5380,6 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
5442
5380
  if (frozen.length === 0) return live.encodeSegment();
5443
5381
  return composedScratch().encodeSegment();
5444
5382
  },
5445
- rebuildTextIndex(schema, language, options) {
5446
- invalidateDocFrequencies();
5447
- if (frozen.length > 0) {
5448
- for (const segment of frozen) {
5449
- const payload = encodeSegmentState(segment);
5450
- if (payload.documentCount === 0) continue;
5451
- live.mergeSegmentPayload(payload, survivorDocumentsOf(segment, payload));
5452
- }
5453
- frozen.length = 0;
5454
- }
5455
- live.rebuildTextIndex(schema, language, options);
5456
- },
5457
5383
  serialize(indexName, totalPartitions, language, schema) {
5458
5384
  if (frozen.length === 0) return live.serialize(indexName, totalPartitions, language, schema);
5459
5385
  return composedScratch().serialize(indexName, totalPartitions, language, schema);
@@ -5678,6 +5604,81 @@ function createCapacityChecks(source) {
5678
5604
  };
5679
5605
  }
5680
5606
 
5607
+ // src/partitioning/manager/stats.ts
5608
+ function aggregateStatsOf(partitions) {
5609
+ let totalDocuments = 0;
5610
+ const docFrequencies = /* @__PURE__ */ Object.create(null);
5611
+ const totalFieldLengths = /* @__PURE__ */ Object.create(null);
5612
+ for (let i = 0; i < partitions.length; i++) {
5613
+ const stats = partitions[i].stats;
5614
+ totalDocuments += stats.totalDocuments;
5615
+ for (const [term, freq] of Object.entries(stats.docFrequencies)) {
5616
+ docFrequencies[term] = (docFrequencies[term] ?? 0) + freq;
5617
+ }
5618
+ for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
5619
+ totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
5620
+ }
5621
+ }
5622
+ return { totalDocuments, docFrequencies, totalFieldLengths };
5623
+ }
5624
+ function estimateBytesOf(partitions, vecIndexes) {
5625
+ let total = 0;
5626
+ for (let i = 0; i < partitions.length; i++) {
5627
+ total += partitions[i].estimateMemoryBytes();
5628
+ }
5629
+ for (const [, vecIndex] of vecIndexes) {
5630
+ total += vecIndex.estimateMemoryBytes();
5631
+ }
5632
+ return total;
5633
+ }
5634
+ function partitionStatsOf(partitions) {
5635
+ return partitions.map((p, i) => ({
5636
+ partitionId: i,
5637
+ documentCount: p.count(),
5638
+ estimatedMemoryBytes: p.estimateMemoryBytes()
5639
+ }));
5640
+ }
5641
+
5642
+ // src/partitioning/manager/write-log.ts
5643
+ function createWriteLogRecorder() {
5644
+ let current = null;
5645
+ return {
5646
+ open() {
5647
+ const log = { docIds: /* @__PURE__ */ new Set(), replaced: false };
5648
+ current = log;
5649
+ return {
5650
+ get partitionsReplaced() {
5651
+ return log.replaced;
5652
+ },
5653
+ get pendingDocuments() {
5654
+ return log.docIds.size;
5655
+ },
5656
+ takeWrittenDocIds() {
5657
+ const taken = log.docIds;
5658
+ log.docIds = /* @__PURE__ */ new Set();
5659
+ return taken;
5660
+ },
5661
+ close() {
5662
+ if (current === log) current = null;
5663
+ }
5664
+ };
5665
+ },
5666
+ documentWritten(docId) {
5667
+ if (current !== null) current.docIds.add(docId);
5668
+ },
5669
+ segmentWritten(segment) {
5670
+ if (current === null) return;
5671
+ for (const internalId of segment.docStore.allInternalIds()) {
5672
+ const docId = segment.docStore.getExternalId(internalId);
5673
+ if (docId !== void 0) current.docIds.add(docId);
5674
+ }
5675
+ },
5676
+ partitionsReplaced() {
5677
+ if (current !== null) current.replaced = true;
5678
+ }
5679
+ };
5680
+ }
5681
+
5681
5682
  // src/partitioning/manager/index.ts
5682
5683
  function createPartitionManager(indexName, config, language, router, initialPartitionCount, vectorIndexes) {
5683
5684
  requirePatternSearchFor(config.schema);
@@ -5689,6 +5690,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
5689
5690
  const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
5690
5691
  let partitions = [];
5691
5692
  const docPartitionMap = /* @__PURE__ */ new Map();
5693
+ const writeLog = createWriteLogRecorder();
5692
5694
  for (let i = 0; i < count; i++) {
5693
5695
  partitions.push(createPartitionIndex(i, trackPositions));
5694
5696
  }
@@ -5776,9 +5778,23 @@ function createPartitionManager(indexName, config, language, router, initialPart
5776
5778
  return [...partitions];
5777
5779
  },
5778
5780
  setPartitions(newPartitions) {
5781
+ writeLog.partitionsReplaced();
5779
5782
  partitions = [...newPartitions];
5780
5783
  rebuildDocPartitionMap();
5781
5784
  },
5785
+ openWriteLog() {
5786
+ return writeLog.open();
5787
+ },
5788
+ adoptRebuiltPartitions(rebuilt) {
5789
+ if (rebuilt.length !== partitions.length) {
5790
+ throw new NarsilError(
5791
+ ErrorCodes.PARTITION_CORRUPTED,
5792
+ `Index "${indexName}" holds ${partitions.length} partitions, but the rebuild produced ${rebuilt.length}`,
5793
+ { indexName, partitionCount: partitions.length, rebuiltCount: rebuilt.length }
5794
+ );
5795
+ }
5796
+ partitions = [...rebuilt];
5797
+ },
5782
5798
  addPartition() {
5783
5799
  const currentMaxPartitions = config.partitions?.maxPartitions;
5784
5800
  if (currentMaxPartitions !== void 0 && partitions.length >= currentMaxPartitions) {
@@ -5789,11 +5805,13 @@ function createPartitionManager(indexName, config, language, router, initialPart
5789
5805
  );
5790
5806
  }
5791
5807
  const partition = createPartitionIndex(partitions.length, trackPositions);
5808
+ writeLog.partitionsReplaced();
5792
5809
  partitions.push(partition);
5793
5810
  return partition;
5794
5811
  },
5795
5812
  removePartition(partitionId) {
5796
5813
  validatePartitionId(partitionId);
5814
+ writeLog.partitionsReplaced();
5797
5815
  partitions.splice(partitionId, 1);
5798
5816
  rebuildDocPartitionMap();
5799
5817
  },
@@ -5807,6 +5825,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
5807
5825
  if (newCount >= partitions.length) {
5808
5826
  return;
5809
5827
  }
5828
+ writeLog.partitionsReplaced();
5810
5829
  for (const [docId, pid] of docPartitionMap) {
5811
5830
  if (pid >= newCount) {
5812
5831
  docPartitionMap.delete(docId);
@@ -5829,6 +5848,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
5829
5848
  const insertOpts = resolveInsertOptions(options);
5830
5849
  partitions[pid].insert(docId, document, config.schema, language, insertOpts);
5831
5850
  docPartitionMap.set(docId, pid);
5851
+ writeLog.documentWritten(docId);
5832
5852
  },
5833
5853
  remove(docId) {
5834
5854
  const pid = locateDocument(docId);
@@ -5837,11 +5857,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
5837
5857
  }
5838
5858
  partitions[pid].remove(docId, config.schema, language, resolveInsertOptions());
5839
5859
  docPartitionMap.delete(docId);
5840
- },
5841
- rebuildTextIndex(partitionId) {
5842
- validatePartitionId(partitionId);
5843
- partitions[partitionId].rebuildTextIndex(config.schema, language, resolveInsertOptions());
5844
- preparePartialWords(partitionId);
5860
+ writeLog.documentWritten(docId);
5845
5861
  },
5846
5862
  beginBatchRemove() {
5847
5863
  for (let i = 0; i < partitions.length; i++) {
@@ -5861,6 +5877,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
5861
5877
  const updateOpts = resolveInsertOptions(options);
5862
5878
  partitions[pid].update(docId, document, config.schema, language, updateOpts);
5863
5879
  docPartitionMap.set(docId, pid);
5880
+ writeLog.documentWritten(docId);
5864
5881
  },
5865
5882
  get(docId, projection) {
5866
5883
  const pid = locateDocument(docId);
@@ -5911,6 +5928,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
5911
5928
  partition.endBatch();
5912
5929
  for (const docId of payload.docIds) {
5913
5930
  docPartitionMap.set(docId, partitionId);
5931
+ writeLog.documentWritten(docId);
5914
5932
  }
5915
5933
  },
5916
5934
  attachFrozenSegment(partitionId, segment) {
@@ -5925,6 +5943,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
5925
5943
  }
5926
5944
  asCompositePartition(partitionId).attachFrozenSegment(segment);
5927
5945
  registerSegmentDocuments(partitionId, segment);
5946
+ writeLog.segmentWritten(segment);
5928
5947
  },
5929
5948
  swapFrozenSegments(partitionId, dropSegmentIds, replacement) {
5930
5949
  validatePartitionId(partitionId);
@@ -5942,42 +5961,19 @@ function createPartitionManager(indexName, config, language, router, initialPart
5942
5961
  },
5943
5962
  deserializePartition(partitionId, data) {
5944
5963
  validatePartitionId(partitionId);
5964
+ writeLog.partitionsReplaced();
5945
5965
  partitions[partitionId].deserialize(data, config.schema);
5946
5966
  preparePartialWords(partitionId);
5947
5967
  rebuildDocPartitionMapForPartition(partitionId);
5948
5968
  },
5949
5969
  getAggregateStats() {
5950
- let totalDocuments = 0;
5951
- const docFrequencies = /* @__PURE__ */ Object.create(null);
5952
- const totalFieldLengths = /* @__PURE__ */ Object.create(null);
5953
- for (let i = 0; i < partitions.length; i++) {
5954
- const stats = partitions[i].stats;
5955
- totalDocuments += stats.totalDocuments;
5956
- for (const [term, freq] of Object.entries(stats.docFrequencies)) {
5957
- docFrequencies[term] = (docFrequencies[term] ?? 0) + freq;
5958
- }
5959
- for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
5960
- totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
5961
- }
5962
- }
5963
- return { totalDocuments, docFrequencies, totalFieldLengths };
5970
+ return aggregateStatsOf(partitions);
5964
5971
  },
5965
5972
  estimateMemoryBytes() {
5966
- let total = 0;
5967
- for (let i = 0; i < partitions.length; i++) {
5968
- total += partitions[i].estimateMemoryBytes();
5969
- }
5970
- for (const [, vecIndex] of vecIndexes) {
5971
- total += vecIndex.estimateMemoryBytes();
5972
- }
5973
- return total;
5973
+ return estimateBytesOf(partitions, vecIndexes);
5974
5974
  },
5975
5975
  getPartitionStats() {
5976
- return partitions.map((p, i) => ({
5977
- partitionId: i,
5978
- documentCount: p.count(),
5979
- estimatedMemoryBytes: p.estimateMemoryBytes()
5980
- }));
5976
+ return partitionStatsOf(partitions);
5981
5977
  },
5982
5978
  getVectorIndexes() {
5983
5979
  return vecIndexes;
@@ -6066,6 +6062,9 @@ function emptyFieldBeforeRestore(state) {
6066
6062
  state.savedFiles = [];
6067
6063
  state.savedSignature = null;
6068
6064
  }
6065
+ function releaseReplacedVectors(state) {
6066
+ for (const ordinal of state.store.releaseReplacedVectors()) state.osq?.removeOrdinal(ordinal);
6067
+ }
6069
6068
  function adoptGraph(state, graph) {
6070
6069
  if (graph === null && state.hnsw !== null) {
6071
6070
  state.hnsw.clear();
@@ -6601,9 +6600,7 @@ function captureCloneCodes(state) {
6601
6600
  const records = new Uint8Array(slots * layout.slotStride);
6602
6601
  const present = new Uint8Array(slots);
6603
6602
  for (let ordinal = 0; ordinal < slots; ordinal++) {
6604
- const docId = state.store.docIdForOrdinal(ordinal);
6605
- if (docId === void 0) continue;
6606
- const record = quantizer.recordOf(docId);
6603
+ const record = quantizer.recordAtOrdinal(ordinal);
6607
6604
  if (record === void 0) continue;
6608
6605
  records.set(record, ordinal * layout.slotStride);
6609
6606
  present[ordinal] = 1;
@@ -6740,7 +6737,7 @@ async function dispatcherFor(state, graph) {
6740
6737
  workerCount: Math.max(1, host.workerCount)
6741
6738
  };
6742
6739
  }
6743
- if (liveSize(state) < WORKER_COPY_MIN_VECTORS) return null;
6740
+ if (liveSize(state) < WORKER_COPY_MIN_VECTORS || !sharedMemoryAvailable()) return null;
6744
6741
  const handle = await shareGraph(state, graph, graph === state.hnsw);
6745
6742
  const pool = state.workerCopyPool;
6746
6743
  if (handle === null || pool === null || state.sharedHandles.get(graph)?.mode !== "shared") return null;
@@ -6830,12 +6827,40 @@ async function insertLocally(state, graph, docIds, admit) {
6830
6827
  state.buffer.delete(docId);
6831
6828
  count += 1;
6832
6829
  if (count % BUILD_CHUNK_SIZE === 0) {
6830
+ noteWrite(state);
6833
6831
  await yieldToEventLoop();
6834
6832
  }
6835
6833
  }
6834
+ if (count > 0) noteWrite(state);
6836
6835
  return !state.disposed;
6837
6836
  }
6838
6837
 
6838
+ // src/vector/vector-index/dead-bits.ts
6839
+ var NO_LIVE_VECTOR_AT_POSITION = -1;
6840
+ function deadBytesFor(count) {
6841
+ return Math.ceil(count / 8);
6842
+ }
6843
+ function isDead(dead, position) {
6844
+ return dead !== null && (dead[position >> 3] >> (position & 7) & 1) === 1;
6845
+ }
6846
+ function countDead(dead, count) {
6847
+ if (dead === null) return 0;
6848
+ let found = 0;
6849
+ for (let position = 0; position < count; position++) {
6850
+ if (isDead(dead, position)) found += 1;
6851
+ }
6852
+ return found;
6853
+ }
6854
+ function deadBitsWhere(count, isDeadAt) {
6855
+ let dead = null;
6856
+ for (let position = 0; position < count; position++) {
6857
+ if (!isDeadAt(position)) continue;
6858
+ dead ??= new Uint8Array(deadBytesFor(count));
6859
+ dead[position >> 3] |= 1 << (position & 7);
6860
+ }
6861
+ return dead;
6862
+ }
6863
+
6839
6864
  // src/vector/vector-index/disk.ts
6840
6865
  function readsFromDisk(state) {
6841
6866
  return state.storage === "disk";
@@ -6846,6 +6871,7 @@ function holdsVectorsOnDisk(state) {
6846
6871
  function* locationsOf(state, layout) {
6847
6872
  const stride = state.dimension * 4;
6848
6873
  for (let i = 0; i < layout.docIds.length; i++) {
6874
+ if (isDead(layout.dead, i)) continue;
6849
6875
  yield [layout.docIds[i], { path: layout.path, offset: layout.vectorsOffset + i * stride }];
6850
6876
  }
6851
6877
  }
@@ -7213,32 +7239,6 @@ function graphPartsOfEarlierPayload(payload) {
7213
7239
  };
7214
7240
  }
7215
7241
 
7216
- // src/vector/vector-index/dead-bits.ts
7217
- var NO_LIVE_VECTOR_AT_POSITION = -1;
7218
- function deadBytesFor(count) {
7219
- return Math.ceil(count / 8);
7220
- }
7221
- function isDead(dead, position) {
7222
- return dead !== null && (dead[position >> 3] >> (position & 7) & 1) === 1;
7223
- }
7224
- function countDead(dead, count) {
7225
- if (dead === null) return 0;
7226
- let found = 0;
7227
- for (let position = 0; position < count; position++) {
7228
- if (isDead(dead, position)) found += 1;
7229
- }
7230
- return found;
7231
- }
7232
- function deadBitsWhere(count, isDeadAt) {
7233
- let dead = null;
7234
- for (let position = 0; position < count; position++) {
7235
- if (!isDeadAt(position)) continue;
7236
- dead ??= new Uint8Array(deadBytesFor(count));
7237
- dead[position >> 3] |= 1 << (position & 7);
7238
- }
7239
- return dead;
7240
- }
7241
-
7242
7242
  // src/vector/vector-index/persistence.ts
7243
7243
  function recordFor(state, quantizer, docId) {
7244
7244
  const record = quantizer.recordOf(docId);
@@ -7260,7 +7260,7 @@ function sameCentroid(state, planned) {
7260
7260
  }
7261
7261
  return true;
7262
7262
  }
7263
- function partCodes(state, docIds, centroid) {
7263
+ function partCodes(state, docIds, centroid, tombstonedAt) {
7264
7264
  const quantizer = state.osq;
7265
7265
  const layout = state.store.handles.codeLayout;
7266
7266
  if (centroid === null || quantizer === null || layout === null) return null;
@@ -7273,7 +7273,8 @@ function partCodes(state, docIds, centroid) {
7273
7273
  }
7274
7274
  const records = new Uint8Array(docIds.length * layout.slotStride);
7275
7275
  for (let i = 0; i < docIds.length; i++) {
7276
- const record = recordFor(state, quantizer, docIds[i]);
7276
+ const tombstoned = tombstonedAt.get(i);
7277
+ const record = tombstoned === void 0 ? recordFor(state, quantizer, docIds[i]) : quantizer.recordAtOrdinal(tombstoned);
7277
7278
  if (record !== void 0) records.set(record, i * layout.slotStride);
7278
7279
  }
7279
7280
  return { bits: quantizer.bits, centroid, records };
@@ -7430,33 +7431,39 @@ function isLive(state, ordinal) {
7430
7431
  const docId = state.store.docIdForOrdinal(ordinal);
7431
7432
  return docId !== void 0 && !state.tombstones.has(docId);
7432
7433
  }
7433
- function liveCountOf(state, file) {
7434
- let live = 0;
7435
- for (let i = 0; i < file.ordinals.length; i++) if (isLive(state, file.ordinals[i])) live += 1;
7436
- return live;
7434
+ function isGraphTombstone(state, ordinal) {
7435
+ return ordinal !== NO_LIVE_VECTOR_AT_POSITION && state.hnsw !== null && state.hnsw.holdsTombstone(ordinal);
7436
+ }
7437
+ function isCarried(state, ordinal) {
7438
+ return isLive(state, ordinal) || isGraphTombstone(state, ordinal);
7439
+ }
7440
+ function carriedCountOf(state, file) {
7441
+ let carried = 0;
7442
+ for (let i = 0; i < file.ordinals.length; i++) if (isCarried(state, file.ordinals[i])) carried += 1;
7443
+ return carried;
7437
7444
  }
7438
7445
  function deadBitsOf(state, file) {
7439
7446
  return deadBitsWhere(file.ordinals.length, (position) => !isLive(state, file.ordinals[position]));
7440
7447
  }
7441
- function filesWorthKeeping(state, listed, calibration, liveCounts) {
7448
+ function filesWorthKeeping(state, listed, calibration, carriedCounts) {
7442
7449
  const kept = [];
7443
7450
  for (const file of listed) {
7444
- const live = liveCountOf(state, file);
7445
- const deadShare = file.ordinals.length === 0 ? 1 : 1 - live / file.ordinals.length;
7446
- if (file.calibration !== calibration || deadShare > VECTOR_FILE_REPLACED_ABOVE_DEAD_SHARE) continue;
7451
+ const carried = carriedCountOf(state, file);
7452
+ const deadWithoutNodeShare = file.ordinals.length === 0 ? 1 : 1 - carried / file.ordinals.length;
7453
+ if (file.calibration !== calibration || deadWithoutNodeShare > VECTOR_FILE_REPLACED_ABOVE_DEAD_SHARE) continue;
7447
7454
  kept.push(file);
7448
- liveCounts.push(live);
7455
+ carriedCounts.push(carried);
7449
7456
  }
7450
7457
  return kept;
7451
7458
  }
7452
- function absorbSmallerPartialFiles(kept, liveCounts, vectorsToWrite, capacity) {
7459
+ function absorbSmallerPartialFiles(kept, carriedCounts, vectorsToWrite, capacity) {
7453
7460
  let pending = vectorsToWrite;
7454
7461
  for (let index = kept.length - 1; index >= 0 && pending > 0; index--) {
7455
7462
  if (kept[index].ordinals.length >= capacity) continue;
7456
- if (liveCounts[index] > pending) return;
7457
- pending += liveCounts[index];
7463
+ if (carriedCounts[index] > pending) return;
7464
+ pending += carriedCounts[index];
7458
7465
  kept.splice(index, 1);
7459
- liveCounts.splice(index, 1);
7466
+ carriedCounts.splice(index, 1);
7460
7467
  }
7461
7468
  }
7462
7469
  function ordinalsOutsideOf(state, kept) {
@@ -7470,14 +7477,15 @@ function ordinalsOutsideOf(state, kept) {
7470
7477
  }
7471
7478
  const outside = [];
7472
7479
  for (let ordinal = 0; ordinal < slots; ordinal++) {
7473
- if (saved[ordinal] === 0 && isLive(state, ordinal)) outside.push(ordinal);
7480
+ if (saved[ordinal] === 0 && isCarried(state, ordinal)) outside.push(ordinal);
7474
7481
  }
7475
7482
  return outside;
7476
7483
  }
7477
- function countOutside(state, liveCounts) {
7484
+ function countOutside(state, carriedCounts) {
7478
7485
  let held = 0;
7479
- for (const live of liveCounts) held += live;
7480
- return Math.max(0, state.store.size - state.tombstones.size - held);
7486
+ for (const carried of carriedCounts) held += carried;
7487
+ const tombstonedNodes = state.hnsw?.tombstoneCount ?? 0;
7488
+ return Math.max(0, state.store.size - state.tombstones.size + tombstonedNodes - held);
7481
7489
  }
7482
7490
  function numberEveryVector(state, kept, written) {
7483
7491
  let total = written.length;
@@ -7488,7 +7496,7 @@ function numberEveryVector(state, kept, written) {
7488
7496
  for (const file of kept) {
7489
7497
  for (let i = 0; i < file.ordinals.length; i++, number++) {
7490
7498
  const ordinal = file.ordinals[i];
7491
- if (!isLive(state, ordinal)) continue;
7499
+ if (!isCarried(state, ordinal)) continue;
7492
7500
  numberOfOrdinal[ordinal] = number;
7493
7501
  ordinalOfNumber[number] = ordinal;
7494
7502
  }
@@ -7509,16 +7517,30 @@ function graphPartsOf(state, numbers) {
7509
7517
  );
7510
7518
  return parts.map((graph) => ({ v: VECTOR_GRAPH_PART_PAYLOAD_VERSION, fieldName: state.fieldName, graphs: [graph] }));
7511
7519
  }
7512
- function readFile(state, planned, docIds, centroid) {
7520
+ function graphPartsWithoutNodes(parts, numbers) {
7521
+ return parts.map((part, index) => ({
7522
+ ...part,
7523
+ graphs: part.graphs.map((graph) => graphWithoutNodes(graph, index * VECTOR_GRAPH_PART_VECTORS, numbers))
7524
+ }));
7525
+ }
7526
+ function plannedVectorOf(state, ordinal) {
7527
+ const docId = state.store.docIdForOrdinal(ordinal) ?? state.store.view.docIdAt(ordinal) ?? "";
7528
+ return { ordinal, docId, tombstoned: !isLive(state, ordinal) };
7529
+ }
7530
+ function readFile(state, planned, centroid) {
7513
7531
  const dimension = state.dimension;
7514
7532
  const vectors = new Float32Array(planned.length * dimension);
7515
7533
  const ordinals = new Int32Array(planned.length).fill(NO_LIVE_VECTOR_AT_POSITION);
7534
+ const tombstonedAt = /* @__PURE__ */ new Map();
7535
+ const docIds = planned.map((vector) => vector.docId);
7516
7536
  for (let i = 0; i < planned.length; i++) {
7517
- const held = state.store.holdsOrdinal(planned[i]) ? planned[i] : state.store.getOrdinal(docIds[i]);
7537
+ const { ordinal, docId, tombstoned } = planned[i];
7538
+ const held = tombstoned || state.store.holdsOrdinal(ordinal) ? ordinal : state.store.getOrdinal(docId);
7518
7539
  const entry = held === void 0 ? void 0 : state.store.entryForOrdinal(held);
7519
7540
  if (held === void 0 || entry === void 0) continue;
7520
7541
  vectors.set(entry.vector, i * dimension);
7521
7542
  ordinals[i] = held;
7543
+ if (tombstoned) tombstonedAt.set(i, held);
7522
7544
  }
7523
7545
  return {
7524
7546
  payload: {
@@ -7526,10 +7548,11 @@ function readFile(state, planned, docIds, centroid) {
7526
7548
  fieldName: state.fieldName,
7527
7549
  dimension,
7528
7550
  docIds,
7529
- codes: partCodes(state, docIds, centroid),
7551
+ codes: partCodes(state, docIds, centroid, tombstonedAt),
7530
7552
  vectors: vectorsToBytes(vectors)
7531
7553
  },
7532
- ordinals
7554
+ ordinals,
7555
+ dead: deadBitsWhere(planned.length, (i) => planned[i].tombstoned || ordinals[i] === NO_LIVE_VECTOR_AT_POSITION)
7533
7556
  };
7534
7557
  }
7535
7558
  function planCheckpoint(state, listedKeys, rewritesGraph = false) {
@@ -7537,14 +7560,20 @@ function planCheckpoint(state, listedKeys, rewritesGraph = false) {
7537
7560
  const calibration = signature.calibration;
7538
7561
  const trusted = registryMatches(state, listedKeys);
7539
7562
  const capacity = fileCapacityOf(state);
7540
- const liveCounts = [];
7541
- const keptFiles = filesWorthKeeping(state, trusted ? state.savedFiles : [], calibration, liveCounts);
7542
- absorbSmallerPartialFiles(keptFiles, liveCounts, countOutside(state, liveCounts), capacity);
7563
+ const carriedCounts = [];
7564
+ const keptFiles = filesWorthKeeping(state, trusted ? state.savedFiles : [], calibration, carriedCounts);
7565
+ absorbSmallerPartialFiles(keptFiles, carriedCounts, countOutside(state, carriedCounts), capacity);
7543
7566
  const written = ordinalsOutsideOf(state, keptFiles);
7544
- const writtenDocIds = written.map((ordinal) => state.store.docIdForOrdinal(ordinal) ?? "");
7567
+ const planned = written.map((ordinal) => plannedVectorOf(state, ordinal));
7545
7568
  const centroid = codedCentroid(state);
7546
7569
  const listedBefore = trusted && listedKeys !== null;
7547
7570
  const unchanged = !rewritesGraph && listedBefore && sameSignature(state.savedSignature, signature) && written.length === 0;
7571
+ let firstWrittenNumber = 0;
7572
+ for (const file of keptFiles) firstWrittenNumber += file.ordinals.length;
7573
+ const graphParts = unchanged ? null : graphPartsOf(state, numberEveryVector(state, keptFiles, written));
7574
+ const vanished = /* @__PURE__ */ new Set();
7575
+ let vanishedWhenStripped = 0;
7576
+ let graphPartsToWrite = graphParts;
7548
7577
  return {
7549
7578
  unchanged,
7550
7579
  keptFiles,
@@ -7555,12 +7584,22 @@ function planCheckpoint(state, listedKeys, rewritesGraph = false) {
7555
7584
  dead: deadBitsOf(state, file)
7556
7585
  })),
7557
7586
  newFiles: Math.ceil(written.length / capacity),
7558
- graphParts: unchanged ? null : graphPartsOf(state, numberEveryVector(state, keptFiles, written)),
7587
+ get graphParts() {
7588
+ if (graphParts !== null && vanished.size !== vanishedWhenStripped) {
7589
+ graphPartsToWrite = graphPartsWithoutNodes(graphParts, vanished);
7590
+ vanishedWhenStripped = vanished.size;
7591
+ }
7592
+ return graphPartsToWrite;
7593
+ },
7559
7594
  signature,
7560
7595
  readNewFile(index) {
7561
7596
  const start = index * capacity;
7562
7597
  const end = start + capacity;
7563
- return readFile(state, written.slice(start, end), writtenDocIds.slice(start, end), centroid);
7598
+ const file = readFile(state, planned.slice(start, end), centroid);
7599
+ for (let i = 0; i < file.ordinals.length; i++) {
7600
+ if (file.ordinals[i] === NO_LIVE_VECTOR_AT_POSITION) vanished.add(firstWrittenNumber + start + i);
7601
+ }
7602
+ return file;
7564
7603
  }
7565
7604
  };
7566
7605
  }
@@ -7593,9 +7632,10 @@ function insertNodesOfLaterGraph(state, graph, later, ordinals) {
7593
7632
  if (docId !== void 0 && !graph.has(docId)) graph.insertOrdinal(ordinal);
7594
7633
  }
7595
7634
  }
7596
- function restoreGraph(state, payload, ordinalOfNumber) {
7635
+ function restoreGraph(state, payload, ordinalOfNumber, tombstoned) {
7597
7636
  const first = payload?.graphs[0]?.parts[0];
7598
7637
  if (payload === null || first === void 0) {
7638
+ releaseReplacedVectors(state);
7599
7639
  bufferEveryVectorOutsideTheGraph(state);
7600
7640
  return;
7601
7641
  }
@@ -7606,11 +7646,13 @@ function restoreGraph(state, payload, ordinalOfNumber) {
7606
7646
  state.osq ?? void 0
7607
7647
  );
7608
7648
  graph.deserializeNumbered(payload.graphs[0], ordinalOfNumber);
7649
+ for (const ordinal of tombstoned) graph.markTombstoneOrdinal(ordinal);
7609
7650
  adoptGraph(state, graph);
7610
7651
  for (const later of payload.graphs.slice(1)) insertNodesOfLaterGraph(state, graph, later, ordinalOfNumber);
7611
7652
  bufferEveryVectorOutsideTheGraph(state);
7612
7653
  if (state.buffer.size <= graph.size) return;
7613
7654
  adoptGraph(state, null);
7655
+ releaseReplacedVectors(state);
7614
7656
  state.buffer.clear();
7615
7657
  bufferEveryVectorOutsideTheGraph(state);
7616
7658
  }
@@ -7619,6 +7661,9 @@ function beginCheckpointRestore(state, shape) {
7619
7661
  const cold = readsFromDisk(state) && (shape.holdsGraph || shape.liveVectors >= state.promotionThreshold);
7620
7662
  const restoresCodes = state.osq !== null && osqBitsOf(state.quantizationMode) !== null && shape.holdsGraph;
7621
7663
  const files = [];
7664
+ const graphNodes = shape.graph?.graphs[0] ?? null;
7665
+ const tombstoned = /* @__PURE__ */ new Set();
7666
+ let nextNumber = 0;
7622
7667
  let sharedCodes = null;
7623
7668
  let codesAgree = true;
7624
7669
  function restoreRecords(payload, ordinals) {
@@ -7640,9 +7685,14 @@ function beginCheckpointRestore(state, shape) {
7640
7685
  const stride = layout.slotStride;
7641
7686
  for (let i = 0; i < ordinals.length; i++) {
7642
7687
  if (ordinals[i] === NO_LIVE_VECTOR_AT_POSITION) continue;
7643
- quantizer.restoreRecord(payload.docIds[i], codes.records.subarray(i * stride, (i + 1) * stride));
7688
+ const record = codes.records.subarray(i * stride, (i + 1) * stride);
7689
+ if (tombstoned.has(ordinals[i])) quantizer.restoreRecordAtOrdinal(ordinals[i], record);
7690
+ else quantizer.restoreRecord(payload.docIds[i], record);
7644
7691
  }
7645
7692
  }
7693
+ function graphHoldsNodeFor(number) {
7694
+ return graphNodes !== null && topLayerInParts(graphNodes, number) >= 0;
7695
+ }
7646
7696
  function addFile(file) {
7647
7697
  const { payload, location } = file;
7648
7698
  if (payload.dimension !== state.dimension) {
@@ -7657,13 +7707,21 @@ function beginCheckpointRestore(state, shape) {
7657
7707
  const vectors = bytesToVectors(payload.vectors);
7658
7708
  const fileIndex = onDisk && cold ? state.store.addVectorFile(location.path) : null;
7659
7709
  const ordinals = new Int32Array(payload.docIds.length).fill(NO_LIVE_VECTOR_AT_POSITION);
7710
+ const firstNumber = nextNumber;
7711
+ nextNumber += payload.docIds.length;
7660
7712
  for (let i = 0; i < payload.docIds.length; i++) {
7661
- if (isDead(file.dead, i)) continue;
7713
+ const dead = isDead(file.dead, i);
7714
+ if (dead && !graphHoldsNodeFor(firstNumber + i)) continue;
7662
7715
  const docId = payload.docIds[i];
7663
- const replaced = state.store.getOrdinal(docId);
7664
- if (replaced !== void 0) state.osq?.removeOrdinal(replaced);
7665
7716
  const vector = vectors.subarray(i * dimension, (i + 1) * dimension);
7666
7717
  const offset = onDisk ? location.vectorsOffset + i * dimension * 4 : 0;
7718
+ if (dead) {
7719
+ ordinals[i] = state.store.insertReplaced(docId, vector, fileIndex === null ? void 0 : { fileIndex, offset });
7720
+ tombstoned.add(ordinals[i]);
7721
+ continue;
7722
+ }
7723
+ const replaced = state.store.getOrdinal(docId);
7724
+ if (replaced !== void 0) state.osq?.removeOrdinal(replaced);
7667
7725
  if (fileIndex !== null) {
7668
7726
  ordinals[i] = state.store.insertCold(docId, magnitude(vector), { fileIndex, offset });
7669
7727
  continue;
@@ -7674,21 +7732,27 @@ function beginCheckpointRestore(state, shape) {
7674
7732
  restoreRecords(payload, ordinals);
7675
7733
  files.push({ id: file.id, key: file.key, ordinals, coded: payload.codes !== null });
7676
7734
  }
7677
- function finish(graph) {
7735
+ function finish() {
7678
7736
  const codesRestored = restoresCodes && codesAgree && sharedCodes !== null;
7679
- if (restoresCodes && !codesRestored) recalibrateFromStore(state);
7737
+ if (restoresCodes && !codesRestored) {
7738
+ recalibrateFromStore(state);
7739
+ for (const ordinal of tombstoned) state.osq?.writeCodes(ordinal);
7740
+ }
7680
7741
  let total = 0;
7681
7742
  for (const file of files) total += file.ordinals.length;
7682
7743
  const ordinalOfNumber = new Int32Array(total);
7683
7744
  let number = 0;
7684
7745
  for (const file of files) {
7685
7746
  for (let i = 0; i < file.ordinals.length; i++) {
7686
- if (!state.store.holdsOrdinal(file.ordinals[i])) file.ordinals[i] = NO_LIVE_VECTOR_AT_POSITION;
7747
+ const ordinal = file.ordinals[i];
7748
+ if (!state.store.holdsOrdinal(ordinal) && !tombstoned.has(ordinal)) {
7749
+ file.ordinals[i] = NO_LIVE_VECTOR_AT_POSITION;
7750
+ }
7687
7751
  }
7688
7752
  ordinalOfNumber.set(file.ordinals, number);
7689
7753
  number += file.ordinals.length;
7690
7754
  }
7691
- restoreGraph(state, graph, ordinalOfNumber);
7755
+ restoreGraph(state, shape.graph, ordinalOfNumber, tombstoned);
7692
7756
  const current = signatureOf(state).calibration;
7693
7757
  const codedUnder = codesRestored && current !== null ? current : CALIBRATION_NEVER_CURRENT;
7694
7758
  const saved = files.map((file) => ({
@@ -7705,11 +7769,12 @@ function beginCheckpointRestore(state, shape) {
7705
7769
 
7706
7770
  // src/vector/vector-index/maintenance.ts
7707
7771
  function compact(state) {
7708
- if (state.tombstones.size === 0) return;
7772
+ if (state.tombstones.size === 0 && state.store.replacedVectorCount === 0) return;
7709
7773
  if (state.hnsw) {
7710
7774
  state.compactedNodeCount += state.hnsw.tombstoneCount;
7711
7775
  state.hnsw.compactTombstones();
7712
7776
  }
7777
+ state.freshGraph?.compactTombstones();
7713
7778
  for (const docId of state.tombstones) {
7714
7779
  state.store.remove(docId);
7715
7780
  state.buffer.delete(docId);
@@ -7717,6 +7782,7 @@ function compact(state) {
7717
7782
  state.osq.remove(docId);
7718
7783
  }
7719
7784
  }
7785
+ releaseReplacedVectors(state);
7720
7786
  state.tombstones.clear();
7721
7787
  }
7722
7788
  function recalibrateWhileNoThreadSearches(state, graph) {
@@ -7748,6 +7814,7 @@ async function foldIntoGraph(state) {
7748
7814
  const graph = state.hnsw;
7749
7815
  if (graph === null || rebuildNeeded) {
7750
7816
  if (state.osq?.isCalibrated()) recalibrateWhileNoThreadSearches(state, graph);
7817
+ else calibrateQuantizer(state);
7751
7818
  await buildGraphFromStore(state);
7752
7819
  } else {
7753
7820
  await insertMissing(state, graph);
@@ -7794,8 +7861,9 @@ function maintenanceStatus(state) {
7794
7861
  const storeSize = state.store.size;
7795
7862
  const graphCount = state.hnsw ? 1 : 0;
7796
7863
  const graphNodes = state.hnsw === null ? storeSize : state.hnsw.size + state.hnsw.tombstoneCount;
7797
- const tombstoneRatio = graphNodes > 0 ? state.tombstones.size / graphNodes : 0;
7798
- const estimatedCompactMs = Math.round(state.tombstones.size * ESTIMATED_MS_PER_TOMBSTONE * state.dimensionScale);
7864
+ const tombstoned = state.tombstones.size + state.store.replacedVectorCount;
7865
+ const tombstoneRatio = graphNodes > 0 ? tombstoned / graphNodes : 0;
7866
+ const estimatedCompactMs = Math.round(tombstoned * ESTIMATED_MS_PER_TOMBSTONE * state.dimensionScale);
7799
7867
  const optimizeVectorCount = state.hnsw === null || graphNeedsRebuild(state) ? storeSize : state.buffer.size;
7800
7868
  const estimatedOptimizeMs = Math.round(optimizeVectorCount * ESTIMATED_MS_PER_VECTOR_REBUILD * state.dimensionScale);
7801
7869
  return {
@@ -8056,19 +8124,19 @@ function searchWithFilter(state, query, k, options, filter) {
8056
8124
  return scannedOutcome(scan2);
8057
8125
  }
8058
8126
  }
8059
- const graphOptions = { filter, efSearch, oversample };
8060
- if (state.buffer.size === 0) {
8061
- const hnswResults2 = state.hnsw.search(query, k, metric, minSimilarity, graphOptions);
8062
- return fetchedOutcome(
8063
- state,
8064
- hnswResults2.map((r) => ({ docId: r.docId, score: r.score })),
8065
- minSimilarity,
8066
- filter
8067
- );
8068
- }
8069
- const hnswResults = state.hnsw.search(query, k, metric, minSimilarity, graphOptions).map((r) => ({ docId: r.docId, score: r.score }));
8070
- const buffered = bruteForceSearch(state, query, k, metric, minSimilarity, bufferCandidates(state, filter));
8071
- return fetchedOutcome(state, mergeResults(hnswResults, buffered.results, k), minSimilarity, filter);
8127
+ const hnswResults = state.hnsw.search(query, k, metric, minSimilarity, { filter, efSearch, oversample }).map((r) => ({ docId: r.docId, score: r.score }));
8128
+ return fetchedOutcome(
8129
+ state,
8130
+ withBufferedVectors(state, query, k, options, filter, hnswResults),
8131
+ minSimilarity,
8132
+ filter
8133
+ );
8134
+ }
8135
+ function withBufferedVectors(state, query, k, options, filter, graphResults) {
8136
+ if (state.buffer.size === 0) return graphResults;
8137
+ const candidates = bufferCandidates(state, filter);
8138
+ const buffered = bruteForceSearch(state, query, k, options.metric, options.minSimilarity, candidates);
8139
+ return mergeResults(graphResults, buffered.results, k);
8072
8140
  }
8073
8141
 
8074
8142
  // src/vector/vector-index/index.ts
@@ -8169,11 +8237,12 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
8169
8237
  }
8170
8238
  noteWrite(state);
8171
8239
  state.tombstones.delete(docId);
8172
- state.store.insert(docId, vector, partitionId);
8240
+ const graphHoldsPrevious = previous !== void 0 && ((state.hnsw?.holdsNode(previous) ?? false) || (state.freshGraph?.holdsNode(previous) ?? false));
8241
+ state.store.insert(docId, vector, partitionId, graphHoldsPrevious);
8173
8242
  if (previous !== void 0) {
8174
8243
  state.hnsw?.markTombstoneOrdinal(previous);
8175
8244
  state.freshGraph?.markTombstoneOrdinal(previous);
8176
- state.osq?.removeOrdinal(previous);
8245
+ if (!graphHoldsPrevious) state.osq?.removeOrdinal(previous);
8177
8246
  }
8178
8247
  state.buffer.add(docId);
8179
8248
  state.pendingLocations.delete(docId);
@@ -8243,10 +8312,13 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
8243
8312
  efSearch: options.efSearch,
8244
8313
  oversample: options.oversample
8245
8314
  });
8246
- if (viaWorkerCopy !== null) return fetchedOutcome(state, viaWorkerCopy, options.minSimilarity, filter);
8247
8315
  if (confined && state.revision !== filterRevision) {
8248
8316
  filter = filterForOptions(state, options);
8249
8317
  }
8318
+ if (viaWorkerCopy !== null) {
8319
+ const results = withBufferedVectors(state, query, k, options, filter, viaWorkerCopy);
8320
+ return fetchedOutcome(state, results, options.minSimilarity, filter);
8321
+ }
8250
8322
  return searchWithFilter(state, query, k, options, filter);
8251
8323
  }
8252
8324
  return {
@@ -10942,4 +11014,4 @@ function createDirectExecutor(options) {
10942
11014
  };
10943
11015
  }
10944
11016
 
10945
- export { CHECKSUM_YIELD_CHUNK_BYTES, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, MAX_FILE_BYTES, NATIVE_CRC32_MIN_BYTES, NO_LIVE_VECTOR_AT_POSITION, SHUTDOWN_TIMEOUT_MS, VALID_HNSW_METRICS, VECTOR_FILE_MAX_VECTORS, VECTOR_FILE_PAYLOAD_VERSION, assembleGraphParts, buildSegmentPayload, bytesToVectors, countDead, createDirectExecutor, createFrozenSegment, createPartitionIndex, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, deadBitsWhere, deadBytesFor, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, decodeVectorFilePayload, decodeVectorGraphPartPayload, decodeVectorGraphPayload, decodeVectorIndexParts, encodeSegmentState, freezeSegmentShared, generateId, getStopWords, getTokenizer, graphPartsOfEarlierPayload, hasStopWords, hasTokenizer, isCompositePartition, isDead, mergeFrozenSegments, mergeSegmentContents, missingWorkerEntry, nodeThreadId, partitionStateOf, readHeapStatistics, registerStopWords, registerTokenizer, resolveIndexAnalysis, resolvePartitionInsertOptions, resolveRequestThreadCount, resolveWorkerCount, resolveWorkerEntry, segmentArraysOf, selectionKeeps, shareSegmentArrays, spawnNodeWorker, splitWorkerBudget, survivorDocumentsOf, vectorsPerFile, vectorsToBytes, workerResourceLimits };
11017
+ export { CHECKSUM_YIELD_CHUNK_BYTES, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, MAX_FILE_BYTES, NATIVE_CRC32_MIN_BYTES, SHUTDOWN_TIMEOUT_MS, VALID_HNSW_METRICS, VECTOR_FILE_MAX_VECTORS, VECTOR_FILE_PAYLOAD_VERSION, assembleGraphParts, buildSegmentPayload, bytesToVectors, countDead, createDirectExecutor, createFrozenSegment, createPartitionIndex, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, deadBytesFor, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, decodeVectorFilePayload, decodeVectorGraphPartPayload, decodeVectorGraphPayload, decodeVectorIndexParts, encodeSegmentState, freezeSegmentShared, generateId, getStopWords, getTokenizer, graphPartsOfEarlierPayload, hasStopWords, hasTokenizer, isCompositePartition, isDead, mergeFrozenSegments, mergeSegmentContents, missingWorkerEntry, nodeThreadId, partitionStateOf, readHeapStatistics, registerStopWords, registerTokenizer, resolveIndexAnalysis, resolvePartitionInsertOptions, resolveRequestThreadCount, resolveWorkerCount, resolveWorkerEntry, segmentArraysOf, selectionKeeps, shareSegmentArrays, spawnNodeWorker, splitWorkerBudget, survivorDocumentsOf, vectorsPerFile, vectorsToBytes, workerResourceLimits };