@delali/narsil 0.3.1-next.84 → 0.3.1-next.86

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,8 +1,8 @@
1
1
  import { createGrowingPartialWords, requirePatternSearchFor, requirePartialWordSearchFor, buildPartialWordTableFrom, patternIndexOf, encodePartialSpellingsOf, countPartialSpelling, joinPatternWorkMeter, countsWithoutScores, fanOutMatchCount, fanOutQuery, sanitizeGlobalStats, expandPartialWords, computeFacets, cloneProjected, projectionKeepsField, setNestedValue, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, PATTERN_HOST, validatedFilterOf, everyValueFacetConfig, mergeFacets, kWayMerge, polygonContainment, polygonBoxes, radiusBoxes, locationCode, undeclaredFilterField, coveringRanges } from './chunk-XYDP7CCJ.mjs';
2
2
  import { getLanguage, detectRuntime } from './chunk-OLXM65YH.mjs';
3
3
  import { MAX_DOC_ID_LENGTH } from './chunk-ZCKPRCYI.mjs';
4
- import { SCRATCH_SLOTS_PER_THREAD_POOL, osqRecordBytes, openSharedVectorField, osqBitsOf, createVectorStore, DISK_STORAGE_BLOCK_BYTES, createOsqQuantizer, magnitude, graphWithoutNodes, createHNSWIndex, nativeStoreScratchBytes, createOrdinalFilter, isTombstoned, addToOrdinalFilter, ordinalFilterValues, nativeScoresOf, toScore, removeFromOrdinalFilter, ordinalFilterHas, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, topLayerInParts, STORE_CALIBRATION_GENERATION, entryForOrd, toDistance, sharedMemoryAvailable, requireWholeNeighbourLists, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, releaseLocksHeldBy, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS } from './chunk-RZXNWQSM.mjs';
5
- import { noteFallback } from './chunk-B3UOQJD3.mjs';
4
+ import { SCRATCH_SLOTS_PER_THREAD_POOL, osqRecordBytes, openSharedVectorField, osqBitsOf, createVectorStore, DISK_STORAGE_BLOCK_BYTES, createOsqQuantizer, STORE_UNPLACED_COUNT, STORE_REMOVED_COUNT, STORE_LIVE_COUNT, ordinalFilterValues, UNPLACED_FLAG_CHUNK_ORDINALS, magnitude, graphWithoutNodes, createHNSWIndex, nativeStoreScratchBytes, nativeScoresOf, toScore, ordinalFilterHas, createOrdinalFilter, addToOrdinalFilter, removeFromOrdinalFilter, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, topLayerInParts, STORE_CALIBRATION_GENERATION, toDistance, sharedMemoryAvailable, requireWholeNeighbourLists, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, releaseLocksHeldBy, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS } from './chunk-B3F7LZXM.mjs';
5
+ import { noteFallback } from './chunk-NFF5TDRG.mjs';
6
6
  import { GrowableUint32, bitsetHas, createBitSet, bitsetSet, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, bitsetFromSet, bitsetIsEmpty } from './chunk-SREPGC6O.mjs';
7
7
  import { GEO_UNIT_METRES } from './chunk-WQBCBKY3.mjs';
8
8
  import { tokenizeOptions, getNestedValue, visitTokens, expandTermPrefix, suggestDisplayTerms, textFieldsChanged, sameFields, codePointOrder, encodeStringBlob, encodeLookupKey, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, getFlatSchema, tokenize, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, getOrCreateFieldNameIndex, getFieldValueByInternalId, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT, partitionDocumentFrequency } from './chunk-4VBG6PEH.mjs';
@@ -6066,6 +6066,7 @@ function releaseReplacedVectors(state) {
6066
6066
  for (const ordinal of state.store.releaseReplacedVectors()) state.osq?.removeOrdinal(ordinal);
6067
6067
  }
6068
6068
  function adoptGraph(state, graph) {
6069
+ if (state.hnsw !== null && state.hnsw !== graph) state.hnsw.retire();
6069
6070
  if (graph === null && state.hnsw !== null) {
6070
6071
  state.hnsw.clear();
6071
6072
  }
@@ -6807,7 +6808,7 @@ async function placeChunk(state, graph, dispatch, chunkDocIds, chunkOrdinals, fa
6807
6808
  continue;
6808
6809
  }
6809
6810
  if (!state.tombstones.has(docId) && !graph.has(docId) && graph.insertOrdinal(ordinal)) placedHere += 1;
6810
- state.buffer.delete(docId);
6811
+ if (graph === state.hnsw) state.buffer.delete(docId);
6811
6812
  }
6812
6813
  if (placedHere > 0 && outcome !== null && !fallback.shortAnswerWarned) {
6813
6814
  fallback.shortAnswerWarned = true;
@@ -6824,7 +6825,7 @@ async function insertLocally(state, graph, docIds, admit) {
6824
6825
  const ordinal = state.store.getOrdinal(docId);
6825
6826
  if (ordinal === void 0) continue;
6826
6827
  graph.insertOrdinal(ordinal);
6827
- state.buffer.delete(docId);
6828
+ if (graph === state.hnsw) state.buffer.delete(docId);
6828
6829
  count += 1;
6829
6830
  if (count % BUILD_CHUNK_SIZE === 0) {
6830
6831
  noteWrite(state);
@@ -6941,6 +6942,10 @@ async function buildGraphFromStore(state) {
6941
6942
  }
6942
6943
  const previous = state.hnsw;
6943
6944
  adoptGraph(state, graph);
6945
+ for (const docId of [...state.buffer]) {
6946
+ const ordinal = state.store.getOrdinal(docId);
6947
+ if (ordinal !== void 0 && graph.holdsNode(ordinal) && !graph.holdsTombstone(ordinal)) state.buffer.delete(docId);
6948
+ }
6944
6949
  if (previous !== null && previous !== graph) dropSharedGraph(state, previous);
6945
6950
  await releasePendingLocations(state);
6946
6951
  }
@@ -7767,6 +7772,87 @@ function beginCheckpointRestore(state, shape) {
7767
7772
  return { addFile, finish };
7768
7773
  }
7769
7774
 
7775
+ // src/vector/vector-index/flagged-documents.ts
7776
+ var FlaggedDocuments = class extends Set {
7777
+ constructor(store, layout) {
7778
+ super();
7779
+ this.store = store;
7780
+ this.layout = layout;
7781
+ }
7782
+ ordinals = /* @__PURE__ */ new Map();
7783
+ handles = null;
7784
+ flags = new Uint8Array(0);
7785
+ chunks = null;
7786
+ add(docId) {
7787
+ super.add(docId);
7788
+ const handles = this.openHandles();
7789
+ if (handles === null) return this;
7790
+ this.flagCurrentOrdinal(docId);
7791
+ Atomics.store(handles.header, this.layout.countWord, this.size);
7792
+ return this;
7793
+ }
7794
+ delete(docId) {
7795
+ const held = super.delete(docId);
7796
+ const handles = this.openHandles();
7797
+ if (handles === null) return held;
7798
+ const ordinal = this.ordinals.get(docId);
7799
+ if (ordinal !== void 0) {
7800
+ this.ordinals.delete(docId);
7801
+ this.unflag(ordinal);
7802
+ }
7803
+ Atomics.store(handles.header, this.layout.countWord, this.size);
7804
+ return held;
7805
+ }
7806
+ clear() {
7807
+ super.clear();
7808
+ const handles = this.openHandles();
7809
+ if (handles !== null) {
7810
+ for (const ordinal of this.ordinals.values()) this.unflag(ordinal);
7811
+ Atomics.store(handles.header, this.layout.countWord, 0);
7812
+ }
7813
+ this.ordinals.clear();
7814
+ }
7815
+ flagCurrentOrdinal(docId) {
7816
+ const ordinal = this.store.getOrdinal(docId);
7817
+ const previous = this.ordinals.get(docId);
7818
+ if (ordinal === void 0 || ordinal === previous) return;
7819
+ if (this.chunks !== null) Atomics.add(this.chunks, Math.floor(ordinal / UNPLACED_FLAG_CHUNK_ORDINALS), 1);
7820
+ Atomics.store(this.flags, ordinal, 1);
7821
+ this.ordinals.set(docId, ordinal);
7822
+ if (previous !== void 0) this.unflag(previous);
7823
+ }
7824
+ unflag(ordinal) {
7825
+ Atomics.store(this.flags, ordinal, 0);
7826
+ if (this.chunks !== null) Atomics.sub(this.chunks, Math.floor(ordinal / UNPLACED_FLAG_CHUNK_ORDINALS), 1);
7827
+ }
7828
+ openHandles() {
7829
+ if (this.store.dimension === 0) return null;
7830
+ const handles = this.store.handles;
7831
+ if (handles === this.handles) return handles;
7832
+ this.handles = handles;
7833
+ this.flags = new Uint8Array(this.layout.flagsOf(handles));
7834
+ const chunksOf = this.layout.chunksOf;
7835
+ this.chunks = chunksOf === null ? null : new Int32Array(chunksOf(handles));
7836
+ this.ordinals.clear();
7837
+ for (const docId of this) this.flagCurrentOrdinal(docId);
7838
+ return handles;
7839
+ }
7840
+ };
7841
+ function createUnplacedVectors(store) {
7842
+ return new FlaggedDocuments(store, {
7843
+ flagsOf: (handles) => handles.unplaced,
7844
+ chunksOf: (handles) => handles.unplacedChunks,
7845
+ countWord: STORE_UNPLACED_COUNT
7846
+ });
7847
+ }
7848
+ function createRemovedDocuments(store) {
7849
+ return new FlaggedDocuments(store, {
7850
+ flagsOf: (handles) => handles.removed,
7851
+ chunksOf: null,
7852
+ countWord: STORE_REMOVED_COUNT
7853
+ });
7854
+ }
7855
+
7770
7856
  // src/vector/vector-index/maintenance.ts
7771
7857
  function compact(state) {
7772
7858
  if (state.tombstones.size === 0 && state.store.replacedVectorCount === 0) return;
@@ -7964,6 +8050,43 @@ function createHeap(compare) {
7964
8050
  };
7965
8051
  }
7966
8052
 
8053
+ // src/vector/vector-index/merge-results.ts
8054
+ function highScoreFirst(a, b) {
8055
+ return b.score - a.score || compareCodePoints(a.docId, b.docId);
8056
+ }
8057
+ function mergeResults(hnswResults, bufferResults, k) {
8058
+ const seen = /* @__PURE__ */ new Set();
8059
+ const merged = [];
8060
+ let hi = 0;
8061
+ let bi = 0;
8062
+ while (merged.length < k && (hi < hnswResults.length || bi < bufferResults.length)) {
8063
+ const h = hi < hnswResults.length ? hnswResults[hi] : void 0;
8064
+ const b = bi < bufferResults.length ? bufferResults[bi] : void 0;
8065
+ let pick;
8066
+ if (h && b) {
8067
+ if (h.score > b.score || h.score === b.score && compareCodePoints(h.docId, b.docId) < 0) {
8068
+ pick = h;
8069
+ hi++;
8070
+ } else {
8071
+ pick = b;
8072
+ bi++;
8073
+ }
8074
+ } else if (h) {
8075
+ pick = h;
8076
+ hi++;
8077
+ } else if (b) {
8078
+ pick = b;
8079
+ bi++;
8080
+ } else {
8081
+ break;
8082
+ }
8083
+ if (seen.has(pick.docId)) continue;
8084
+ seen.add(pick.docId);
8085
+ merged.push(pick);
8086
+ }
8087
+ return merged;
8088
+ }
8089
+
7967
8090
  // src/vector/vector-index/search.ts
7968
8091
  function* bufferCandidates(state, filter) {
7969
8092
  for (const docId of state.buffer) {
@@ -7993,9 +8116,6 @@ function fetchedOutcome(state, results, minSimilarity, filter) {
7993
8116
  function scannedOutcome(scan2) {
7994
8117
  return { results: scan2.results, matched: scan2.matched, matchedExact: true };
7995
8118
  }
7996
- function highScoreFirst(a, b) {
7997
- return b.score - a.score || compareCodePoints(a.docId, b.docId);
7998
- }
7999
8119
  function liveCandidates(state, candidates) {
8000
8120
  const docIds = [];
8001
8121
  const ordinals = [];
@@ -8061,38 +8181,6 @@ function bruteForceSearch(state, query, k, metric, minSimilarity, candidates) {
8061
8181
  }
8062
8182
  return { results: heap.toSortedArray().reverse(), matched };
8063
8183
  }
8064
- function mergeResults(hnswResults, bufferResults, k) {
8065
- const seen = /* @__PURE__ */ new Set();
8066
- const merged = [];
8067
- let hi = 0;
8068
- let bi = 0;
8069
- while (merged.length < k && (hi < hnswResults.length || bi < bufferResults.length)) {
8070
- const h = hi < hnswResults.length ? hnswResults[hi] : void 0;
8071
- const b = bi < bufferResults.length ? bufferResults[bi] : void 0;
8072
- let pick;
8073
- if (h && b) {
8074
- if (h.score > b.score || h.score === b.score && compareCodePoints(h.docId, b.docId) < 0) {
8075
- pick = h;
8076
- hi++;
8077
- } else {
8078
- pick = b;
8079
- bi++;
8080
- }
8081
- } else if (h) {
8082
- pick = h;
8083
- hi++;
8084
- } else if (b) {
8085
- pick = b;
8086
- bi++;
8087
- } else {
8088
- break;
8089
- }
8090
- if (seen.has(pick.docId)) continue;
8091
- seen.add(pick.docId);
8092
- merged.push(pick);
8093
- }
8094
- return merged;
8095
- }
8096
8184
  function search(state, query, k, options) {
8097
8185
  return searchWithFilter(state, query, k, options, filterForOptions(state, options));
8098
8186
  }
@@ -8178,8 +8266,8 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
8178
8266
  hnswConfig,
8179
8267
  workerCopies,
8180
8268
  store,
8181
- tombstones: /* @__PURE__ */ new Set(),
8182
- buffer: /* @__PURE__ */ new Set(),
8269
+ tombstones: createRemovedDocuments(store),
8270
+ buffer: createUnplacedVectors(store),
8183
8271
  pendingLocations: /* @__PURE__ */ new Map(),
8184
8272
  savedFiles: [],
8185
8273
  savedSignature: null,
@@ -10563,56 +10651,94 @@ function runSegmentAction(entry, action) {
10563
10651
  }
10564
10652
  }
10565
10653
 
10566
- // src/vector/shared-field/searcher.ts
10567
- function bestFirst(a, b) {
10568
- return b.score - a.score || (a.docId < b.docId ? -1 : a.docId > b.docId ? 1 : 0);
10654
+ // src/vector/shared-field/store-flags.ts
10655
+ function bytesOf(flags, source) {
10656
+ if (flags.source !== source) {
10657
+ flags.source = source;
10658
+ flags.view = new Uint8Array(source);
10659
+ }
10660
+ return flags.view;
10661
+ }
10662
+ function countsOf(chunks, source) {
10663
+ if (chunks.source !== source) {
10664
+ chunks.source = source;
10665
+ chunks.view = new Int32Array(source);
10666
+ }
10667
+ return chunks.view;
10569
10668
  }
10570
- function distancesInTypeScript(state, query, metric, ordinals) {
10669
+ function readStoreFlags(store) {
10670
+ const unplaced = { source: null, view: new Uint8Array(0) };
10671
+ const unplacedChunks = { source: null, view: new Int32Array(0) };
10672
+ const removed = { source: null, view: new Uint8Array(0) };
10673
+ return {
10674
+ liveDocuments() {
10675
+ const header = store.handles.header;
10676
+ return Atomics.load(header, STORE_LIVE_COUNT) - Atomics.load(header, STORE_REMOVED_COUNT);
10677
+ },
10678
+ isRemoved(ordinal) {
10679
+ const bytes = bytesOf(removed, store.handles.removed);
10680
+ return ordinal < bytes.length && Atomics.load(bytes, ordinal) === 1;
10681
+ },
10682
+ unplacedOrdinals() {
10683
+ const handles = store.handles;
10684
+ if (Atomics.load(handles.header, STORE_UNPLACED_COUNT) === 0) return [];
10685
+ const bytes = bytesOf(unplaced, handles.unplaced);
10686
+ const counts = countsOf(unplacedChunks, handles.unplacedChunks);
10687
+ const end = Math.min(store.slots, bytes.length);
10688
+ const ordinals = [];
10689
+ for (let chunk = 0; chunk < counts.length; chunk++) {
10690
+ const from = chunk * UNPLACED_FLAG_CHUNK_ORDINALS;
10691
+ if (from >= end) break;
10692
+ if (Atomics.load(counts, chunk) === 0) continue;
10693
+ const to = Math.min(end, from + UNPLACED_FLAG_CHUNK_ORDINALS);
10694
+ for (let ordinal = from; ordinal < to; ordinal++) {
10695
+ if (Atomics.load(bytes, ordinal) === 1) ordinals.push(ordinal);
10696
+ }
10697
+ }
10698
+ return ordinals;
10699
+ }
10700
+ };
10701
+ }
10702
+
10703
+ // src/vector/shared-field/searcher.ts
10704
+ function distancesInTypeScript(store, query, metric, ordinals) {
10571
10705
  noteFallback("score");
10572
- const arenaQuery = state.store.prepareQueryArena(query);
10706
+ const arenaQuery = store.prepareQueryArena(query);
10573
10707
  const queryMagnitude = arenaQuery ? arenaQuery.magnitude : magnitude(query);
10574
10708
  const distances = new Float64Array(ordinals.length).fill(Number.POSITIVE_INFINITY);
10575
10709
  for (let i = 0; i < ordinals.length; i++) {
10576
10710
  if (arenaQuery) {
10577
- distances[i] = state.store.distanceFromArena(arenaQuery, ordinals[i], metric);
10711
+ distances[i] = store.distanceFromArena(arenaQuery, ordinals[i], metric);
10578
10712
  continue;
10579
10713
  }
10580
- const entry = entryForOrd(state, ordinals[i]);
10714
+ const entry = store.entryForOrdinal(ordinals[i]);
10581
10715
  if (entry) distances[i] = toDistance(query, entry.vector, queryMagnitude, entry.magnitude, metric);
10582
10716
  }
10583
10717
  return distances;
10584
10718
  }
10585
- function bruteForceOrdinals(state, docIdOf, query, k, metric, minSimilarity, filter) {
10586
- const ordinals = [];
10587
- const docIds = [];
10588
- for (const ord of ordinalFilterValues(filter)) {
10589
- if (isTombstoned(state, ord)) continue;
10590
- const docId = docIdOf(ord);
10591
- if (docId === void 0) continue;
10592
- ordinals.push(ord);
10593
- docIds.push(docId);
10594
- }
10595
- const distances = nativeScoresOf(state.store.handles, query, metric, ordinals) ?? distancesInTypeScript(state, query, metric, ordinals);
10596
- const heap = createBoundedMaxHeap(bestFirst, k);
10719
+ function scoreCandidates(store, candidates, query, k, metric, minSimilarity) {
10720
+ const { ordinals, docIds } = candidates;
10721
+ if (ordinals.length === 0) return { results: [], matched: 0 };
10722
+ const distances = nativeScoresOf(store.handles, query, metric, ordinals) ?? distancesInTypeScript(store, query, metric, ordinals);
10723
+ const heap = createBoundedMaxHeap(highScoreFirst, k);
10597
10724
  let matched = 0;
10598
10725
  for (let i = 0; i < ordinals.length; i++) {
10599
10726
  if (distances[i] === Number.POSITIVE_INFINITY) continue;
10600
10727
  const score = toScore(distances[i], metric);
10601
10728
  if (score < minSimilarity) continue;
10602
10729
  matched++;
10603
- heap.push({ ord: ordinals[i], score, docId: docIds[i] });
10730
+ heap.push({ docId: docIds[i], score });
10604
10731
  }
10605
- return { hits: heap.toSortedArray().reverse(), matched, matchedExact: true };
10732
+ return { results: heap.toSortedArray().reverse(), matched };
10606
10733
  }
10607
10734
  function createSharedVectorSearcher(options) {
10608
10735
  const { fieldName, view, holdsDocument } = options;
10736
+ const flags = readStoreFlags(view.store);
10609
10737
  function filterForDocIds(allowed) {
10610
10738
  const filter = createOrdinalFilter(view.store.slots);
10611
- const graph = view.graph;
10612
10739
  for (const docId of allowed) {
10613
10740
  const ordinal = view.ordinalOf(docId);
10614
- if (ordinal === void 0) continue;
10615
- if (graph !== null && isTombstoned(graph, ordinal)) continue;
10741
+ if (ordinal === void 0 || flags.isRemoved(ordinal)) continue;
10616
10742
  addToOrdinalFilter(filter, ordinal);
10617
10743
  }
10618
10744
  return filter;
@@ -10622,9 +10748,9 @@ function createSharedVectorSearcher(options) {
10622
10748
  const filter = createOrdinalFilter(slots);
10623
10749
  for (let ordinal = 0; ordinal < slots; ordinal++) {
10624
10750
  const partition = view.store.partitionAt(ordinal);
10625
- if (partition !== void 0 && allowed.has(partition) && view.store.holdsOrdinal(ordinal)) {
10626
- addToOrdinalFilter(filter, ordinal);
10627
- }
10751
+ if (partition === void 0 || !allowed.has(partition)) continue;
10752
+ if (!view.store.holdsOrdinal(ordinal) || flags.isRemoved(ordinal)) continue;
10753
+ addToOrdinalFilter(filter, ordinal);
10628
10754
  }
10629
10755
  return filter;
10630
10756
  }
@@ -10633,18 +10759,62 @@ function createSharedVectorSearcher(options) {
10633
10759
  if (searchOptions.filterPartitions !== void 0) return filterForPartitions(searchOptions.filterPartitions);
10634
10760
  return void 0;
10635
10761
  }
10636
- function hitsFor(query, k, searchOptions, filter) {
10762
+ function candidatesOf(ordinals, filter) {
10763
+ const candidates = { ordinals: [], docIds: [] };
10764
+ for (const ordinal of ordinals) {
10765
+ if (filter !== void 0 && !ordinalFilterHas(filter, ordinal)) continue;
10766
+ if (flags.isRemoved(ordinal)) continue;
10767
+ const docId = view.docIdOf(ordinal);
10768
+ if (docId === void 0 || view.ordinalOf(docId) !== ordinal) continue;
10769
+ candidates.ordinals.push(ordinal);
10770
+ candidates.docIds.push(docId);
10771
+ }
10772
+ return candidates;
10773
+ }
10774
+ function scanned(scan2) {
10775
+ return { results: scan2.results, matched: scan2.matched, matchedExact: true, resolvedEveryHit: true };
10776
+ }
10777
+ function search2(query, k, searchOptions, filter) {
10637
10778
  const { metric, minSimilarity, efSearch, oversample } = searchOptions;
10638
- const liveSize2 = view.liveSize;
10779
+ const empty = { results: [], matched: 0, matchedExact: true, resolvedEveryHit: true };
10780
+ const liveDocuments = flags.liveDocuments();
10781
+ if (liveDocuments <= 0) return empty;
10782
+ if (filter !== void 0 && filter.count === 0) return empty;
10783
+ function fetched(results, resolvedEveryHit2) {
10784
+ if (minSimilarity === Number.NEGATIVE_INFINITY) {
10785
+ const matched = filter === void 0 ? liveDocuments : filter.count;
10786
+ return { results, matched, matchedExact: true, resolvedEveryHit: resolvedEveryHit2 };
10787
+ }
10788
+ return { results, matched: results.length, matchedExact: false, resolvedEveryHit: resolvedEveryHit2 };
10789
+ }
10790
+ if (k <= 0) return fetched([], true);
10639
10791
  const graph = view.graph;
10640
- if (graph !== null && filter !== void 0 && liveSize2 > 0 && filter.count / liveSize2 < view.handles.filterThreshold) {
10641
- return bruteForceOrdinals(graph, view.docIdOf, query, k, metric, minSimilarity, filter);
10792
+ if (graph === null) {
10793
+ const ordinals = filter === void 0 ? flags.unplacedOrdinals() : ordinalFilterValues(filter);
10794
+ return scanned(scoreCandidates(view.store, candidatesOf(ordinals, filter), query, k, metric, minSimilarity));
10642
10795
  }
10643
- const hits = view.searchOrdinals(query, k, metric, minSimilarity, { filter, efSearch, oversample });
10644
- if (minSimilarity === Number.NEGATIVE_INFINITY) {
10645
- return { hits, matched: filter === void 0 ? liveSize2 : filter.count, matchedExact: true };
10796
+ if (filter !== void 0) {
10797
+ const graphLiveSize = view.liveSize;
10798
+ const selectivity = graphLiveSize > 0 ? filter.count / graphLiveSize : 1;
10799
+ if (selectivity < view.handles.filterThreshold) {
10800
+ const candidates = candidatesOf(ordinalFilterValues(filter), filter);
10801
+ return scanned(scoreCandidates(view.store, candidates, query, k, metric, minSimilarity));
10802
+ }
10803
+ }
10804
+ let resolvedEveryHit = true;
10805
+ const graphResults = [];
10806
+ for (const hit of view.searchOrdinals(query, k, metric, minSimilarity, { filter, efSearch, oversample })) {
10807
+ const docId = view.docIdOf(hit.ord);
10808
+ if (docId === void 0) {
10809
+ resolvedEveryHit = false;
10810
+ continue;
10811
+ }
10812
+ graphResults.push({ docId, score: hit.score });
10646
10813
  }
10647
- return { hits, matched: hits.length, matchedExact: false };
10814
+ const unplaced = flags.unplacedOrdinals();
10815
+ if (unplaced.length === 0) return fetched(graphResults, resolvedEveryHit);
10816
+ const scan2 = scoreCandidates(view.store, candidatesOf(unplaced, filter), query, k, metric, minSimilarity);
10817
+ return fetched(mergeResults(graphResults, scan2.results, k), resolvedEveryHit);
10648
10818
  }
10649
10819
  return {
10650
10820
  fieldName,
@@ -10659,22 +10829,20 @@ function createSharedVectorSearcher(options) {
10659
10829
  { expected: view.handles.dimension, received: query.length }
10660
10830
  );
10661
10831
  }
10662
- const filter = filterFor(searchOptions);
10663
- if (filter !== void 0 && filter.count === 0) {
10664
- return Promise.resolve({ results: [], matched: 0, matchedExact: true });
10665
- }
10666
- const scanned = hitsFor(query, k, searchOptions, filter);
10667
- const results = [];
10668
- for (const hit of scanned.hits) {
10669
- const docId = view.docIdOf(hit.ord);
10670
- if (docId === void 0 || !holdsDocument(docId)) continue;
10671
- results.push({ docId, score: hit.score });
10832
+ const outcome = search2(query, k, searchOptions, filterFor(searchOptions));
10833
+ if (view.graphRetired) {
10834
+ throw new NarsilError(
10835
+ ErrorCodes.INDEX_NOT_FOUND,
10836
+ `The main thread replaced the graph of vector field "${fieldName}" while this thread searched it`,
10837
+ { fieldName }
10838
+ );
10672
10839
  }
10673
- const holdsEveryHit = results.length === scanned.hits.length;
10840
+ const results = outcome.results.filter((result) => holdsDocument(result.docId));
10841
+ const holdsEveryHit = outcome.resolvedEveryHit && results.length === outcome.results.length;
10674
10842
  return Promise.resolve({
10675
10843
  results,
10676
- matched: scanned.matched,
10677
- matchedExact: scanned.matchedExact && holdsEveryHit
10844
+ matched: outcome.matched,
10845
+ matchedExact: outcome.matchedExact && holdsEveryHit
10678
10846
  });
10679
10847
  }
10680
10848
  };
@@ -10709,7 +10877,8 @@ function loadHeldVectorCopy(held, manager, fieldName, handle, handles, threadSlo
10709
10877
  );
10710
10878
  }
10711
10879
  function holdsVectorField(held, fieldName) {
10712
- return held.fields.get(fieldName)?.view.readsEveryVector === true;
10880
+ const view = held.fields.get(fieldName)?.view;
10881
+ return view?.readsEveryVector === true && !view.graphRetired;
10713
10882
  }
10714
10883
  function dropHeldVectorCopy(held, fieldName, handle) {
10715
10884
  held.builds.get(handle)?.view.close();
@@ -10780,6 +10949,9 @@ function createDirectExecutor(options) {
10780
10949
  return vectorIndexes;
10781
10950
  }
10782
10951
  function createIndex(indexName, config, language, analysisStale = false) {
10952
+ addIndex(indexName, config, language, analysisStale, null);
10953
+ }
10954
+ function addIndex(indexName, config, language, analysisStale, awaitedCopyLoad) {
10783
10955
  if (indexes.has(indexName)) {
10784
10956
  throw new NarsilError(ErrorCodes.INDEX_ALREADY_EXISTS, `Index "${indexName}" already exists`, {
10785
10957
  indexName
@@ -10800,7 +10972,8 @@ function createDirectExecutor(options) {
10800
10972
  },
10801
10973
  vectorIndexes,
10802
10974
  vectorCopies: createHeldVectorCopies(),
10803
- analysisStale
10975
+ analysisStale,
10976
+ awaitedCopyLoad
10804
10977
  });
10805
10978
  }
10806
10979
  function dropIndex(indexName) {
@@ -10822,7 +10995,7 @@ function createDirectExecutor(options) {
10822
10995
  }
10823
10996
  function queryContextOf(indexName) {
10824
10997
  const entry = indexes.get(indexName);
10825
- if (entry === void 0) return void 0;
10998
+ if (entry === void 0 || entry.awaitedCopyLoad !== null) return void 0;
10826
10999
  return {
10827
11000
  manager: entry.manager,
10828
11001
  config: entry.config,
@@ -10852,7 +11025,12 @@ function createDirectExecutor(options) {
10852
11025
  }
10853
11026
  case "createIndex": {
10854
11027
  const language = getLanguage(action.config.language ?? "english");
10855
- createIndex(action.indexName, action.config, language, action.analysisStale ?? false);
11028
+ addIndex(action.indexName, action.config, language, action.analysisStale ?? false, action.copyLoad ?? null);
11029
+ return void 0;
11030
+ }
11031
+ case "copyLoaded": {
11032
+ const entry = indexes.get(action.indexName);
11033
+ if (entry?.awaitedCopyLoad === action.copyLoad) entry.awaitedCopyLoad = null;
10856
11034
  return void 0;
10857
11035
  }
10858
11036
  case "dropIndex": {
@@ -9,6 +9,7 @@ var KNOWN_ACTION_TYPES = [
9
9
  "has",
10
10
  "count",
11
11
  "createIndex",
12
+ "copyLoaded",
12
13
  "loadVectorCopy",
13
14
  "dropVectorCopy",
14
15
  "insertVectorOrdinals",
@@ -1,11 +1,11 @@
1
- export { createDirectExecutor } from './chunk-IA67SOJO.mjs';
1
+ export { createDirectExecutor } from './chunk-XPK3GB2P.mjs';
2
2
  import './chunk-XYDP7CCJ.mjs';
3
3
  import './chunk-OLXM65YH.mjs';
4
4
  import './chunk-ZCKPRCYI.mjs';
5
5
  import './chunk-TGP5M5R7.mjs';
6
- import './chunk-RZXNWQSM.mjs';
6
+ import './chunk-B3F7LZXM.mjs';
7
7
  import './chunk-ZJ4CSAB2.mjs';
8
- import './chunk-B3UOQJD3.mjs';
8
+ import './chunk-NFF5TDRG.mjs';
9
9
  import './chunk-SREPGC6O.mjs';
10
10
  import './chunk-WQBCBKY3.mjs';
11
11
  import './chunk-4VBG6PEH.mjs';
@@ -1,17 +1,17 @@
1
1
  import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
2
2
  import { validateIndexName, indexConfigKey, withPartitionCount, indexSettingsOf, putIndexMetadata, getIndexMetadata, getClusterIndexConfig } from './chunk-GNYXCLJS.mjs';
3
- import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, encodeIndexConfiguration, buildArchiveFromMemory, indexMetadataOf, archiveStream, applyIndexEntry, applyDeleteEntry, requireConfigurationWithinSizeLimit, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, memoryCheckpointStore, readSnapshotArchive, createByteReader, openChecksummedEnvelope, loadPartitionSegments, readListedFile, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, crc32Init, crc32Final, crc32Update, computeEntryChecksum, createEntryMessage, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-XIQZKKTK.mjs';
3
+ import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, encodeIndexConfiguration, buildArchiveFromMemory, indexMetadataOf, archiveStream, applyIndexEntry, applyDeleteEntry, requireConfigurationWithinSizeLimit, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, memoryCheckpointStore, readSnapshotArchive, createByteReader, openChecksummedEnvelope, loadPartitionSegments, readListedFile, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, crc32Init, crc32Final, crc32Update, computeEntryChecksum, createEntryMessage, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-IDLP54FH.mjs';
4
4
  import { now, listBindingOf, requireSurfaceFormsForPartialFields, queryBindingOf, hitsKeptPerGroup, clampLimit, resolveHybridFusion, dedupePinnedEntries, foldGroupReducer } from './chunk-4D6OH4EF.mjs';
5
5
  import { ReplicationMessageTypes, TransportErrorCodes, ClusterMessageTypes, QueryMessageTypes, TransportError } from './chunk-GCM3JFVA.mjs';
6
6
  import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET, providedDocId } from './chunk-2453RJYK.mjs';
7
- import { generateId, readHeapStatistics, decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-IA67SOJO.mjs';
7
+ import { generateId, readHeapStatistics, decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-XPK3GB2P.mjs';
8
8
  import { oversampledShardSize, resolveProjection, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, normalizeSort, sortModesOf, encodePageCursor, applyProjection, readSortValues, requireWithinResultWindow, fnv1a, sortSignatureEntry, FILTER_EXPRESSION_KEYS, filterTestCount, fnv1aUint32s, mergePartitionStats, FIELD_FILTER_OPERATORS } from './chunk-XYDP7CCJ.mjs';
9
9
  import { getLanguage } from './chunk-OLXM65YH.mjs';
10
10
  import { DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_INDEX_NAME_LENGTH, MAX_DOC_ID_LENGTH, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, INDEX_NAME_PATTERN } from './chunk-ZCKPRCYI.mjs';
11
11
  import './chunk-TGP5M5R7.mjs';
12
- import './chunk-RZXNWQSM.mjs';
12
+ import './chunk-B3F7LZXM.mjs';
13
13
  import { MIN_OVERSAMPLE } from './chunk-ZJ4CSAB2.mjs';
14
- import './chunk-B3UOQJD3.mjs';
14
+ import './chunk-NFF5TDRG.mjs';
15
15
  import { isPatternTolerance } from './chunk-SREPGC6O.mjs';
16
16
  import { MAX_FILTER_DEPTH, MAX_FILTER_TESTS, MAX_FILTER_ARRAY_SIZE } from './chunk-WQBCBKY3.mjs';
17
17
  import './chunk-4VBG6PEH.mjs';
@@ -23,7 +23,7 @@ import './chunk-U2BKYEC7.mjs';
23
23
  import { toComparableSortValue, compareComparableKeys, isSortMode, SORT_MODES, compareSortValues } from './chunk-R6A773WK.mjs';
24
24
  import { compareCodePoints, exceedsCodePoints } from './chunk-YTJCGKM4.mjs';
25
25
  import { collectAndReturnPages } from './chunk-5XNIRLFC.mjs';
26
- import './chunk-POWOTUKJ.mjs';
26
+ import './chunk-YRX6XMOQ.mjs';
27
27
  import { ErrorCodes, NarsilError, describeError } from './chunk-TPJU7NDJ.mjs';
28
28
  import { decode, encode } from '@msgpack/msgpack';
29
29
  import { randomUUID } from 'crypto';