@delali/narsil 0.3.1-next.84 → 0.3.1-next.85
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-XIQZKKTK.mjs → chunk-67MQF5CV.mjs} +39 -10
- package/dist/{chunk-B3UOQJD3.mjs → chunk-NFF5TDRG.mjs} +1 -1
- package/dist/{chunk-RZXNWQSM.mjs → chunk-NIWDQTZB.mjs} +591 -193
- package/dist/{chunk-IA67SOJO.mjs → chunk-RCSXJCI5.mjs} +271 -93
- package/dist/{chunk-XZF4LZ3X.mjs → chunk-RM6FRYEC.mjs} +31 -6
- package/dist/{chunk-POWOTUKJ.mjs → chunk-YRX6XMOQ.mjs} +1 -0
- package/dist/{direct-executor-6WWFY5EI.mjs → direct-executor-NY7ZTZIH.mjs} +3 -3
- package/dist/distribution.mjs +5 -5
- package/dist/index.browser.mjs +763 -161
- package/dist/index.mjs +5 -5
- package/dist/persistence/durability/checkpoint-worker.mjs +5 -5
- package/dist/server.mjs +31 -16
- package/dist/{thread-33M5AOEI.mjs → thread-EUUNJNKU.mjs} +226 -41
- package/dist/vector/search-worker.d.ts +8 -0
- package/dist/vector/search-worker.mjs +2 -2
- package/dist/workers/entry.mjs +4 -4
- package/package.json +7 -7
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { createGrowingPartialWords, requirePatternSearchFor, requirePartialWordSearchFor, buildPartialWordTableFrom, patternIndexOf, encodePartialSpellingsOf, countPartialSpelling, joinPatternWorkMeter, countsWithoutScores, fanOutMatchCount, fanOutQuery, sanitizeGlobalStats, expandPartialWords, computeFacets, cloneProjected, projectionKeepsField, setNestedValue, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, PATTERN_HOST, validatedFilterOf, everyValueFacetConfig, mergeFacets, kWayMerge, polygonContainment, polygonBoxes, radiusBoxes, locationCode, undeclaredFilterField, coveringRanges } from './chunk-XYDP7CCJ.mjs';
|
|
2
2
|
import { getLanguage, detectRuntime } from './chunk-OLXM65YH.mjs';
|
|
3
3
|
import { MAX_DOC_ID_LENGTH } from './chunk-ZCKPRCYI.mjs';
|
|
4
|
-
import { SCRATCH_SLOTS_PER_THREAD_POOL, osqRecordBytes, openSharedVectorField, osqBitsOf, createVectorStore, DISK_STORAGE_BLOCK_BYTES, createOsqQuantizer,
|
|
5
|
-
import { noteFallback } from './chunk-
|
|
4
|
+
import { SCRATCH_SLOTS_PER_THREAD_POOL, osqRecordBytes, openSharedVectorField, osqBitsOf, createVectorStore, DISK_STORAGE_BLOCK_BYTES, createOsqQuantizer, STORE_UNPLACED_COUNT, STORE_REMOVED_COUNT, STORE_LIVE_COUNT, ordinalFilterValues, UNPLACED_FLAG_CHUNK_ORDINALS, magnitude, graphWithoutNodes, createHNSWIndex, nativeStoreScratchBytes, nativeScoresOf, toScore, ordinalFilterHas, createOrdinalFilter, addToOrdinalFilter, removeFromOrdinalFilter, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, topLayerInParts, STORE_CALIBRATION_GENERATION, toDistance, sharedMemoryAvailable, requireWholeNeighbourLists, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, releaseLocksHeldBy, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS } from './chunk-NIWDQTZB.mjs';
|
|
5
|
+
import { noteFallback } from './chunk-NFF5TDRG.mjs';
|
|
6
6
|
import { GrowableUint32, bitsetHas, createBitSet, bitsetSet, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, bitsetFromSet, bitsetIsEmpty } from './chunk-SREPGC6O.mjs';
|
|
7
7
|
import { GEO_UNIT_METRES } from './chunk-WQBCBKY3.mjs';
|
|
8
8
|
import { tokenizeOptions, getNestedValue, visitTokens, expandTermPrefix, suggestDisplayTerms, textFieldsChanged, sameFields, codePointOrder, encodeStringBlob, encodeLookupKey, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, getFlatSchema, tokenize, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, getOrCreateFieldNameIndex, getFieldValueByInternalId, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT, partitionDocumentFrequency } from './chunk-4VBG6PEH.mjs';
|
|
@@ -6066,6 +6066,7 @@ function releaseReplacedVectors(state) {
|
|
|
6066
6066
|
for (const ordinal of state.store.releaseReplacedVectors()) state.osq?.removeOrdinal(ordinal);
|
|
6067
6067
|
}
|
|
6068
6068
|
function adoptGraph(state, graph) {
|
|
6069
|
+
if (state.hnsw !== null && state.hnsw !== graph) state.hnsw.retire();
|
|
6069
6070
|
if (graph === null && state.hnsw !== null) {
|
|
6070
6071
|
state.hnsw.clear();
|
|
6071
6072
|
}
|
|
@@ -6807,7 +6808,7 @@ async function placeChunk(state, graph, dispatch, chunkDocIds, chunkOrdinals, fa
|
|
|
6807
6808
|
continue;
|
|
6808
6809
|
}
|
|
6809
6810
|
if (!state.tombstones.has(docId) && !graph.has(docId) && graph.insertOrdinal(ordinal)) placedHere += 1;
|
|
6810
|
-
state.buffer.delete(docId);
|
|
6811
|
+
if (graph === state.hnsw) state.buffer.delete(docId);
|
|
6811
6812
|
}
|
|
6812
6813
|
if (placedHere > 0 && outcome !== null && !fallback.shortAnswerWarned) {
|
|
6813
6814
|
fallback.shortAnswerWarned = true;
|
|
@@ -6824,7 +6825,7 @@ async function insertLocally(state, graph, docIds, admit) {
|
|
|
6824
6825
|
const ordinal = state.store.getOrdinal(docId);
|
|
6825
6826
|
if (ordinal === void 0) continue;
|
|
6826
6827
|
graph.insertOrdinal(ordinal);
|
|
6827
|
-
state.buffer.delete(docId);
|
|
6828
|
+
if (graph === state.hnsw) state.buffer.delete(docId);
|
|
6828
6829
|
count += 1;
|
|
6829
6830
|
if (count % BUILD_CHUNK_SIZE === 0) {
|
|
6830
6831
|
noteWrite(state);
|
|
@@ -6941,6 +6942,10 @@ async function buildGraphFromStore(state) {
|
|
|
6941
6942
|
}
|
|
6942
6943
|
const previous = state.hnsw;
|
|
6943
6944
|
adoptGraph(state, graph);
|
|
6945
|
+
for (const docId of [...state.buffer]) {
|
|
6946
|
+
const ordinal = state.store.getOrdinal(docId);
|
|
6947
|
+
if (ordinal !== void 0 && graph.holdsNode(ordinal) && !graph.holdsTombstone(ordinal)) state.buffer.delete(docId);
|
|
6948
|
+
}
|
|
6944
6949
|
if (previous !== null && previous !== graph) dropSharedGraph(state, previous);
|
|
6945
6950
|
await releasePendingLocations(state);
|
|
6946
6951
|
}
|
|
@@ -7767,6 +7772,87 @@ function beginCheckpointRestore(state, shape) {
|
|
|
7767
7772
|
return { addFile, finish };
|
|
7768
7773
|
}
|
|
7769
7774
|
|
|
7775
|
+
// src/vector/vector-index/flagged-documents.ts
|
|
7776
|
+
var FlaggedDocuments = class extends Set {
|
|
7777
|
+
constructor(store, layout) {
|
|
7778
|
+
super();
|
|
7779
|
+
this.store = store;
|
|
7780
|
+
this.layout = layout;
|
|
7781
|
+
}
|
|
7782
|
+
ordinals = /* @__PURE__ */ new Map();
|
|
7783
|
+
handles = null;
|
|
7784
|
+
flags = new Uint8Array(0);
|
|
7785
|
+
chunks = null;
|
|
7786
|
+
add(docId) {
|
|
7787
|
+
super.add(docId);
|
|
7788
|
+
const handles = this.openHandles();
|
|
7789
|
+
if (handles === null) return this;
|
|
7790
|
+
this.flagCurrentOrdinal(docId);
|
|
7791
|
+
Atomics.store(handles.header, this.layout.countWord, this.size);
|
|
7792
|
+
return this;
|
|
7793
|
+
}
|
|
7794
|
+
delete(docId) {
|
|
7795
|
+
const held = super.delete(docId);
|
|
7796
|
+
const handles = this.openHandles();
|
|
7797
|
+
if (handles === null) return held;
|
|
7798
|
+
const ordinal = this.ordinals.get(docId);
|
|
7799
|
+
if (ordinal !== void 0) {
|
|
7800
|
+
this.ordinals.delete(docId);
|
|
7801
|
+
this.unflag(ordinal);
|
|
7802
|
+
}
|
|
7803
|
+
Atomics.store(handles.header, this.layout.countWord, this.size);
|
|
7804
|
+
return held;
|
|
7805
|
+
}
|
|
7806
|
+
clear() {
|
|
7807
|
+
super.clear();
|
|
7808
|
+
const handles = this.openHandles();
|
|
7809
|
+
if (handles !== null) {
|
|
7810
|
+
for (const ordinal of this.ordinals.values()) this.unflag(ordinal);
|
|
7811
|
+
Atomics.store(handles.header, this.layout.countWord, 0);
|
|
7812
|
+
}
|
|
7813
|
+
this.ordinals.clear();
|
|
7814
|
+
}
|
|
7815
|
+
flagCurrentOrdinal(docId) {
|
|
7816
|
+
const ordinal = this.store.getOrdinal(docId);
|
|
7817
|
+
const previous = this.ordinals.get(docId);
|
|
7818
|
+
if (ordinal === void 0 || ordinal === previous) return;
|
|
7819
|
+
if (this.chunks !== null) Atomics.add(this.chunks, Math.floor(ordinal / UNPLACED_FLAG_CHUNK_ORDINALS), 1);
|
|
7820
|
+
Atomics.store(this.flags, ordinal, 1);
|
|
7821
|
+
this.ordinals.set(docId, ordinal);
|
|
7822
|
+
if (previous !== void 0) this.unflag(previous);
|
|
7823
|
+
}
|
|
7824
|
+
unflag(ordinal) {
|
|
7825
|
+
Atomics.store(this.flags, ordinal, 0);
|
|
7826
|
+
if (this.chunks !== null) Atomics.sub(this.chunks, Math.floor(ordinal / UNPLACED_FLAG_CHUNK_ORDINALS), 1);
|
|
7827
|
+
}
|
|
7828
|
+
openHandles() {
|
|
7829
|
+
if (this.store.dimension === 0) return null;
|
|
7830
|
+
const handles = this.store.handles;
|
|
7831
|
+
if (handles === this.handles) return handles;
|
|
7832
|
+
this.handles = handles;
|
|
7833
|
+
this.flags = new Uint8Array(this.layout.flagsOf(handles));
|
|
7834
|
+
const chunksOf = this.layout.chunksOf;
|
|
7835
|
+
this.chunks = chunksOf === null ? null : new Int32Array(chunksOf(handles));
|
|
7836
|
+
this.ordinals.clear();
|
|
7837
|
+
for (const docId of this) this.flagCurrentOrdinal(docId);
|
|
7838
|
+
return handles;
|
|
7839
|
+
}
|
|
7840
|
+
};
|
|
7841
|
+
function createUnplacedVectors(store) {
|
|
7842
|
+
return new FlaggedDocuments(store, {
|
|
7843
|
+
flagsOf: (handles) => handles.unplaced,
|
|
7844
|
+
chunksOf: (handles) => handles.unplacedChunks,
|
|
7845
|
+
countWord: STORE_UNPLACED_COUNT
|
|
7846
|
+
});
|
|
7847
|
+
}
|
|
7848
|
+
function createRemovedDocuments(store) {
|
|
7849
|
+
return new FlaggedDocuments(store, {
|
|
7850
|
+
flagsOf: (handles) => handles.removed,
|
|
7851
|
+
chunksOf: null,
|
|
7852
|
+
countWord: STORE_REMOVED_COUNT
|
|
7853
|
+
});
|
|
7854
|
+
}
|
|
7855
|
+
|
|
7770
7856
|
// src/vector/vector-index/maintenance.ts
|
|
7771
7857
|
function compact(state) {
|
|
7772
7858
|
if (state.tombstones.size === 0 && state.store.replacedVectorCount === 0) return;
|
|
@@ -7964,6 +8050,43 @@ function createHeap(compare) {
|
|
|
7964
8050
|
};
|
|
7965
8051
|
}
|
|
7966
8052
|
|
|
8053
|
+
// src/vector/vector-index/merge-results.ts
|
|
8054
|
+
function highScoreFirst(a, b) {
|
|
8055
|
+
return b.score - a.score || compareCodePoints(a.docId, b.docId);
|
|
8056
|
+
}
|
|
8057
|
+
function mergeResults(hnswResults, bufferResults, k) {
|
|
8058
|
+
const seen = /* @__PURE__ */ new Set();
|
|
8059
|
+
const merged = [];
|
|
8060
|
+
let hi = 0;
|
|
8061
|
+
let bi = 0;
|
|
8062
|
+
while (merged.length < k && (hi < hnswResults.length || bi < bufferResults.length)) {
|
|
8063
|
+
const h = hi < hnswResults.length ? hnswResults[hi] : void 0;
|
|
8064
|
+
const b = bi < bufferResults.length ? bufferResults[bi] : void 0;
|
|
8065
|
+
let pick;
|
|
8066
|
+
if (h && b) {
|
|
8067
|
+
if (h.score > b.score || h.score === b.score && compareCodePoints(h.docId, b.docId) < 0) {
|
|
8068
|
+
pick = h;
|
|
8069
|
+
hi++;
|
|
8070
|
+
} else {
|
|
8071
|
+
pick = b;
|
|
8072
|
+
bi++;
|
|
8073
|
+
}
|
|
8074
|
+
} else if (h) {
|
|
8075
|
+
pick = h;
|
|
8076
|
+
hi++;
|
|
8077
|
+
} else if (b) {
|
|
8078
|
+
pick = b;
|
|
8079
|
+
bi++;
|
|
8080
|
+
} else {
|
|
8081
|
+
break;
|
|
8082
|
+
}
|
|
8083
|
+
if (seen.has(pick.docId)) continue;
|
|
8084
|
+
seen.add(pick.docId);
|
|
8085
|
+
merged.push(pick);
|
|
8086
|
+
}
|
|
8087
|
+
return merged;
|
|
8088
|
+
}
|
|
8089
|
+
|
|
7967
8090
|
// src/vector/vector-index/search.ts
|
|
7968
8091
|
function* bufferCandidates(state, filter) {
|
|
7969
8092
|
for (const docId of state.buffer) {
|
|
@@ -7993,9 +8116,6 @@ function fetchedOutcome(state, results, minSimilarity, filter) {
|
|
|
7993
8116
|
function scannedOutcome(scan2) {
|
|
7994
8117
|
return { results: scan2.results, matched: scan2.matched, matchedExact: true };
|
|
7995
8118
|
}
|
|
7996
|
-
function highScoreFirst(a, b) {
|
|
7997
|
-
return b.score - a.score || compareCodePoints(a.docId, b.docId);
|
|
7998
|
-
}
|
|
7999
8119
|
function liveCandidates(state, candidates) {
|
|
8000
8120
|
const docIds = [];
|
|
8001
8121
|
const ordinals = [];
|
|
@@ -8061,38 +8181,6 @@ function bruteForceSearch(state, query, k, metric, minSimilarity, candidates) {
|
|
|
8061
8181
|
}
|
|
8062
8182
|
return { results: heap.toSortedArray().reverse(), matched };
|
|
8063
8183
|
}
|
|
8064
|
-
function mergeResults(hnswResults, bufferResults, k) {
|
|
8065
|
-
const seen = /* @__PURE__ */ new Set();
|
|
8066
|
-
const merged = [];
|
|
8067
|
-
let hi = 0;
|
|
8068
|
-
let bi = 0;
|
|
8069
|
-
while (merged.length < k && (hi < hnswResults.length || bi < bufferResults.length)) {
|
|
8070
|
-
const h = hi < hnswResults.length ? hnswResults[hi] : void 0;
|
|
8071
|
-
const b = bi < bufferResults.length ? bufferResults[bi] : void 0;
|
|
8072
|
-
let pick;
|
|
8073
|
-
if (h && b) {
|
|
8074
|
-
if (h.score > b.score || h.score === b.score && compareCodePoints(h.docId, b.docId) < 0) {
|
|
8075
|
-
pick = h;
|
|
8076
|
-
hi++;
|
|
8077
|
-
} else {
|
|
8078
|
-
pick = b;
|
|
8079
|
-
bi++;
|
|
8080
|
-
}
|
|
8081
|
-
} else if (h) {
|
|
8082
|
-
pick = h;
|
|
8083
|
-
hi++;
|
|
8084
|
-
} else if (b) {
|
|
8085
|
-
pick = b;
|
|
8086
|
-
bi++;
|
|
8087
|
-
} else {
|
|
8088
|
-
break;
|
|
8089
|
-
}
|
|
8090
|
-
if (seen.has(pick.docId)) continue;
|
|
8091
|
-
seen.add(pick.docId);
|
|
8092
|
-
merged.push(pick);
|
|
8093
|
-
}
|
|
8094
|
-
return merged;
|
|
8095
|
-
}
|
|
8096
8184
|
function search(state, query, k, options) {
|
|
8097
8185
|
return searchWithFilter(state, query, k, options, filterForOptions(state, options));
|
|
8098
8186
|
}
|
|
@@ -8178,8 +8266,8 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
8178
8266
|
hnswConfig,
|
|
8179
8267
|
workerCopies,
|
|
8180
8268
|
store,
|
|
8181
|
-
tombstones:
|
|
8182
|
-
buffer:
|
|
8269
|
+
tombstones: createRemovedDocuments(store),
|
|
8270
|
+
buffer: createUnplacedVectors(store),
|
|
8183
8271
|
pendingLocations: /* @__PURE__ */ new Map(),
|
|
8184
8272
|
savedFiles: [],
|
|
8185
8273
|
savedSignature: null,
|
|
@@ -10563,56 +10651,94 @@ function runSegmentAction(entry, action) {
|
|
|
10563
10651
|
}
|
|
10564
10652
|
}
|
|
10565
10653
|
|
|
10566
|
-
// src/vector/shared-field/
|
|
10567
|
-
function
|
|
10568
|
-
|
|
10654
|
+
// src/vector/shared-field/store-flags.ts
|
|
10655
|
+
function bytesOf(flags, source) {
|
|
10656
|
+
if (flags.source !== source) {
|
|
10657
|
+
flags.source = source;
|
|
10658
|
+
flags.view = new Uint8Array(source);
|
|
10659
|
+
}
|
|
10660
|
+
return flags.view;
|
|
10661
|
+
}
|
|
10662
|
+
function countsOf(chunks, source) {
|
|
10663
|
+
if (chunks.source !== source) {
|
|
10664
|
+
chunks.source = source;
|
|
10665
|
+
chunks.view = new Int32Array(source);
|
|
10666
|
+
}
|
|
10667
|
+
return chunks.view;
|
|
10569
10668
|
}
|
|
10570
|
-
function
|
|
10669
|
+
function readStoreFlags(store) {
|
|
10670
|
+
const unplaced = { source: null, view: new Uint8Array(0) };
|
|
10671
|
+
const unplacedChunks = { source: null, view: new Int32Array(0) };
|
|
10672
|
+
const removed = { source: null, view: new Uint8Array(0) };
|
|
10673
|
+
return {
|
|
10674
|
+
liveDocuments() {
|
|
10675
|
+
const header = store.handles.header;
|
|
10676
|
+
return Atomics.load(header, STORE_LIVE_COUNT) - Atomics.load(header, STORE_REMOVED_COUNT);
|
|
10677
|
+
},
|
|
10678
|
+
isRemoved(ordinal) {
|
|
10679
|
+
const bytes = bytesOf(removed, store.handles.removed);
|
|
10680
|
+
return ordinal < bytes.length && Atomics.load(bytes, ordinal) === 1;
|
|
10681
|
+
},
|
|
10682
|
+
unplacedOrdinals() {
|
|
10683
|
+
const handles = store.handles;
|
|
10684
|
+
if (Atomics.load(handles.header, STORE_UNPLACED_COUNT) === 0) return [];
|
|
10685
|
+
const bytes = bytesOf(unplaced, handles.unplaced);
|
|
10686
|
+
const counts = countsOf(unplacedChunks, handles.unplacedChunks);
|
|
10687
|
+
const end = Math.min(store.slots, bytes.length);
|
|
10688
|
+
const ordinals = [];
|
|
10689
|
+
for (let chunk = 0; chunk < counts.length; chunk++) {
|
|
10690
|
+
const from = chunk * UNPLACED_FLAG_CHUNK_ORDINALS;
|
|
10691
|
+
if (from >= end) break;
|
|
10692
|
+
if (Atomics.load(counts, chunk) === 0) continue;
|
|
10693
|
+
const to = Math.min(end, from + UNPLACED_FLAG_CHUNK_ORDINALS);
|
|
10694
|
+
for (let ordinal = from; ordinal < to; ordinal++) {
|
|
10695
|
+
if (Atomics.load(bytes, ordinal) === 1) ordinals.push(ordinal);
|
|
10696
|
+
}
|
|
10697
|
+
}
|
|
10698
|
+
return ordinals;
|
|
10699
|
+
}
|
|
10700
|
+
};
|
|
10701
|
+
}
|
|
10702
|
+
|
|
10703
|
+
// src/vector/shared-field/searcher.ts
|
|
10704
|
+
function distancesInTypeScript(store, query, metric, ordinals) {
|
|
10571
10705
|
noteFallback("score");
|
|
10572
|
-
const arenaQuery =
|
|
10706
|
+
const arenaQuery = store.prepareQueryArena(query);
|
|
10573
10707
|
const queryMagnitude = arenaQuery ? arenaQuery.magnitude : magnitude(query);
|
|
10574
10708
|
const distances = new Float64Array(ordinals.length).fill(Number.POSITIVE_INFINITY);
|
|
10575
10709
|
for (let i = 0; i < ordinals.length; i++) {
|
|
10576
10710
|
if (arenaQuery) {
|
|
10577
|
-
distances[i] =
|
|
10711
|
+
distances[i] = store.distanceFromArena(arenaQuery, ordinals[i], metric);
|
|
10578
10712
|
continue;
|
|
10579
10713
|
}
|
|
10580
|
-
const entry =
|
|
10714
|
+
const entry = store.entryForOrdinal(ordinals[i]);
|
|
10581
10715
|
if (entry) distances[i] = toDistance(query, entry.vector, queryMagnitude, entry.magnitude, metric);
|
|
10582
10716
|
}
|
|
10583
10717
|
return distances;
|
|
10584
10718
|
}
|
|
10585
|
-
function
|
|
10586
|
-
const ordinals =
|
|
10587
|
-
|
|
10588
|
-
|
|
10589
|
-
|
|
10590
|
-
const docId = docIdOf(ord);
|
|
10591
|
-
if (docId === void 0) continue;
|
|
10592
|
-
ordinals.push(ord);
|
|
10593
|
-
docIds.push(docId);
|
|
10594
|
-
}
|
|
10595
|
-
const distances = nativeScoresOf(state.store.handles, query, metric, ordinals) ?? distancesInTypeScript(state, query, metric, ordinals);
|
|
10596
|
-
const heap = createBoundedMaxHeap(bestFirst, k);
|
|
10719
|
+
function scoreCandidates(store, candidates, query, k, metric, minSimilarity) {
|
|
10720
|
+
const { ordinals, docIds } = candidates;
|
|
10721
|
+
if (ordinals.length === 0) return { results: [], matched: 0 };
|
|
10722
|
+
const distances = nativeScoresOf(store.handles, query, metric, ordinals) ?? distancesInTypeScript(store, query, metric, ordinals);
|
|
10723
|
+
const heap = createBoundedMaxHeap(highScoreFirst, k);
|
|
10597
10724
|
let matched = 0;
|
|
10598
10725
|
for (let i = 0; i < ordinals.length; i++) {
|
|
10599
10726
|
if (distances[i] === Number.POSITIVE_INFINITY) continue;
|
|
10600
10727
|
const score = toScore(distances[i], metric);
|
|
10601
10728
|
if (score < minSimilarity) continue;
|
|
10602
10729
|
matched++;
|
|
10603
|
-
heap.push({
|
|
10730
|
+
heap.push({ docId: docIds[i], score });
|
|
10604
10731
|
}
|
|
10605
|
-
return {
|
|
10732
|
+
return { results: heap.toSortedArray().reverse(), matched };
|
|
10606
10733
|
}
|
|
10607
10734
|
function createSharedVectorSearcher(options) {
|
|
10608
10735
|
const { fieldName, view, holdsDocument } = options;
|
|
10736
|
+
const flags = readStoreFlags(view.store);
|
|
10609
10737
|
function filterForDocIds(allowed) {
|
|
10610
10738
|
const filter = createOrdinalFilter(view.store.slots);
|
|
10611
|
-
const graph = view.graph;
|
|
10612
10739
|
for (const docId of allowed) {
|
|
10613
10740
|
const ordinal = view.ordinalOf(docId);
|
|
10614
|
-
if (ordinal === void 0) continue;
|
|
10615
|
-
if (graph !== null && isTombstoned(graph, ordinal)) continue;
|
|
10741
|
+
if (ordinal === void 0 || flags.isRemoved(ordinal)) continue;
|
|
10616
10742
|
addToOrdinalFilter(filter, ordinal);
|
|
10617
10743
|
}
|
|
10618
10744
|
return filter;
|
|
@@ -10622,9 +10748,9 @@ function createSharedVectorSearcher(options) {
|
|
|
10622
10748
|
const filter = createOrdinalFilter(slots);
|
|
10623
10749
|
for (let ordinal = 0; ordinal < slots; ordinal++) {
|
|
10624
10750
|
const partition = view.store.partitionAt(ordinal);
|
|
10625
|
-
if (partition
|
|
10626
|
-
|
|
10627
|
-
|
|
10751
|
+
if (partition === void 0 || !allowed.has(partition)) continue;
|
|
10752
|
+
if (!view.store.holdsOrdinal(ordinal) || flags.isRemoved(ordinal)) continue;
|
|
10753
|
+
addToOrdinalFilter(filter, ordinal);
|
|
10628
10754
|
}
|
|
10629
10755
|
return filter;
|
|
10630
10756
|
}
|
|
@@ -10633,18 +10759,62 @@ function createSharedVectorSearcher(options) {
|
|
|
10633
10759
|
if (searchOptions.filterPartitions !== void 0) return filterForPartitions(searchOptions.filterPartitions);
|
|
10634
10760
|
return void 0;
|
|
10635
10761
|
}
|
|
10636
|
-
function
|
|
10762
|
+
function candidatesOf(ordinals, filter) {
|
|
10763
|
+
const candidates = { ordinals: [], docIds: [] };
|
|
10764
|
+
for (const ordinal of ordinals) {
|
|
10765
|
+
if (filter !== void 0 && !ordinalFilterHas(filter, ordinal)) continue;
|
|
10766
|
+
if (flags.isRemoved(ordinal)) continue;
|
|
10767
|
+
const docId = view.docIdOf(ordinal);
|
|
10768
|
+
if (docId === void 0 || view.ordinalOf(docId) !== ordinal) continue;
|
|
10769
|
+
candidates.ordinals.push(ordinal);
|
|
10770
|
+
candidates.docIds.push(docId);
|
|
10771
|
+
}
|
|
10772
|
+
return candidates;
|
|
10773
|
+
}
|
|
10774
|
+
function scanned(scan2) {
|
|
10775
|
+
return { results: scan2.results, matched: scan2.matched, matchedExact: true, resolvedEveryHit: true };
|
|
10776
|
+
}
|
|
10777
|
+
function search2(query, k, searchOptions, filter) {
|
|
10637
10778
|
const { metric, minSimilarity, efSearch, oversample } = searchOptions;
|
|
10638
|
-
const
|
|
10779
|
+
const empty = { results: [], matched: 0, matchedExact: true, resolvedEveryHit: true };
|
|
10780
|
+
const liveDocuments = flags.liveDocuments();
|
|
10781
|
+
if (liveDocuments <= 0) return empty;
|
|
10782
|
+
if (filter !== void 0 && filter.count === 0) return empty;
|
|
10783
|
+
function fetched(results, resolvedEveryHit2) {
|
|
10784
|
+
if (minSimilarity === Number.NEGATIVE_INFINITY) {
|
|
10785
|
+
const matched = filter === void 0 ? liveDocuments : filter.count;
|
|
10786
|
+
return { results, matched, matchedExact: true, resolvedEveryHit: resolvedEveryHit2 };
|
|
10787
|
+
}
|
|
10788
|
+
return { results, matched: results.length, matchedExact: false, resolvedEveryHit: resolvedEveryHit2 };
|
|
10789
|
+
}
|
|
10790
|
+
if (k <= 0) return fetched([], true);
|
|
10639
10791
|
const graph = view.graph;
|
|
10640
|
-
if (graph
|
|
10641
|
-
|
|
10792
|
+
if (graph === null) {
|
|
10793
|
+
const ordinals = filter === void 0 ? flags.unplacedOrdinals() : ordinalFilterValues(filter);
|
|
10794
|
+
return scanned(scoreCandidates(view.store, candidatesOf(ordinals, filter), query, k, metric, minSimilarity));
|
|
10642
10795
|
}
|
|
10643
|
-
|
|
10644
|
-
|
|
10645
|
-
|
|
10796
|
+
if (filter !== void 0) {
|
|
10797
|
+
const graphLiveSize = view.liveSize;
|
|
10798
|
+
const selectivity = graphLiveSize > 0 ? filter.count / graphLiveSize : 1;
|
|
10799
|
+
if (selectivity < view.handles.filterThreshold) {
|
|
10800
|
+
const candidates = candidatesOf(ordinalFilterValues(filter), filter);
|
|
10801
|
+
return scanned(scoreCandidates(view.store, candidates, query, k, metric, minSimilarity));
|
|
10802
|
+
}
|
|
10803
|
+
}
|
|
10804
|
+
let resolvedEveryHit = true;
|
|
10805
|
+
const graphResults = [];
|
|
10806
|
+
for (const hit of view.searchOrdinals(query, k, metric, minSimilarity, { filter, efSearch, oversample })) {
|
|
10807
|
+
const docId = view.docIdOf(hit.ord);
|
|
10808
|
+
if (docId === void 0) {
|
|
10809
|
+
resolvedEveryHit = false;
|
|
10810
|
+
continue;
|
|
10811
|
+
}
|
|
10812
|
+
graphResults.push({ docId, score: hit.score });
|
|
10646
10813
|
}
|
|
10647
|
-
|
|
10814
|
+
const unplaced = flags.unplacedOrdinals();
|
|
10815
|
+
if (unplaced.length === 0) return fetched(graphResults, resolvedEveryHit);
|
|
10816
|
+
const scan2 = scoreCandidates(view.store, candidatesOf(unplaced, filter), query, k, metric, minSimilarity);
|
|
10817
|
+
return fetched(mergeResults(graphResults, scan2.results, k), resolvedEveryHit);
|
|
10648
10818
|
}
|
|
10649
10819
|
return {
|
|
10650
10820
|
fieldName,
|
|
@@ -10659,22 +10829,20 @@ function createSharedVectorSearcher(options) {
|
|
|
10659
10829
|
{ expected: view.handles.dimension, received: query.length }
|
|
10660
10830
|
);
|
|
10661
10831
|
}
|
|
10662
|
-
const
|
|
10663
|
-
if (
|
|
10664
|
-
|
|
10665
|
-
|
|
10666
|
-
|
|
10667
|
-
|
|
10668
|
-
|
|
10669
|
-
const docId = view.docIdOf(hit.ord);
|
|
10670
|
-
if (docId === void 0 || !holdsDocument(docId)) continue;
|
|
10671
|
-
results.push({ docId, score: hit.score });
|
|
10832
|
+
const outcome = search2(query, k, searchOptions, filterFor(searchOptions));
|
|
10833
|
+
if (view.graphRetired) {
|
|
10834
|
+
throw new NarsilError(
|
|
10835
|
+
ErrorCodes.INDEX_NOT_FOUND,
|
|
10836
|
+
`The main thread replaced the graph of vector field "${fieldName}" while this thread searched it`,
|
|
10837
|
+
{ fieldName }
|
|
10838
|
+
);
|
|
10672
10839
|
}
|
|
10673
|
-
const
|
|
10840
|
+
const results = outcome.results.filter((result) => holdsDocument(result.docId));
|
|
10841
|
+
const holdsEveryHit = outcome.resolvedEveryHit && results.length === outcome.results.length;
|
|
10674
10842
|
return Promise.resolve({
|
|
10675
10843
|
results,
|
|
10676
|
-
matched:
|
|
10677
|
-
matchedExact:
|
|
10844
|
+
matched: outcome.matched,
|
|
10845
|
+
matchedExact: outcome.matchedExact && holdsEveryHit
|
|
10678
10846
|
});
|
|
10679
10847
|
}
|
|
10680
10848
|
};
|
|
@@ -10709,7 +10877,8 @@ function loadHeldVectorCopy(held, manager, fieldName, handle, handles, threadSlo
|
|
|
10709
10877
|
);
|
|
10710
10878
|
}
|
|
10711
10879
|
function holdsVectorField(held, fieldName) {
|
|
10712
|
-
|
|
10880
|
+
const view = held.fields.get(fieldName)?.view;
|
|
10881
|
+
return view?.readsEveryVector === true && !view.graphRetired;
|
|
10713
10882
|
}
|
|
10714
10883
|
function dropHeldVectorCopy(held, fieldName, handle) {
|
|
10715
10884
|
held.builds.get(handle)?.view.close();
|
|
@@ -10780,6 +10949,9 @@ function createDirectExecutor(options) {
|
|
|
10780
10949
|
return vectorIndexes;
|
|
10781
10950
|
}
|
|
10782
10951
|
function createIndex(indexName, config, language, analysisStale = false) {
|
|
10952
|
+
addIndex(indexName, config, language, analysisStale, null);
|
|
10953
|
+
}
|
|
10954
|
+
function addIndex(indexName, config, language, analysisStale, awaitedCopyLoad) {
|
|
10783
10955
|
if (indexes.has(indexName)) {
|
|
10784
10956
|
throw new NarsilError(ErrorCodes.INDEX_ALREADY_EXISTS, `Index "${indexName}" already exists`, {
|
|
10785
10957
|
indexName
|
|
@@ -10800,7 +10972,8 @@ function createDirectExecutor(options) {
|
|
|
10800
10972
|
},
|
|
10801
10973
|
vectorIndexes,
|
|
10802
10974
|
vectorCopies: createHeldVectorCopies(),
|
|
10803
|
-
analysisStale
|
|
10975
|
+
analysisStale,
|
|
10976
|
+
awaitedCopyLoad
|
|
10804
10977
|
});
|
|
10805
10978
|
}
|
|
10806
10979
|
function dropIndex(indexName) {
|
|
@@ -10822,7 +10995,7 @@ function createDirectExecutor(options) {
|
|
|
10822
10995
|
}
|
|
10823
10996
|
function queryContextOf(indexName) {
|
|
10824
10997
|
const entry = indexes.get(indexName);
|
|
10825
|
-
if (entry === void 0) return void 0;
|
|
10998
|
+
if (entry === void 0 || entry.awaitedCopyLoad !== null) return void 0;
|
|
10826
10999
|
return {
|
|
10827
11000
|
manager: entry.manager,
|
|
10828
11001
|
config: entry.config,
|
|
@@ -10852,7 +11025,12 @@ function createDirectExecutor(options) {
|
|
|
10852
11025
|
}
|
|
10853
11026
|
case "createIndex": {
|
|
10854
11027
|
const language = getLanguage(action.config.language ?? "english");
|
|
10855
|
-
|
|
11028
|
+
addIndex(action.indexName, action.config, language, action.analysisStale ?? false, action.copyLoad ?? null);
|
|
11029
|
+
return void 0;
|
|
11030
|
+
}
|
|
11031
|
+
case "copyLoaded": {
|
|
11032
|
+
const entry = indexes.get(action.indexName);
|
|
11033
|
+
if (entry?.awaitedCopyLoad === action.copyLoad) entry.awaitedCopyLoad = null;
|
|
10856
11034
|
return void 0;
|
|
10857
11035
|
}
|
|
10858
11036
|
case "dropIndex": {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-
|
|
1
|
+
import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-NFF5TDRG.mjs';
|
|
2
2
|
import { MAX_FILTER_DEPTH } from './chunk-WQBCBKY3.mjs';
|
|
3
3
|
import { MAX_SORT_FIELDS } from './chunk-KJ7LB3TR.mjs';
|
|
4
4
|
import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
|
|
@@ -251,6 +251,7 @@ var MIN_UNBOUNDED_SERVER_LIMIT = 0;
|
|
|
251
251
|
var RESPONSE_STREAM_THRESHOLD_BYTES = 64 * 1024;
|
|
252
252
|
var MAX_INDEX_TOUCH_INTERVAL_MS = 1e3;
|
|
253
253
|
var INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT = 4;
|
|
254
|
+
var REQUEST_THREAD_DEPARTURE_DRAIN_MS = 100;
|
|
254
255
|
var DEFAULT_MAX_RETAINED_TASKS = 1e3;
|
|
255
256
|
var DEFAULT_TASK_PAGE_SIZE = 20;
|
|
256
257
|
var TASK_PROGRESS_WRITE_INTERVAL_MS = 250;
|
|
@@ -913,8 +914,8 @@ function sinkWritingAfterStatus(res, headers) {
|
|
|
913
914
|
res.writeHeader(key, value);
|
|
914
915
|
return sink;
|
|
915
916
|
},
|
|
916
|
-
end(body) {
|
|
917
|
-
res.end(body);
|
|
917
|
+
end(body, closeConnection) {
|
|
918
|
+
res.end(body, closeConnection);
|
|
918
919
|
return sink;
|
|
919
920
|
},
|
|
920
921
|
write(chunk) {
|
|
@@ -981,7 +982,7 @@ function sendDenial(res, authorization) {
|
|
|
981
982
|
sendError(res, authorization.denial.status, authorization.denial.code, authorization.denial.message);
|
|
982
983
|
}
|
|
983
984
|
function createRouteRunner(deps) {
|
|
984
|
-
const { authorize, gate, corsHeaders } = deps;
|
|
985
|
+
const { authorize, gate, corsHeaders, track } = deps;
|
|
985
986
|
return (handler, opts) => {
|
|
986
987
|
const paramCount = opts.paramCount ?? 0;
|
|
987
988
|
const needsBody = opts.needsBody ?? false;
|
|
@@ -1026,6 +1027,7 @@ function createRouteRunner(deps) {
|
|
|
1026
1027
|
);
|
|
1027
1028
|
return;
|
|
1028
1029
|
}
|
|
1030
|
+
const release = track?.(res, abort);
|
|
1029
1031
|
const bodyPromise = needsBody ? readBody(res, opts.maxBytes, abort, declaredBytes) : Promise.resolve(null);
|
|
1030
1032
|
const hookPromise = hookContext !== null && authorize !== void 0 ? authorize(hookContext) : Promise.resolve({ allowed: true });
|
|
1031
1033
|
const spoolPromise = opts.spoolsBody === true ? spoolBody(
|
|
@@ -1034,7 +1036,7 @@ function createRouteRunner(deps) {
|
|
|
1034
1036
|
abort,
|
|
1035
1037
|
hookPromise.then((authorization) => authorization.allowed)
|
|
1036
1038
|
).catch(() => null) : Promise.resolve(null);
|
|
1037
|
-
Promise.all([bodyPromise, spoolPromise, hookPromise]).then(async ([rawBody, spooledBody, authorization]) => {
|
|
1039
|
+
const running = Promise.all([bodyPromise, spoolPromise, hookPromise]).then(async ([rawBody, spooledBody, authorization]) => {
|
|
1038
1040
|
const discardSpool = () => {
|
|
1039
1041
|
void spooledBody?.remove().catch(() => void 0);
|
|
1040
1042
|
};
|
|
@@ -1064,6 +1066,7 @@ function createRouteRunner(deps) {
|
|
|
1064
1066
|
}
|
|
1065
1067
|
}).catch(() => {
|
|
1066
1068
|
});
|
|
1069
|
+
if (release !== void 0) void running.then(release);
|
|
1067
1070
|
};
|
|
1068
1071
|
};
|
|
1069
1072
|
}
|
|
@@ -1195,4 +1198,26 @@ function childAppOf(app) {
|
|
|
1195
1198
|
return candidate;
|
|
1196
1199
|
}
|
|
1197
1200
|
|
|
1198
|
-
|
|
1201
|
+
// src/server/request-threads/departure.ts
|
|
1202
|
+
var DEPARTURE_WORD = 0;
|
|
1203
|
+
var AWAITING_DEPARTURE = 0;
|
|
1204
|
+
var DEPARTURE_CONFIRMED = 1;
|
|
1205
|
+
function createDepartureWord() {
|
|
1206
|
+
return new Int32Array(new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT));
|
|
1207
|
+
}
|
|
1208
|
+
function confirmDeparture(word) {
|
|
1209
|
+
Atomics.store(word, DEPARTURE_WORD, DEPARTURE_CONFIRMED);
|
|
1210
|
+
Atomics.notify(word, DEPARTURE_WORD);
|
|
1211
|
+
}
|
|
1212
|
+
function departureConfirmed(word) {
|
|
1213
|
+
return Atomics.load(word, DEPARTURE_WORD) === DEPARTURE_CONFIRMED;
|
|
1214
|
+
}
|
|
1215
|
+
function awaitDepartureConfirmation(word) {
|
|
1216
|
+
while (!departureConfirmed(word)) Atomics.wait(word, DEPARTURE_WORD, AWAITING_DEPARTURE);
|
|
1217
|
+
}
|
|
1218
|
+
async function departureConfirmation(word) {
|
|
1219
|
+
const waiting = Atomics.waitAsync(word, DEPARTURE_WORD, AWAITING_DEPARTURE);
|
|
1220
|
+
if (waiting.async) await waiting.value;
|
|
1221
|
+
}
|
|
1222
|
+
|
|
1223
|
+
export { ASYNC_IMPORT_CAPABILITY, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_RETAINED_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_TASK_PAGE_SIZE, DOCUMENT_EXPORT_CAPABILITY, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, INDEX_LIFECYCLE_CAPABILITY, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, MAIN_THREAD_GATE_SLOT, MAX_INDEX_TOUCH_INTERVAL_MS, MIN_COUNTED_SERVER_LIMIT, MIN_UNBOUNDED_SERVER_LIMIT, REBUILD_ANALYSIS_CAPABILITY, RELAY_STREAM_CHUNKS_IN_FLIGHT, REQUEST_THREAD_DEPARTURE_DRAIN_MS, RUNNING_TASK_TTL_MS, SERVER_CAPABILITIES, SNAPSHOT_ARCHIVE_CONTENT_TYPE, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, TASK_PROGRESS_WRITE_INTERVAL_MS, TERMINAL_TASK_TTL_MS, acceptorOf, authorizerFor, awaitDepartureConfirmation, badRequest, childAppOf, clearGateSlot, confirmDeparture, corsHeaderReader, createDepartureWord, createDocumentReadHandlers, createRouteRunner, createSharedGateBuffer, createSharedRequestGate, createVersionHandler, departureConfirmation, departureConfirmed, encodeJson, gateSlotOfWorker, httpStatusForNarsilError, loadUWebSockets, parseJson, parseJsonOptional, parseTaskListQuery, refuseRegex, registerServerRoutes, rejectInvalid, resolveCors, respondError, respondJson, sendError, sendRelayed, sendStream, serializeBatchResult, serializeNarsilError, serverRoutes, statusLine, toHttpError, validateBatch, validateCreateIndex, validateDocumentBody, validateQuery, validateRebalance, validateSuggest, versionReport };
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
export { createDirectExecutor } from './chunk-
|
|
1
|
+
export { createDirectExecutor } from './chunk-RCSXJCI5.mjs';
|
|
2
2
|
import './chunk-XYDP7CCJ.mjs';
|
|
3
3
|
import './chunk-OLXM65YH.mjs';
|
|
4
4
|
import './chunk-ZCKPRCYI.mjs';
|
|
5
5
|
import './chunk-TGP5M5R7.mjs';
|
|
6
|
-
import './chunk-
|
|
6
|
+
import './chunk-NIWDQTZB.mjs';
|
|
7
7
|
import './chunk-ZJ4CSAB2.mjs';
|
|
8
|
-
import './chunk-
|
|
8
|
+
import './chunk-NFF5TDRG.mjs';
|
|
9
9
|
import './chunk-SREPGC6O.mjs';
|
|
10
10
|
import './chunk-WQBCBKY3.mjs';
|
|
11
11
|
import './chunk-4VBG6PEH.mjs';
|