@delali/narsil 0.3.1-next.83 → 0.3.1-next.85
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-7JZZ3HNV.mjs → chunk-67MQF5CV.mjs} +215 -48
- package/dist/{chunk-B3UOQJD3.mjs → chunk-NFF5TDRG.mjs} +1 -1
- package/dist/{chunk-RZXNWQSM.mjs → chunk-NIWDQTZB.mjs} +591 -193
- package/dist/{chunk-CUP2E4OH.mjs → chunk-RCSXJCI5.mjs} +373 -199
- package/dist/{chunk-XZF4LZ3X.mjs → chunk-RM6FRYEC.mjs} +31 -6
- package/dist/{chunk-POWOTUKJ.mjs → chunk-YRX6XMOQ.mjs} +1 -0
- package/dist/{direct-executor-6HASZ36Y.mjs → direct-executor-NY7ZTZIH.mjs} +3 -3
- package/dist/distribution.mjs +22 -7
- package/dist/index.browser.mjs +1040 -304
- package/dist/index.mjs +5 -5
- package/dist/persistence/durability/checkpoint-worker.mjs +5 -5
- package/dist/server.mjs +31 -16
- package/dist/{thread-33M5AOEI.mjs → thread-EUUNJNKU.mjs} +226 -41
- package/dist/vector/search-worker.d.ts +8 -0
- package/dist/vector/search-worker.mjs +2 -2
- package/dist/workers/entry.mjs +4 -4
- package/package.json +7 -7
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { createGrowingPartialWords, requirePatternSearchFor, requirePartialWordSearchFor, buildPartialWordTableFrom, patternIndexOf, encodePartialSpellingsOf, countPartialSpelling, joinPatternWorkMeter, countsWithoutScores, fanOutMatchCount, fanOutQuery, sanitizeGlobalStats, expandPartialWords, computeFacets, cloneProjected, projectionKeepsField, setNestedValue, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, PATTERN_HOST, validatedFilterOf, everyValueFacetConfig, mergeFacets, kWayMerge, polygonContainment, polygonBoxes, radiusBoxes, locationCode, undeclaredFilterField, coveringRanges } from './chunk-XYDP7CCJ.mjs';
|
|
2
2
|
import { getLanguage, detectRuntime } from './chunk-OLXM65YH.mjs';
|
|
3
3
|
import { MAX_DOC_ID_LENGTH } from './chunk-ZCKPRCYI.mjs';
|
|
4
|
-
import { SCRATCH_SLOTS_PER_THREAD_POOL, osqRecordBytes, openSharedVectorField, osqBitsOf, createVectorStore, DISK_STORAGE_BLOCK_BYTES, createOsqQuantizer,
|
|
5
|
-
import { noteFallback } from './chunk-
|
|
4
|
+
import { SCRATCH_SLOTS_PER_THREAD_POOL, osqRecordBytes, openSharedVectorField, osqBitsOf, createVectorStore, DISK_STORAGE_BLOCK_BYTES, createOsqQuantizer, STORE_UNPLACED_COUNT, STORE_REMOVED_COUNT, STORE_LIVE_COUNT, ordinalFilterValues, UNPLACED_FLAG_CHUNK_ORDINALS, magnitude, graphWithoutNodes, createHNSWIndex, nativeStoreScratchBytes, nativeScoresOf, toScore, ordinalFilterHas, createOrdinalFilter, addToOrdinalFilter, removeFromOrdinalFilter, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, topLayerInParts, STORE_CALIBRATION_GENERATION, toDistance, sharedMemoryAvailable, requireWholeNeighbourLists, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, releaseLocksHeldBy, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS } from './chunk-NIWDQTZB.mjs';
|
|
5
|
+
import { noteFallback } from './chunk-NFF5TDRG.mjs';
|
|
6
6
|
import { GrowableUint32, bitsetHas, createBitSet, bitsetSet, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, bitsetFromSet, bitsetIsEmpty } from './chunk-SREPGC6O.mjs';
|
|
7
7
|
import { GEO_UNIT_METRES } from './chunk-WQBCBKY3.mjs';
|
|
8
8
|
import { tokenizeOptions, getNestedValue, visitTokens, expandTermPrefix, suggestDisplayTerms, textFieldsChanged, sameFields, codePointOrder, encodeStringBlob, encodeLookupKey, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, getFlatSchema, tokenize, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, getOrCreateFieldNameIndex, getFieldValueByInternalId, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT, partitionDocumentFrequency } from './chunk-4VBG6PEH.mjs';
|
|
@@ -3394,65 +3394,6 @@ function mergeSegmentState(target, source) {
|
|
|
3394
3394
|
target.scoreBuffer = null;
|
|
3395
3395
|
}
|
|
3396
3396
|
|
|
3397
|
-
// src/core/partition/rebuild.ts
|
|
3398
|
-
var EMPTY_STATS = {
|
|
3399
|
-
totalDocuments: 0,
|
|
3400
|
-
totalFieldLengths: {},
|
|
3401
|
-
averageFieldLengths: {},
|
|
3402
|
-
docFrequencies: {}
|
|
3403
|
-
};
|
|
3404
|
-
function textFieldsOf(fields) {
|
|
3405
|
-
return fields.filter((field) => isWordIndexedBase(field.base));
|
|
3406
|
-
}
|
|
3407
|
-
function rebuildTextIndex(state, schema, language, options) {
|
|
3408
|
-
const textFields = textFieldsOf(schemaFieldsOf(schema));
|
|
3409
|
-
state.invertedIdx.clear();
|
|
3410
|
-
state.surfaceRegistry.clear();
|
|
3411
|
-
state.stats.deserialize(EMPTY_STATS);
|
|
3412
|
-
state.invertedIdx.beginBatch();
|
|
3413
|
-
try {
|
|
3414
|
-
for (const [docId, stored] of state.docStore.all()) {
|
|
3415
|
-
const internalId = state.docStore.getInternalId(docId);
|
|
3416
|
-
if (internalId === void 0) continue;
|
|
3417
|
-
const fields = stored.fields;
|
|
3418
|
-
const fieldLengths = {};
|
|
3419
|
-
const tokensByField = {};
|
|
3420
|
-
for (const { path: fieldPath, base } of textFields) {
|
|
3421
|
-
const value = getNestedValue(fields, fieldPath);
|
|
3422
|
-
if (value === void 0 || value === null) continue;
|
|
3423
|
-
if (base === "string") {
|
|
3424
|
-
indexStringField(
|
|
3425
|
-
state,
|
|
3426
|
-
internalId,
|
|
3427
|
-
fieldPath,
|
|
3428
|
-
value,
|
|
3429
|
-
language,
|
|
3430
|
-
options,
|
|
3431
|
-
fieldLengths,
|
|
3432
|
-
tokensByField
|
|
3433
|
-
);
|
|
3434
|
-
} else {
|
|
3435
|
-
indexStringArrayField(
|
|
3436
|
-
state,
|
|
3437
|
-
internalId,
|
|
3438
|
-
fieldPath,
|
|
3439
|
-
value,
|
|
3440
|
-
language,
|
|
3441
|
-
options,
|
|
3442
|
-
fieldLengths,
|
|
3443
|
-
tokensByField
|
|
3444
|
-
);
|
|
3445
|
-
}
|
|
3446
|
-
}
|
|
3447
|
-
state.docStore.storeRef(docId, stored.fields, fieldLengths);
|
|
3448
|
-
state.stats.addDocument(fieldLengths, tokensByField);
|
|
3449
|
-
}
|
|
3450
|
-
} finally {
|
|
3451
|
-
state.invertedIdx.endBatch();
|
|
3452
|
-
state.partialWords.forgetSpellings();
|
|
3453
|
-
}
|
|
3454
|
-
}
|
|
3455
|
-
|
|
3456
3397
|
// src/core/partition/block-bounds.ts
|
|
3457
3398
|
var ABSENT_FIELD_LENGTH_BOUND = 0;
|
|
3458
3399
|
var boundsByList = /* @__PURE__ */ new WeakMap();
|
|
@@ -4873,9 +4814,6 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
4873
4814
|
if (internalId !== void 0) recordSortValues(state, internalId, document);
|
|
4874
4815
|
}
|
|
4875
4816
|
},
|
|
4876
|
-
rebuildTextIndex(schema, language, options) {
|
|
4877
|
-
rebuildTextIndex(state, schema, language, options);
|
|
4878
|
-
},
|
|
4879
4817
|
get(docId, projection) {
|
|
4880
4818
|
const stored = state.docStore.get(docId);
|
|
4881
4819
|
if (!stored) return void 0;
|
|
@@ -5442,18 +5380,6 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
5442
5380
|
if (frozen.length === 0) return live.encodeSegment();
|
|
5443
5381
|
return composedScratch().encodeSegment();
|
|
5444
5382
|
},
|
|
5445
|
-
rebuildTextIndex(schema, language, options) {
|
|
5446
|
-
invalidateDocFrequencies();
|
|
5447
|
-
if (frozen.length > 0) {
|
|
5448
|
-
for (const segment of frozen) {
|
|
5449
|
-
const payload = encodeSegmentState(segment);
|
|
5450
|
-
if (payload.documentCount === 0) continue;
|
|
5451
|
-
live.mergeSegmentPayload(payload, survivorDocumentsOf(segment, payload));
|
|
5452
|
-
}
|
|
5453
|
-
frozen.length = 0;
|
|
5454
|
-
}
|
|
5455
|
-
live.rebuildTextIndex(schema, language, options);
|
|
5456
|
-
},
|
|
5457
5383
|
serialize(indexName, totalPartitions, language, schema) {
|
|
5458
5384
|
if (frozen.length === 0) return live.serialize(indexName, totalPartitions, language, schema);
|
|
5459
5385
|
return composedScratch().serialize(indexName, totalPartitions, language, schema);
|
|
@@ -5678,6 +5604,81 @@ function createCapacityChecks(source) {
|
|
|
5678
5604
|
};
|
|
5679
5605
|
}
|
|
5680
5606
|
|
|
5607
|
+
// src/partitioning/manager/stats.ts
|
|
5608
|
+
function aggregateStatsOf(partitions) {
|
|
5609
|
+
let totalDocuments = 0;
|
|
5610
|
+
const docFrequencies = /* @__PURE__ */ Object.create(null);
|
|
5611
|
+
const totalFieldLengths = /* @__PURE__ */ Object.create(null);
|
|
5612
|
+
for (let i = 0; i < partitions.length; i++) {
|
|
5613
|
+
const stats = partitions[i].stats;
|
|
5614
|
+
totalDocuments += stats.totalDocuments;
|
|
5615
|
+
for (const [term, freq] of Object.entries(stats.docFrequencies)) {
|
|
5616
|
+
docFrequencies[term] = (docFrequencies[term] ?? 0) + freq;
|
|
5617
|
+
}
|
|
5618
|
+
for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
|
|
5619
|
+
totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
|
|
5620
|
+
}
|
|
5621
|
+
}
|
|
5622
|
+
return { totalDocuments, docFrequencies, totalFieldLengths };
|
|
5623
|
+
}
|
|
5624
|
+
function estimateBytesOf(partitions, vecIndexes) {
|
|
5625
|
+
let total = 0;
|
|
5626
|
+
for (let i = 0; i < partitions.length; i++) {
|
|
5627
|
+
total += partitions[i].estimateMemoryBytes();
|
|
5628
|
+
}
|
|
5629
|
+
for (const [, vecIndex] of vecIndexes) {
|
|
5630
|
+
total += vecIndex.estimateMemoryBytes();
|
|
5631
|
+
}
|
|
5632
|
+
return total;
|
|
5633
|
+
}
|
|
5634
|
+
function partitionStatsOf(partitions) {
|
|
5635
|
+
return partitions.map((p, i) => ({
|
|
5636
|
+
partitionId: i,
|
|
5637
|
+
documentCount: p.count(),
|
|
5638
|
+
estimatedMemoryBytes: p.estimateMemoryBytes()
|
|
5639
|
+
}));
|
|
5640
|
+
}
|
|
5641
|
+
|
|
5642
|
+
// src/partitioning/manager/write-log.ts
|
|
5643
|
+
function createWriteLogRecorder() {
|
|
5644
|
+
let current = null;
|
|
5645
|
+
return {
|
|
5646
|
+
open() {
|
|
5647
|
+
const log = { docIds: /* @__PURE__ */ new Set(), replaced: false };
|
|
5648
|
+
current = log;
|
|
5649
|
+
return {
|
|
5650
|
+
get partitionsReplaced() {
|
|
5651
|
+
return log.replaced;
|
|
5652
|
+
},
|
|
5653
|
+
get pendingDocuments() {
|
|
5654
|
+
return log.docIds.size;
|
|
5655
|
+
},
|
|
5656
|
+
takeWrittenDocIds() {
|
|
5657
|
+
const taken = log.docIds;
|
|
5658
|
+
log.docIds = /* @__PURE__ */ new Set();
|
|
5659
|
+
return taken;
|
|
5660
|
+
},
|
|
5661
|
+
close() {
|
|
5662
|
+
if (current === log) current = null;
|
|
5663
|
+
}
|
|
5664
|
+
};
|
|
5665
|
+
},
|
|
5666
|
+
documentWritten(docId) {
|
|
5667
|
+
if (current !== null) current.docIds.add(docId);
|
|
5668
|
+
},
|
|
5669
|
+
segmentWritten(segment) {
|
|
5670
|
+
if (current === null) return;
|
|
5671
|
+
for (const internalId of segment.docStore.allInternalIds()) {
|
|
5672
|
+
const docId = segment.docStore.getExternalId(internalId);
|
|
5673
|
+
if (docId !== void 0) current.docIds.add(docId);
|
|
5674
|
+
}
|
|
5675
|
+
},
|
|
5676
|
+
partitionsReplaced() {
|
|
5677
|
+
if (current !== null) current.replaced = true;
|
|
5678
|
+
}
|
|
5679
|
+
};
|
|
5680
|
+
}
|
|
5681
|
+
|
|
5681
5682
|
// src/partitioning/manager/index.ts
|
|
5682
5683
|
function createPartitionManager(indexName, config, language, router, initialPartitionCount, vectorIndexes) {
|
|
5683
5684
|
requirePatternSearchFor(config.schema);
|
|
@@ -5689,6 +5690,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
5689
5690
|
const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
|
|
5690
5691
|
let partitions = [];
|
|
5691
5692
|
const docPartitionMap = /* @__PURE__ */ new Map();
|
|
5693
|
+
const writeLog = createWriteLogRecorder();
|
|
5692
5694
|
for (let i = 0; i < count; i++) {
|
|
5693
5695
|
partitions.push(createPartitionIndex(i, trackPositions));
|
|
5694
5696
|
}
|
|
@@ -5776,9 +5778,23 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
5776
5778
|
return [...partitions];
|
|
5777
5779
|
},
|
|
5778
5780
|
setPartitions(newPartitions) {
|
|
5781
|
+
writeLog.partitionsReplaced();
|
|
5779
5782
|
partitions = [...newPartitions];
|
|
5780
5783
|
rebuildDocPartitionMap();
|
|
5781
5784
|
},
|
|
5785
|
+
openWriteLog() {
|
|
5786
|
+
return writeLog.open();
|
|
5787
|
+
},
|
|
5788
|
+
adoptRebuiltPartitions(rebuilt) {
|
|
5789
|
+
if (rebuilt.length !== partitions.length) {
|
|
5790
|
+
throw new NarsilError(
|
|
5791
|
+
ErrorCodes.PARTITION_CORRUPTED,
|
|
5792
|
+
`Index "${indexName}" holds ${partitions.length} partitions, but the rebuild produced ${rebuilt.length}`,
|
|
5793
|
+
{ indexName, partitionCount: partitions.length, rebuiltCount: rebuilt.length }
|
|
5794
|
+
);
|
|
5795
|
+
}
|
|
5796
|
+
partitions = [...rebuilt];
|
|
5797
|
+
},
|
|
5782
5798
|
addPartition() {
|
|
5783
5799
|
const currentMaxPartitions = config.partitions?.maxPartitions;
|
|
5784
5800
|
if (currentMaxPartitions !== void 0 && partitions.length >= currentMaxPartitions) {
|
|
@@ -5789,11 +5805,13 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
5789
5805
|
);
|
|
5790
5806
|
}
|
|
5791
5807
|
const partition = createPartitionIndex(partitions.length, trackPositions);
|
|
5808
|
+
writeLog.partitionsReplaced();
|
|
5792
5809
|
partitions.push(partition);
|
|
5793
5810
|
return partition;
|
|
5794
5811
|
},
|
|
5795
5812
|
removePartition(partitionId) {
|
|
5796
5813
|
validatePartitionId(partitionId);
|
|
5814
|
+
writeLog.partitionsReplaced();
|
|
5797
5815
|
partitions.splice(partitionId, 1);
|
|
5798
5816
|
rebuildDocPartitionMap();
|
|
5799
5817
|
},
|
|
@@ -5807,6 +5825,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
5807
5825
|
if (newCount >= partitions.length) {
|
|
5808
5826
|
return;
|
|
5809
5827
|
}
|
|
5828
|
+
writeLog.partitionsReplaced();
|
|
5810
5829
|
for (const [docId, pid] of docPartitionMap) {
|
|
5811
5830
|
if (pid >= newCount) {
|
|
5812
5831
|
docPartitionMap.delete(docId);
|
|
@@ -5829,6 +5848,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
5829
5848
|
const insertOpts = resolveInsertOptions(options);
|
|
5830
5849
|
partitions[pid].insert(docId, document, config.schema, language, insertOpts);
|
|
5831
5850
|
docPartitionMap.set(docId, pid);
|
|
5851
|
+
writeLog.documentWritten(docId);
|
|
5832
5852
|
},
|
|
5833
5853
|
remove(docId) {
|
|
5834
5854
|
const pid = locateDocument(docId);
|
|
@@ -5837,11 +5857,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
5837
5857
|
}
|
|
5838
5858
|
partitions[pid].remove(docId, config.schema, language, resolveInsertOptions());
|
|
5839
5859
|
docPartitionMap.delete(docId);
|
|
5840
|
-
|
|
5841
|
-
rebuildTextIndex(partitionId) {
|
|
5842
|
-
validatePartitionId(partitionId);
|
|
5843
|
-
partitions[partitionId].rebuildTextIndex(config.schema, language, resolveInsertOptions());
|
|
5844
|
-
preparePartialWords(partitionId);
|
|
5860
|
+
writeLog.documentWritten(docId);
|
|
5845
5861
|
},
|
|
5846
5862
|
beginBatchRemove() {
|
|
5847
5863
|
for (let i = 0; i < partitions.length; i++) {
|
|
@@ -5861,6 +5877,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
5861
5877
|
const updateOpts = resolveInsertOptions(options);
|
|
5862
5878
|
partitions[pid].update(docId, document, config.schema, language, updateOpts);
|
|
5863
5879
|
docPartitionMap.set(docId, pid);
|
|
5880
|
+
writeLog.documentWritten(docId);
|
|
5864
5881
|
},
|
|
5865
5882
|
get(docId, projection) {
|
|
5866
5883
|
const pid = locateDocument(docId);
|
|
@@ -5911,6 +5928,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
5911
5928
|
partition.endBatch();
|
|
5912
5929
|
for (const docId of payload.docIds) {
|
|
5913
5930
|
docPartitionMap.set(docId, partitionId);
|
|
5931
|
+
writeLog.documentWritten(docId);
|
|
5914
5932
|
}
|
|
5915
5933
|
},
|
|
5916
5934
|
attachFrozenSegment(partitionId, segment) {
|
|
@@ -5925,6 +5943,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
5925
5943
|
}
|
|
5926
5944
|
asCompositePartition(partitionId).attachFrozenSegment(segment);
|
|
5927
5945
|
registerSegmentDocuments(partitionId, segment);
|
|
5946
|
+
writeLog.segmentWritten(segment);
|
|
5928
5947
|
},
|
|
5929
5948
|
swapFrozenSegments(partitionId, dropSegmentIds, replacement) {
|
|
5930
5949
|
validatePartitionId(partitionId);
|
|
@@ -5942,42 +5961,19 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
5942
5961
|
},
|
|
5943
5962
|
deserializePartition(partitionId, data) {
|
|
5944
5963
|
validatePartitionId(partitionId);
|
|
5964
|
+
writeLog.partitionsReplaced();
|
|
5945
5965
|
partitions[partitionId].deserialize(data, config.schema);
|
|
5946
5966
|
preparePartialWords(partitionId);
|
|
5947
5967
|
rebuildDocPartitionMapForPartition(partitionId);
|
|
5948
5968
|
},
|
|
5949
5969
|
getAggregateStats() {
|
|
5950
|
-
|
|
5951
|
-
const docFrequencies = /* @__PURE__ */ Object.create(null);
|
|
5952
|
-
const totalFieldLengths = /* @__PURE__ */ Object.create(null);
|
|
5953
|
-
for (let i = 0; i < partitions.length; i++) {
|
|
5954
|
-
const stats = partitions[i].stats;
|
|
5955
|
-
totalDocuments += stats.totalDocuments;
|
|
5956
|
-
for (const [term, freq] of Object.entries(stats.docFrequencies)) {
|
|
5957
|
-
docFrequencies[term] = (docFrequencies[term] ?? 0) + freq;
|
|
5958
|
-
}
|
|
5959
|
-
for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
|
|
5960
|
-
totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
|
|
5961
|
-
}
|
|
5962
|
-
}
|
|
5963
|
-
return { totalDocuments, docFrequencies, totalFieldLengths };
|
|
5970
|
+
return aggregateStatsOf(partitions);
|
|
5964
5971
|
},
|
|
5965
5972
|
estimateMemoryBytes() {
|
|
5966
|
-
|
|
5967
|
-
for (let i = 0; i < partitions.length; i++) {
|
|
5968
|
-
total += partitions[i].estimateMemoryBytes();
|
|
5969
|
-
}
|
|
5970
|
-
for (const [, vecIndex] of vecIndexes) {
|
|
5971
|
-
total += vecIndex.estimateMemoryBytes();
|
|
5972
|
-
}
|
|
5973
|
-
return total;
|
|
5973
|
+
return estimateBytesOf(partitions, vecIndexes);
|
|
5974
5974
|
},
|
|
5975
5975
|
getPartitionStats() {
|
|
5976
|
-
return partitions
|
|
5977
|
-
partitionId: i,
|
|
5978
|
-
documentCount: p.count(),
|
|
5979
|
-
estimatedMemoryBytes: p.estimateMemoryBytes()
|
|
5980
|
-
}));
|
|
5976
|
+
return partitionStatsOf(partitions);
|
|
5981
5977
|
},
|
|
5982
5978
|
getVectorIndexes() {
|
|
5983
5979
|
return vecIndexes;
|
|
@@ -6070,6 +6066,7 @@ function releaseReplacedVectors(state) {
|
|
|
6070
6066
|
for (const ordinal of state.store.releaseReplacedVectors()) state.osq?.removeOrdinal(ordinal);
|
|
6071
6067
|
}
|
|
6072
6068
|
function adoptGraph(state, graph) {
|
|
6069
|
+
if (state.hnsw !== null && state.hnsw !== graph) state.hnsw.retire();
|
|
6073
6070
|
if (graph === null && state.hnsw !== null) {
|
|
6074
6071
|
state.hnsw.clear();
|
|
6075
6072
|
}
|
|
@@ -6811,7 +6808,7 @@ async function placeChunk(state, graph, dispatch, chunkDocIds, chunkOrdinals, fa
|
|
|
6811
6808
|
continue;
|
|
6812
6809
|
}
|
|
6813
6810
|
if (!state.tombstones.has(docId) && !graph.has(docId) && graph.insertOrdinal(ordinal)) placedHere += 1;
|
|
6814
|
-
state.buffer.delete(docId);
|
|
6811
|
+
if (graph === state.hnsw) state.buffer.delete(docId);
|
|
6815
6812
|
}
|
|
6816
6813
|
if (placedHere > 0 && outcome !== null && !fallback.shortAnswerWarned) {
|
|
6817
6814
|
fallback.shortAnswerWarned = true;
|
|
@@ -6828,7 +6825,7 @@ async function insertLocally(state, graph, docIds, admit) {
|
|
|
6828
6825
|
const ordinal = state.store.getOrdinal(docId);
|
|
6829
6826
|
if (ordinal === void 0) continue;
|
|
6830
6827
|
graph.insertOrdinal(ordinal);
|
|
6831
|
-
state.buffer.delete(docId);
|
|
6828
|
+
if (graph === state.hnsw) state.buffer.delete(docId);
|
|
6832
6829
|
count += 1;
|
|
6833
6830
|
if (count % BUILD_CHUNK_SIZE === 0) {
|
|
6834
6831
|
noteWrite(state);
|
|
@@ -6945,6 +6942,10 @@ async function buildGraphFromStore(state) {
|
|
|
6945
6942
|
}
|
|
6946
6943
|
const previous = state.hnsw;
|
|
6947
6944
|
adoptGraph(state, graph);
|
|
6945
|
+
for (const docId of [...state.buffer]) {
|
|
6946
|
+
const ordinal = state.store.getOrdinal(docId);
|
|
6947
|
+
if (ordinal !== void 0 && graph.holdsNode(ordinal) && !graph.holdsTombstone(ordinal)) state.buffer.delete(docId);
|
|
6948
|
+
}
|
|
6948
6949
|
if (previous !== null && previous !== graph) dropSharedGraph(state, previous);
|
|
6949
6950
|
await releasePendingLocations(state);
|
|
6950
6951
|
}
|
|
@@ -7771,6 +7772,87 @@ function beginCheckpointRestore(state, shape) {
|
|
|
7771
7772
|
return { addFile, finish };
|
|
7772
7773
|
}
|
|
7773
7774
|
|
|
7775
|
+
// src/vector/vector-index/flagged-documents.ts
|
|
7776
|
+
var FlaggedDocuments = class extends Set {
|
|
7777
|
+
constructor(store, layout) {
|
|
7778
|
+
super();
|
|
7779
|
+
this.store = store;
|
|
7780
|
+
this.layout = layout;
|
|
7781
|
+
}
|
|
7782
|
+
ordinals = /* @__PURE__ */ new Map();
|
|
7783
|
+
handles = null;
|
|
7784
|
+
flags = new Uint8Array(0);
|
|
7785
|
+
chunks = null;
|
|
7786
|
+
add(docId) {
|
|
7787
|
+
super.add(docId);
|
|
7788
|
+
const handles = this.openHandles();
|
|
7789
|
+
if (handles === null) return this;
|
|
7790
|
+
this.flagCurrentOrdinal(docId);
|
|
7791
|
+
Atomics.store(handles.header, this.layout.countWord, this.size);
|
|
7792
|
+
return this;
|
|
7793
|
+
}
|
|
7794
|
+
delete(docId) {
|
|
7795
|
+
const held = super.delete(docId);
|
|
7796
|
+
const handles = this.openHandles();
|
|
7797
|
+
if (handles === null) return held;
|
|
7798
|
+
const ordinal = this.ordinals.get(docId);
|
|
7799
|
+
if (ordinal !== void 0) {
|
|
7800
|
+
this.ordinals.delete(docId);
|
|
7801
|
+
this.unflag(ordinal);
|
|
7802
|
+
}
|
|
7803
|
+
Atomics.store(handles.header, this.layout.countWord, this.size);
|
|
7804
|
+
return held;
|
|
7805
|
+
}
|
|
7806
|
+
clear() {
|
|
7807
|
+
super.clear();
|
|
7808
|
+
const handles = this.openHandles();
|
|
7809
|
+
if (handles !== null) {
|
|
7810
|
+
for (const ordinal of this.ordinals.values()) this.unflag(ordinal);
|
|
7811
|
+
Atomics.store(handles.header, this.layout.countWord, 0);
|
|
7812
|
+
}
|
|
7813
|
+
this.ordinals.clear();
|
|
7814
|
+
}
|
|
7815
|
+
flagCurrentOrdinal(docId) {
|
|
7816
|
+
const ordinal = this.store.getOrdinal(docId);
|
|
7817
|
+
const previous = this.ordinals.get(docId);
|
|
7818
|
+
if (ordinal === void 0 || ordinal === previous) return;
|
|
7819
|
+
if (this.chunks !== null) Atomics.add(this.chunks, Math.floor(ordinal / UNPLACED_FLAG_CHUNK_ORDINALS), 1);
|
|
7820
|
+
Atomics.store(this.flags, ordinal, 1);
|
|
7821
|
+
this.ordinals.set(docId, ordinal);
|
|
7822
|
+
if (previous !== void 0) this.unflag(previous);
|
|
7823
|
+
}
|
|
7824
|
+
unflag(ordinal) {
|
|
7825
|
+
Atomics.store(this.flags, ordinal, 0);
|
|
7826
|
+
if (this.chunks !== null) Atomics.sub(this.chunks, Math.floor(ordinal / UNPLACED_FLAG_CHUNK_ORDINALS), 1);
|
|
7827
|
+
}
|
|
7828
|
+
openHandles() {
|
|
7829
|
+
if (this.store.dimension === 0) return null;
|
|
7830
|
+
const handles = this.store.handles;
|
|
7831
|
+
if (handles === this.handles) return handles;
|
|
7832
|
+
this.handles = handles;
|
|
7833
|
+
this.flags = new Uint8Array(this.layout.flagsOf(handles));
|
|
7834
|
+
const chunksOf = this.layout.chunksOf;
|
|
7835
|
+
this.chunks = chunksOf === null ? null : new Int32Array(chunksOf(handles));
|
|
7836
|
+
this.ordinals.clear();
|
|
7837
|
+
for (const docId of this) this.flagCurrentOrdinal(docId);
|
|
7838
|
+
return handles;
|
|
7839
|
+
}
|
|
7840
|
+
};
|
|
7841
|
+
function createUnplacedVectors(store) {
|
|
7842
|
+
return new FlaggedDocuments(store, {
|
|
7843
|
+
flagsOf: (handles) => handles.unplaced,
|
|
7844
|
+
chunksOf: (handles) => handles.unplacedChunks,
|
|
7845
|
+
countWord: STORE_UNPLACED_COUNT
|
|
7846
|
+
});
|
|
7847
|
+
}
|
|
7848
|
+
function createRemovedDocuments(store) {
|
|
7849
|
+
return new FlaggedDocuments(store, {
|
|
7850
|
+
flagsOf: (handles) => handles.removed,
|
|
7851
|
+
chunksOf: null,
|
|
7852
|
+
countWord: STORE_REMOVED_COUNT
|
|
7853
|
+
});
|
|
7854
|
+
}
|
|
7855
|
+
|
|
7774
7856
|
// src/vector/vector-index/maintenance.ts
|
|
7775
7857
|
function compact(state) {
|
|
7776
7858
|
if (state.tombstones.size === 0 && state.store.replacedVectorCount === 0) return;
|
|
@@ -7968,6 +8050,43 @@ function createHeap(compare) {
|
|
|
7968
8050
|
};
|
|
7969
8051
|
}
|
|
7970
8052
|
|
|
8053
|
+
// src/vector/vector-index/merge-results.ts
|
|
8054
|
+
function highScoreFirst(a, b) {
|
|
8055
|
+
return b.score - a.score || compareCodePoints(a.docId, b.docId);
|
|
8056
|
+
}
|
|
8057
|
+
function mergeResults(hnswResults, bufferResults, k) {
|
|
8058
|
+
const seen = /* @__PURE__ */ new Set();
|
|
8059
|
+
const merged = [];
|
|
8060
|
+
let hi = 0;
|
|
8061
|
+
let bi = 0;
|
|
8062
|
+
while (merged.length < k && (hi < hnswResults.length || bi < bufferResults.length)) {
|
|
8063
|
+
const h = hi < hnswResults.length ? hnswResults[hi] : void 0;
|
|
8064
|
+
const b = bi < bufferResults.length ? bufferResults[bi] : void 0;
|
|
8065
|
+
let pick;
|
|
8066
|
+
if (h && b) {
|
|
8067
|
+
if (h.score > b.score || h.score === b.score && compareCodePoints(h.docId, b.docId) < 0) {
|
|
8068
|
+
pick = h;
|
|
8069
|
+
hi++;
|
|
8070
|
+
} else {
|
|
8071
|
+
pick = b;
|
|
8072
|
+
bi++;
|
|
8073
|
+
}
|
|
8074
|
+
} else if (h) {
|
|
8075
|
+
pick = h;
|
|
8076
|
+
hi++;
|
|
8077
|
+
} else if (b) {
|
|
8078
|
+
pick = b;
|
|
8079
|
+
bi++;
|
|
8080
|
+
} else {
|
|
8081
|
+
break;
|
|
8082
|
+
}
|
|
8083
|
+
if (seen.has(pick.docId)) continue;
|
|
8084
|
+
seen.add(pick.docId);
|
|
8085
|
+
merged.push(pick);
|
|
8086
|
+
}
|
|
8087
|
+
return merged;
|
|
8088
|
+
}
|
|
8089
|
+
|
|
7971
8090
|
// src/vector/vector-index/search.ts
|
|
7972
8091
|
function* bufferCandidates(state, filter) {
|
|
7973
8092
|
for (const docId of state.buffer) {
|
|
@@ -7997,9 +8116,6 @@ function fetchedOutcome(state, results, minSimilarity, filter) {
|
|
|
7997
8116
|
function scannedOutcome(scan2) {
|
|
7998
8117
|
return { results: scan2.results, matched: scan2.matched, matchedExact: true };
|
|
7999
8118
|
}
|
|
8000
|
-
function highScoreFirst(a, b) {
|
|
8001
|
-
return b.score - a.score || compareCodePoints(a.docId, b.docId);
|
|
8002
|
-
}
|
|
8003
8119
|
function liveCandidates(state, candidates) {
|
|
8004
8120
|
const docIds = [];
|
|
8005
8121
|
const ordinals = [];
|
|
@@ -8065,38 +8181,6 @@ function bruteForceSearch(state, query, k, metric, minSimilarity, candidates) {
|
|
|
8065
8181
|
}
|
|
8066
8182
|
return { results: heap.toSortedArray().reverse(), matched };
|
|
8067
8183
|
}
|
|
8068
|
-
function mergeResults(hnswResults, bufferResults, k) {
|
|
8069
|
-
const seen = /* @__PURE__ */ new Set();
|
|
8070
|
-
const merged = [];
|
|
8071
|
-
let hi = 0;
|
|
8072
|
-
let bi = 0;
|
|
8073
|
-
while (merged.length < k && (hi < hnswResults.length || bi < bufferResults.length)) {
|
|
8074
|
-
const h = hi < hnswResults.length ? hnswResults[hi] : void 0;
|
|
8075
|
-
const b = bi < bufferResults.length ? bufferResults[bi] : void 0;
|
|
8076
|
-
let pick;
|
|
8077
|
-
if (h && b) {
|
|
8078
|
-
if (h.score > b.score || h.score === b.score && compareCodePoints(h.docId, b.docId) < 0) {
|
|
8079
|
-
pick = h;
|
|
8080
|
-
hi++;
|
|
8081
|
-
} else {
|
|
8082
|
-
pick = b;
|
|
8083
|
-
bi++;
|
|
8084
|
-
}
|
|
8085
|
-
} else if (h) {
|
|
8086
|
-
pick = h;
|
|
8087
|
-
hi++;
|
|
8088
|
-
} else if (b) {
|
|
8089
|
-
pick = b;
|
|
8090
|
-
bi++;
|
|
8091
|
-
} else {
|
|
8092
|
-
break;
|
|
8093
|
-
}
|
|
8094
|
-
if (seen.has(pick.docId)) continue;
|
|
8095
|
-
seen.add(pick.docId);
|
|
8096
|
-
merged.push(pick);
|
|
8097
|
-
}
|
|
8098
|
-
return merged;
|
|
8099
|
-
}
|
|
8100
8184
|
function search(state, query, k, options) {
|
|
8101
8185
|
return searchWithFilter(state, query, k, options, filterForOptions(state, options));
|
|
8102
8186
|
}
|
|
@@ -8182,8 +8266,8 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
8182
8266
|
hnswConfig,
|
|
8183
8267
|
workerCopies,
|
|
8184
8268
|
store,
|
|
8185
|
-
tombstones:
|
|
8186
|
-
buffer:
|
|
8269
|
+
tombstones: createRemovedDocuments(store),
|
|
8270
|
+
buffer: createUnplacedVectors(store),
|
|
8187
8271
|
pendingLocations: /* @__PURE__ */ new Map(),
|
|
8188
8272
|
savedFiles: [],
|
|
8189
8273
|
savedSignature: null,
|
|
@@ -10567,56 +10651,94 @@ function runSegmentAction(entry, action) {
|
|
|
10567
10651
|
}
|
|
10568
10652
|
}
|
|
10569
10653
|
|
|
10570
|
-
// src/vector/shared-field/
|
|
10571
|
-
function
|
|
10572
|
-
|
|
10654
|
+
// src/vector/shared-field/store-flags.ts
|
|
10655
|
+
function bytesOf(flags, source) {
|
|
10656
|
+
if (flags.source !== source) {
|
|
10657
|
+
flags.source = source;
|
|
10658
|
+
flags.view = new Uint8Array(source);
|
|
10659
|
+
}
|
|
10660
|
+
return flags.view;
|
|
10573
10661
|
}
|
|
10574
|
-
function
|
|
10662
|
+
function countsOf(chunks, source) {
|
|
10663
|
+
if (chunks.source !== source) {
|
|
10664
|
+
chunks.source = source;
|
|
10665
|
+
chunks.view = new Int32Array(source);
|
|
10666
|
+
}
|
|
10667
|
+
return chunks.view;
|
|
10668
|
+
}
|
|
10669
|
+
function readStoreFlags(store) {
|
|
10670
|
+
const unplaced = { source: null, view: new Uint8Array(0) };
|
|
10671
|
+
const unplacedChunks = { source: null, view: new Int32Array(0) };
|
|
10672
|
+
const removed = { source: null, view: new Uint8Array(0) };
|
|
10673
|
+
return {
|
|
10674
|
+
liveDocuments() {
|
|
10675
|
+
const header = store.handles.header;
|
|
10676
|
+
return Atomics.load(header, STORE_LIVE_COUNT) - Atomics.load(header, STORE_REMOVED_COUNT);
|
|
10677
|
+
},
|
|
10678
|
+
isRemoved(ordinal) {
|
|
10679
|
+
const bytes = bytesOf(removed, store.handles.removed);
|
|
10680
|
+
return ordinal < bytes.length && Atomics.load(bytes, ordinal) === 1;
|
|
10681
|
+
},
|
|
10682
|
+
unplacedOrdinals() {
|
|
10683
|
+
const handles = store.handles;
|
|
10684
|
+
if (Atomics.load(handles.header, STORE_UNPLACED_COUNT) === 0) return [];
|
|
10685
|
+
const bytes = bytesOf(unplaced, handles.unplaced);
|
|
10686
|
+
const counts = countsOf(unplacedChunks, handles.unplacedChunks);
|
|
10687
|
+
const end = Math.min(store.slots, bytes.length);
|
|
10688
|
+
const ordinals = [];
|
|
10689
|
+
for (let chunk = 0; chunk < counts.length; chunk++) {
|
|
10690
|
+
const from = chunk * UNPLACED_FLAG_CHUNK_ORDINALS;
|
|
10691
|
+
if (from >= end) break;
|
|
10692
|
+
if (Atomics.load(counts, chunk) === 0) continue;
|
|
10693
|
+
const to = Math.min(end, from + UNPLACED_FLAG_CHUNK_ORDINALS);
|
|
10694
|
+
for (let ordinal = from; ordinal < to; ordinal++) {
|
|
10695
|
+
if (Atomics.load(bytes, ordinal) === 1) ordinals.push(ordinal);
|
|
10696
|
+
}
|
|
10697
|
+
}
|
|
10698
|
+
return ordinals;
|
|
10699
|
+
}
|
|
10700
|
+
};
|
|
10701
|
+
}
|
|
10702
|
+
|
|
10703
|
+
// src/vector/shared-field/searcher.ts
|
|
10704
|
+
function distancesInTypeScript(store, query, metric, ordinals) {
|
|
10575
10705
|
noteFallback("score");
|
|
10576
|
-
const arenaQuery =
|
|
10706
|
+
const arenaQuery = store.prepareQueryArena(query);
|
|
10577
10707
|
const queryMagnitude = arenaQuery ? arenaQuery.magnitude : magnitude(query);
|
|
10578
10708
|
const distances = new Float64Array(ordinals.length).fill(Number.POSITIVE_INFINITY);
|
|
10579
10709
|
for (let i = 0; i < ordinals.length; i++) {
|
|
10580
10710
|
if (arenaQuery) {
|
|
10581
|
-
distances[i] =
|
|
10711
|
+
distances[i] = store.distanceFromArena(arenaQuery, ordinals[i], metric);
|
|
10582
10712
|
continue;
|
|
10583
10713
|
}
|
|
10584
|
-
const entry =
|
|
10714
|
+
const entry = store.entryForOrdinal(ordinals[i]);
|
|
10585
10715
|
if (entry) distances[i] = toDistance(query, entry.vector, queryMagnitude, entry.magnitude, metric);
|
|
10586
10716
|
}
|
|
10587
10717
|
return distances;
|
|
10588
10718
|
}
|
|
10589
|
-
function
|
|
10590
|
-
const ordinals =
|
|
10591
|
-
|
|
10592
|
-
|
|
10593
|
-
|
|
10594
|
-
const docId = docIdOf(ord);
|
|
10595
|
-
if (docId === void 0) continue;
|
|
10596
|
-
ordinals.push(ord);
|
|
10597
|
-
docIds.push(docId);
|
|
10598
|
-
}
|
|
10599
|
-
const distances = nativeScoresOf(state.store.handles, query, metric, ordinals) ?? distancesInTypeScript(state, query, metric, ordinals);
|
|
10600
|
-
const heap = createBoundedMaxHeap(bestFirst, k);
|
|
10719
|
+
function scoreCandidates(store, candidates, query, k, metric, minSimilarity) {
|
|
10720
|
+
const { ordinals, docIds } = candidates;
|
|
10721
|
+
if (ordinals.length === 0) return { results: [], matched: 0 };
|
|
10722
|
+
const distances = nativeScoresOf(store.handles, query, metric, ordinals) ?? distancesInTypeScript(store, query, metric, ordinals);
|
|
10723
|
+
const heap = createBoundedMaxHeap(highScoreFirst, k);
|
|
10601
10724
|
let matched = 0;
|
|
10602
10725
|
for (let i = 0; i < ordinals.length; i++) {
|
|
10603
10726
|
if (distances[i] === Number.POSITIVE_INFINITY) continue;
|
|
10604
10727
|
const score = toScore(distances[i], metric);
|
|
10605
10728
|
if (score < minSimilarity) continue;
|
|
10606
10729
|
matched++;
|
|
10607
|
-
heap.push({
|
|
10730
|
+
heap.push({ docId: docIds[i], score });
|
|
10608
10731
|
}
|
|
10609
|
-
return {
|
|
10732
|
+
return { results: heap.toSortedArray().reverse(), matched };
|
|
10610
10733
|
}
|
|
10611
10734
|
function createSharedVectorSearcher(options) {
|
|
10612
10735
|
const { fieldName, view, holdsDocument } = options;
|
|
10736
|
+
const flags = readStoreFlags(view.store);
|
|
10613
10737
|
function filterForDocIds(allowed) {
|
|
10614
10738
|
const filter = createOrdinalFilter(view.store.slots);
|
|
10615
|
-
const graph = view.graph;
|
|
10616
10739
|
for (const docId of allowed) {
|
|
10617
10740
|
const ordinal = view.ordinalOf(docId);
|
|
10618
|
-
if (ordinal === void 0) continue;
|
|
10619
|
-
if (graph !== null && isTombstoned(graph, ordinal)) continue;
|
|
10741
|
+
if (ordinal === void 0 || flags.isRemoved(ordinal)) continue;
|
|
10620
10742
|
addToOrdinalFilter(filter, ordinal);
|
|
10621
10743
|
}
|
|
10622
10744
|
return filter;
|
|
@@ -10626,9 +10748,9 @@ function createSharedVectorSearcher(options) {
|
|
|
10626
10748
|
const filter = createOrdinalFilter(slots);
|
|
10627
10749
|
for (let ordinal = 0; ordinal < slots; ordinal++) {
|
|
10628
10750
|
const partition = view.store.partitionAt(ordinal);
|
|
10629
|
-
if (partition
|
|
10630
|
-
|
|
10631
|
-
|
|
10751
|
+
if (partition === void 0 || !allowed.has(partition)) continue;
|
|
10752
|
+
if (!view.store.holdsOrdinal(ordinal) || flags.isRemoved(ordinal)) continue;
|
|
10753
|
+
addToOrdinalFilter(filter, ordinal);
|
|
10632
10754
|
}
|
|
10633
10755
|
return filter;
|
|
10634
10756
|
}
|
|
@@ -10637,18 +10759,62 @@ function createSharedVectorSearcher(options) {
|
|
|
10637
10759
|
if (searchOptions.filterPartitions !== void 0) return filterForPartitions(searchOptions.filterPartitions);
|
|
10638
10760
|
return void 0;
|
|
10639
10761
|
}
|
|
10640
|
-
function
|
|
10762
|
+
function candidatesOf(ordinals, filter) {
|
|
10763
|
+
const candidates = { ordinals: [], docIds: [] };
|
|
10764
|
+
for (const ordinal of ordinals) {
|
|
10765
|
+
if (filter !== void 0 && !ordinalFilterHas(filter, ordinal)) continue;
|
|
10766
|
+
if (flags.isRemoved(ordinal)) continue;
|
|
10767
|
+
const docId = view.docIdOf(ordinal);
|
|
10768
|
+
if (docId === void 0 || view.ordinalOf(docId) !== ordinal) continue;
|
|
10769
|
+
candidates.ordinals.push(ordinal);
|
|
10770
|
+
candidates.docIds.push(docId);
|
|
10771
|
+
}
|
|
10772
|
+
return candidates;
|
|
10773
|
+
}
|
|
10774
|
+
function scanned(scan2) {
|
|
10775
|
+
return { results: scan2.results, matched: scan2.matched, matchedExact: true, resolvedEveryHit: true };
|
|
10776
|
+
}
|
|
10777
|
+
function search2(query, k, searchOptions, filter) {
|
|
10641
10778
|
const { metric, minSimilarity, efSearch, oversample } = searchOptions;
|
|
10642
|
-
const
|
|
10779
|
+
const empty = { results: [], matched: 0, matchedExact: true, resolvedEveryHit: true };
|
|
10780
|
+
const liveDocuments = flags.liveDocuments();
|
|
10781
|
+
if (liveDocuments <= 0) return empty;
|
|
10782
|
+
if (filter !== void 0 && filter.count === 0) return empty;
|
|
10783
|
+
function fetched(results, resolvedEveryHit2) {
|
|
10784
|
+
if (minSimilarity === Number.NEGATIVE_INFINITY) {
|
|
10785
|
+
const matched = filter === void 0 ? liveDocuments : filter.count;
|
|
10786
|
+
return { results, matched, matchedExact: true, resolvedEveryHit: resolvedEveryHit2 };
|
|
10787
|
+
}
|
|
10788
|
+
return { results, matched: results.length, matchedExact: false, resolvedEveryHit: resolvedEveryHit2 };
|
|
10789
|
+
}
|
|
10790
|
+
if (k <= 0) return fetched([], true);
|
|
10643
10791
|
const graph = view.graph;
|
|
10644
|
-
if (graph
|
|
10645
|
-
|
|
10792
|
+
if (graph === null) {
|
|
10793
|
+
const ordinals = filter === void 0 ? flags.unplacedOrdinals() : ordinalFilterValues(filter);
|
|
10794
|
+
return scanned(scoreCandidates(view.store, candidatesOf(ordinals, filter), query, k, metric, minSimilarity));
|
|
10646
10795
|
}
|
|
10647
|
-
|
|
10648
|
-
|
|
10649
|
-
|
|
10796
|
+
if (filter !== void 0) {
|
|
10797
|
+
const graphLiveSize = view.liveSize;
|
|
10798
|
+
const selectivity = graphLiveSize > 0 ? filter.count / graphLiveSize : 1;
|
|
10799
|
+
if (selectivity < view.handles.filterThreshold) {
|
|
10800
|
+
const candidates = candidatesOf(ordinalFilterValues(filter), filter);
|
|
10801
|
+
return scanned(scoreCandidates(view.store, candidates, query, k, metric, minSimilarity));
|
|
10802
|
+
}
|
|
10803
|
+
}
|
|
10804
|
+
let resolvedEveryHit = true;
|
|
10805
|
+
const graphResults = [];
|
|
10806
|
+
for (const hit of view.searchOrdinals(query, k, metric, minSimilarity, { filter, efSearch, oversample })) {
|
|
10807
|
+
const docId = view.docIdOf(hit.ord);
|
|
10808
|
+
if (docId === void 0) {
|
|
10809
|
+
resolvedEveryHit = false;
|
|
10810
|
+
continue;
|
|
10811
|
+
}
|
|
10812
|
+
graphResults.push({ docId, score: hit.score });
|
|
10650
10813
|
}
|
|
10651
|
-
|
|
10814
|
+
const unplaced = flags.unplacedOrdinals();
|
|
10815
|
+
if (unplaced.length === 0) return fetched(graphResults, resolvedEveryHit);
|
|
10816
|
+
const scan2 = scoreCandidates(view.store, candidatesOf(unplaced, filter), query, k, metric, minSimilarity);
|
|
10817
|
+
return fetched(mergeResults(graphResults, scan2.results, k), resolvedEveryHit);
|
|
10652
10818
|
}
|
|
10653
10819
|
return {
|
|
10654
10820
|
fieldName,
|
|
@@ -10663,22 +10829,20 @@ function createSharedVectorSearcher(options) {
|
|
|
10663
10829
|
{ expected: view.handles.dimension, received: query.length }
|
|
10664
10830
|
);
|
|
10665
10831
|
}
|
|
10666
|
-
const
|
|
10667
|
-
if (
|
|
10668
|
-
|
|
10669
|
-
|
|
10670
|
-
|
|
10671
|
-
|
|
10672
|
-
|
|
10673
|
-
const docId = view.docIdOf(hit.ord);
|
|
10674
|
-
if (docId === void 0 || !holdsDocument(docId)) continue;
|
|
10675
|
-
results.push({ docId, score: hit.score });
|
|
10832
|
+
const outcome = search2(query, k, searchOptions, filterFor(searchOptions));
|
|
10833
|
+
if (view.graphRetired) {
|
|
10834
|
+
throw new NarsilError(
|
|
10835
|
+
ErrorCodes.INDEX_NOT_FOUND,
|
|
10836
|
+
`The main thread replaced the graph of vector field "${fieldName}" while this thread searched it`,
|
|
10837
|
+
{ fieldName }
|
|
10838
|
+
);
|
|
10676
10839
|
}
|
|
10677
|
-
const
|
|
10840
|
+
const results = outcome.results.filter((result) => holdsDocument(result.docId));
|
|
10841
|
+
const holdsEveryHit = outcome.resolvedEveryHit && results.length === outcome.results.length;
|
|
10678
10842
|
return Promise.resolve({
|
|
10679
10843
|
results,
|
|
10680
|
-
matched:
|
|
10681
|
-
matchedExact:
|
|
10844
|
+
matched: outcome.matched,
|
|
10845
|
+
matchedExact: outcome.matchedExact && holdsEveryHit
|
|
10682
10846
|
});
|
|
10683
10847
|
}
|
|
10684
10848
|
};
|
|
@@ -10713,7 +10877,8 @@ function loadHeldVectorCopy(held, manager, fieldName, handle, handles, threadSlo
|
|
|
10713
10877
|
);
|
|
10714
10878
|
}
|
|
10715
10879
|
function holdsVectorField(held, fieldName) {
|
|
10716
|
-
|
|
10880
|
+
const view = held.fields.get(fieldName)?.view;
|
|
10881
|
+
return view?.readsEveryVector === true && !view.graphRetired;
|
|
10717
10882
|
}
|
|
10718
10883
|
function dropHeldVectorCopy(held, fieldName, handle) {
|
|
10719
10884
|
held.builds.get(handle)?.view.close();
|
|
@@ -10784,6 +10949,9 @@ function createDirectExecutor(options) {
|
|
|
10784
10949
|
return vectorIndexes;
|
|
10785
10950
|
}
|
|
10786
10951
|
function createIndex(indexName, config, language, analysisStale = false) {
|
|
10952
|
+
addIndex(indexName, config, language, analysisStale, null);
|
|
10953
|
+
}
|
|
10954
|
+
function addIndex(indexName, config, language, analysisStale, awaitedCopyLoad) {
|
|
10787
10955
|
if (indexes.has(indexName)) {
|
|
10788
10956
|
throw new NarsilError(ErrorCodes.INDEX_ALREADY_EXISTS, `Index "${indexName}" already exists`, {
|
|
10789
10957
|
indexName
|
|
@@ -10804,7 +10972,8 @@ function createDirectExecutor(options) {
|
|
|
10804
10972
|
},
|
|
10805
10973
|
vectorIndexes,
|
|
10806
10974
|
vectorCopies: createHeldVectorCopies(),
|
|
10807
|
-
analysisStale
|
|
10975
|
+
analysisStale,
|
|
10976
|
+
awaitedCopyLoad
|
|
10808
10977
|
});
|
|
10809
10978
|
}
|
|
10810
10979
|
function dropIndex(indexName) {
|
|
@@ -10826,7 +10995,7 @@ function createDirectExecutor(options) {
|
|
|
10826
10995
|
}
|
|
10827
10996
|
function queryContextOf(indexName) {
|
|
10828
10997
|
const entry = indexes.get(indexName);
|
|
10829
|
-
if (entry === void 0) return void 0;
|
|
10998
|
+
if (entry === void 0 || entry.awaitedCopyLoad !== null) return void 0;
|
|
10830
10999
|
return {
|
|
10831
11000
|
manager: entry.manager,
|
|
10832
11001
|
config: entry.config,
|
|
@@ -10856,7 +11025,12 @@ function createDirectExecutor(options) {
|
|
|
10856
11025
|
}
|
|
10857
11026
|
case "createIndex": {
|
|
10858
11027
|
const language = getLanguage(action.config.language ?? "english");
|
|
10859
|
-
|
|
11028
|
+
addIndex(action.indexName, action.config, language, action.analysisStale ?? false, action.copyLoad ?? null);
|
|
11029
|
+
return void 0;
|
|
11030
|
+
}
|
|
11031
|
+
case "copyLoaded": {
|
|
11032
|
+
const entry = indexes.get(action.indexName);
|
|
11033
|
+
if (entry?.awaitedCopyLoad === action.copyLoad) entry.awaitedCopyLoad = null;
|
|
10860
11034
|
return void 0;
|
|
10861
11035
|
}
|
|
10862
11036
|
case "dropIndex": {
|