@delali/narsil 0.3.1-next.82 → 0.3.1-next.84
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-4CWHWOBI.mjs → chunk-IA67SOJO.mjs} +268 -196
- package/dist/{chunk-IN5CH4P2.mjs → chunk-RZXNWQSM.mjs} +181 -92
- package/dist/{chunk-IH3GKNDQ.mjs → chunk-XIQZKKTK.mjs} +196 -48
- package/dist/{direct-executor-JVXDR5Y5.mjs → direct-executor-6WWFY5EI.mjs} +2 -2
- package/dist/distribution.mjs +20 -5
- package/dist/index.browser.mjs +678 -369
- package/dist/index.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +3 -3
- package/dist/vector/search-worker.d.ts +2 -0
- package/dist/vector/search-worker.mjs +6 -5
- package/dist/workers/entry.mjs +1 -1
- package/package.json +7 -7
package/dist/index.browser.mjs
CHANGED
|
@@ -7612,65 +7612,6 @@ function mergeSegmentState(target, source) {
|
|
|
7612
7612
|
target.scoreBuffer = null;
|
|
7613
7613
|
}
|
|
7614
7614
|
|
|
7615
|
-
// src/core/partition/rebuild.ts
|
|
7616
|
-
var EMPTY_STATS = {
|
|
7617
|
-
totalDocuments: 0,
|
|
7618
|
-
totalFieldLengths: {},
|
|
7619
|
-
averageFieldLengths: {},
|
|
7620
|
-
docFrequencies: {}
|
|
7621
|
-
};
|
|
7622
|
-
function textFieldsOf(fields) {
|
|
7623
|
-
return fields.filter((field) => isWordIndexedBase(field.base));
|
|
7624
|
-
}
|
|
7625
|
-
function rebuildTextIndex(state, schema, language, options) {
|
|
7626
|
-
const textFields = textFieldsOf(schemaFieldsOf(schema));
|
|
7627
|
-
state.invertedIdx.clear();
|
|
7628
|
-
state.surfaceRegistry.clear();
|
|
7629
|
-
state.stats.deserialize(EMPTY_STATS);
|
|
7630
|
-
state.invertedIdx.beginBatch();
|
|
7631
|
-
try {
|
|
7632
|
-
for (const [docId, stored] of state.docStore.all()) {
|
|
7633
|
-
const internalId = state.docStore.getInternalId(docId);
|
|
7634
|
-
if (internalId === void 0) continue;
|
|
7635
|
-
const fields = stored.fields;
|
|
7636
|
-
const fieldLengths = {};
|
|
7637
|
-
const tokensByField = {};
|
|
7638
|
-
for (const { path: fieldPath, base } of textFields) {
|
|
7639
|
-
const value = getNestedValue(fields, fieldPath);
|
|
7640
|
-
if (value === void 0 || value === null) continue;
|
|
7641
|
-
if (base === "string") {
|
|
7642
|
-
indexStringField(
|
|
7643
|
-
state,
|
|
7644
|
-
internalId,
|
|
7645
|
-
fieldPath,
|
|
7646
|
-
value,
|
|
7647
|
-
language,
|
|
7648
|
-
options,
|
|
7649
|
-
fieldLengths,
|
|
7650
|
-
tokensByField
|
|
7651
|
-
);
|
|
7652
|
-
} else {
|
|
7653
|
-
indexStringArrayField(
|
|
7654
|
-
state,
|
|
7655
|
-
internalId,
|
|
7656
|
-
fieldPath,
|
|
7657
|
-
value,
|
|
7658
|
-
language,
|
|
7659
|
-
options,
|
|
7660
|
-
fieldLengths,
|
|
7661
|
-
tokensByField
|
|
7662
|
-
);
|
|
7663
|
-
}
|
|
7664
|
-
}
|
|
7665
|
-
state.docStore.storeRef(docId, stored.fields, fieldLengths);
|
|
7666
|
-
state.stats.addDocument(fieldLengths, tokensByField);
|
|
7667
|
-
}
|
|
7668
|
-
} finally {
|
|
7669
|
-
state.invertedIdx.endBatch();
|
|
7670
|
-
state.partialWords.forgetSpellings();
|
|
7671
|
-
}
|
|
7672
|
-
}
|
|
7673
|
-
|
|
7674
7615
|
// src/core/partition/block-bounds.ts
|
|
7675
7616
|
var ABSENT_FIELD_LENGTH_BOUND = 0;
|
|
7676
7617
|
var boundsByList = /* @__PURE__ */ new WeakMap();
|
|
@@ -9163,9 +9104,6 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
9163
9104
|
if (internalId !== void 0) recordSortValues(state, internalId, document2);
|
|
9164
9105
|
}
|
|
9165
9106
|
},
|
|
9166
|
-
rebuildTextIndex(schema, language, options) {
|
|
9167
|
-
rebuildTextIndex(state, schema, language, options);
|
|
9168
|
-
},
|
|
9169
9107
|
get(docId, projection) {
|
|
9170
9108
|
const stored = state.docStore.get(docId);
|
|
9171
9109
|
if (!stored) return void 0;
|
|
@@ -9259,6 +9197,8 @@ function yieldToEventLoop() {
|
|
|
9259
9197
|
// src/partitioning/constants.ts
|
|
9260
9198
|
var REBALANCE_CHUNK_SIZE = 1e3;
|
|
9261
9199
|
var DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE = 1e4;
|
|
9200
|
+
var TEXT_REBUILD_CHUNK_DOCUMENTS = 1e3;
|
|
9201
|
+
var TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS = 1e3;
|
|
9262
9202
|
|
|
9263
9203
|
// src/partitioning/rebalancer.ts
|
|
9264
9204
|
function createRebalancer() {
|
|
@@ -11162,18 +11102,6 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
11162
11102
|
if (frozen.length === 0) return live.encodeSegment();
|
|
11163
11103
|
return composedScratch().encodeSegment();
|
|
11164
11104
|
},
|
|
11165
|
-
rebuildTextIndex(schema, language, options) {
|
|
11166
|
-
invalidateDocFrequencies();
|
|
11167
|
-
if (frozen.length > 0) {
|
|
11168
|
-
for (const segment of frozen) {
|
|
11169
|
-
const payload = encodeSegmentState(segment);
|
|
11170
|
-
if (payload.documentCount === 0) continue;
|
|
11171
|
-
live.mergeSegmentPayload(payload, survivorDocumentsOf(segment, payload));
|
|
11172
|
-
}
|
|
11173
|
-
frozen.length = 0;
|
|
11174
|
-
}
|
|
11175
|
-
live.rebuildTextIndex(schema, language, options);
|
|
11176
|
-
},
|
|
11177
11105
|
serialize(indexName, totalPartitions, language, schema) {
|
|
11178
11106
|
if (frozen.length === 0) return live.serialize(indexName, totalPartitions, language, schema);
|
|
11179
11107
|
return composedScratch().serialize(indexName, totalPartitions, language, schema);
|
|
@@ -11417,6 +11345,81 @@ function setNestedValue(obj, path, value) {
|
|
|
11417
11345
|
current[segments[segments.length - 1]] = value;
|
|
11418
11346
|
}
|
|
11419
11347
|
|
|
11348
|
+
// src/partitioning/manager/stats.ts
|
|
11349
|
+
function aggregateStatsOf(partitions) {
|
|
11350
|
+
let totalDocuments = 0;
|
|
11351
|
+
const docFrequencies = /* @__PURE__ */ Object.create(null);
|
|
11352
|
+
const totalFieldLengths = /* @__PURE__ */ Object.create(null);
|
|
11353
|
+
for (let i = 0; i < partitions.length; i++) {
|
|
11354
|
+
const stats = partitions[i].stats;
|
|
11355
|
+
totalDocuments += stats.totalDocuments;
|
|
11356
|
+
for (const [term, freq] of Object.entries(stats.docFrequencies)) {
|
|
11357
|
+
docFrequencies[term] = (docFrequencies[term] ?? 0) + freq;
|
|
11358
|
+
}
|
|
11359
|
+
for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
|
|
11360
|
+
totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
|
|
11361
|
+
}
|
|
11362
|
+
}
|
|
11363
|
+
return { totalDocuments, docFrequencies, totalFieldLengths };
|
|
11364
|
+
}
|
|
11365
|
+
function estimateBytesOf(partitions, vecIndexes) {
|
|
11366
|
+
let total = 0;
|
|
11367
|
+
for (let i = 0; i < partitions.length; i++) {
|
|
11368
|
+
total += partitions[i].estimateMemoryBytes();
|
|
11369
|
+
}
|
|
11370
|
+
for (const [, vecIndex] of vecIndexes) {
|
|
11371
|
+
total += vecIndex.estimateMemoryBytes();
|
|
11372
|
+
}
|
|
11373
|
+
return total;
|
|
11374
|
+
}
|
|
11375
|
+
function partitionStatsOf(partitions) {
|
|
11376
|
+
return partitions.map((p, i) => ({
|
|
11377
|
+
partitionId: i,
|
|
11378
|
+
documentCount: p.count(),
|
|
11379
|
+
estimatedMemoryBytes: p.estimateMemoryBytes()
|
|
11380
|
+
}));
|
|
11381
|
+
}
|
|
11382
|
+
|
|
11383
|
+
// src/partitioning/manager/write-log.ts
|
|
11384
|
+
function createWriteLogRecorder() {
|
|
11385
|
+
let current = null;
|
|
11386
|
+
return {
|
|
11387
|
+
open() {
|
|
11388
|
+
const log = { docIds: /* @__PURE__ */ new Set(), replaced: false };
|
|
11389
|
+
current = log;
|
|
11390
|
+
return {
|
|
11391
|
+
get partitionsReplaced() {
|
|
11392
|
+
return log.replaced;
|
|
11393
|
+
},
|
|
11394
|
+
get pendingDocuments() {
|
|
11395
|
+
return log.docIds.size;
|
|
11396
|
+
},
|
|
11397
|
+
takeWrittenDocIds() {
|
|
11398
|
+
const taken = log.docIds;
|
|
11399
|
+
log.docIds = /* @__PURE__ */ new Set();
|
|
11400
|
+
return taken;
|
|
11401
|
+
},
|
|
11402
|
+
close() {
|
|
11403
|
+
if (current === log) current = null;
|
|
11404
|
+
}
|
|
11405
|
+
};
|
|
11406
|
+
},
|
|
11407
|
+
documentWritten(docId) {
|
|
11408
|
+
if (current !== null) current.docIds.add(docId);
|
|
11409
|
+
},
|
|
11410
|
+
segmentWritten(segment) {
|
|
11411
|
+
if (current === null) return;
|
|
11412
|
+
for (const internalId of segment.docStore.allInternalIds()) {
|
|
11413
|
+
const docId = segment.docStore.getExternalId(internalId);
|
|
11414
|
+
if (docId !== void 0) current.docIds.add(docId);
|
|
11415
|
+
}
|
|
11416
|
+
},
|
|
11417
|
+
partitionsReplaced() {
|
|
11418
|
+
if (current !== null) current.replaced = true;
|
|
11419
|
+
}
|
|
11420
|
+
};
|
|
11421
|
+
}
|
|
11422
|
+
|
|
11420
11423
|
// src/partitioning/manager/index.ts
|
|
11421
11424
|
function createPartitionManager(indexName, config, language, router, initialPartitionCount, vectorIndexes) {
|
|
11422
11425
|
requirePatternSearchFor(config.schema);
|
|
@@ -11428,6 +11431,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11428
11431
|
const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
|
|
11429
11432
|
let partitions = [];
|
|
11430
11433
|
const docPartitionMap = /* @__PURE__ */ new Map();
|
|
11434
|
+
const writeLog = createWriteLogRecorder();
|
|
11431
11435
|
for (let i = 0; i < count; i++) {
|
|
11432
11436
|
partitions.push(createPartitionIndex(i, trackPositions));
|
|
11433
11437
|
}
|
|
@@ -11515,9 +11519,23 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11515
11519
|
return [...partitions];
|
|
11516
11520
|
},
|
|
11517
11521
|
setPartitions(newPartitions) {
|
|
11522
|
+
writeLog.partitionsReplaced();
|
|
11518
11523
|
partitions = [...newPartitions];
|
|
11519
11524
|
rebuildDocPartitionMap();
|
|
11520
11525
|
},
|
|
11526
|
+
openWriteLog() {
|
|
11527
|
+
return writeLog.open();
|
|
11528
|
+
},
|
|
11529
|
+
adoptRebuiltPartitions(rebuilt) {
|
|
11530
|
+
if (rebuilt.length !== partitions.length) {
|
|
11531
|
+
throw new NarsilError(
|
|
11532
|
+
ErrorCodes.PARTITION_CORRUPTED,
|
|
11533
|
+
`Index "${indexName}" holds ${partitions.length} partitions, but the rebuild produced ${rebuilt.length}`,
|
|
11534
|
+
{ indexName, partitionCount: partitions.length, rebuiltCount: rebuilt.length }
|
|
11535
|
+
);
|
|
11536
|
+
}
|
|
11537
|
+
partitions = [...rebuilt];
|
|
11538
|
+
},
|
|
11521
11539
|
addPartition() {
|
|
11522
11540
|
const currentMaxPartitions = config.partitions?.maxPartitions;
|
|
11523
11541
|
if (currentMaxPartitions !== void 0 && partitions.length >= currentMaxPartitions) {
|
|
@@ -11528,11 +11546,13 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11528
11546
|
);
|
|
11529
11547
|
}
|
|
11530
11548
|
const partition = createPartitionIndex(partitions.length, trackPositions);
|
|
11549
|
+
writeLog.partitionsReplaced();
|
|
11531
11550
|
partitions.push(partition);
|
|
11532
11551
|
return partition;
|
|
11533
11552
|
},
|
|
11534
11553
|
removePartition(partitionId) {
|
|
11535
11554
|
validatePartitionId(partitionId);
|
|
11555
|
+
writeLog.partitionsReplaced();
|
|
11536
11556
|
partitions.splice(partitionId, 1);
|
|
11537
11557
|
rebuildDocPartitionMap();
|
|
11538
11558
|
},
|
|
@@ -11546,6 +11566,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11546
11566
|
if (newCount >= partitions.length) {
|
|
11547
11567
|
return;
|
|
11548
11568
|
}
|
|
11569
|
+
writeLog.partitionsReplaced();
|
|
11549
11570
|
for (const [docId, pid] of docPartitionMap) {
|
|
11550
11571
|
if (pid >= newCount) {
|
|
11551
11572
|
docPartitionMap.delete(docId);
|
|
@@ -11568,6 +11589,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11568
11589
|
const insertOpts = resolveInsertOptions(options);
|
|
11569
11590
|
partitions[pid].insert(docId, document2, config.schema, language, insertOpts);
|
|
11570
11591
|
docPartitionMap.set(docId, pid);
|
|
11592
|
+
writeLog.documentWritten(docId);
|
|
11571
11593
|
},
|
|
11572
11594
|
remove(docId) {
|
|
11573
11595
|
const pid = locateDocument(docId);
|
|
@@ -11576,11 +11598,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11576
11598
|
}
|
|
11577
11599
|
partitions[pid].remove(docId, config.schema, language, resolveInsertOptions());
|
|
11578
11600
|
docPartitionMap.delete(docId);
|
|
11579
|
-
|
|
11580
|
-
rebuildTextIndex(partitionId) {
|
|
11581
|
-
validatePartitionId(partitionId);
|
|
11582
|
-
partitions[partitionId].rebuildTextIndex(config.schema, language, resolveInsertOptions());
|
|
11583
|
-
preparePartialWords(partitionId);
|
|
11601
|
+
writeLog.documentWritten(docId);
|
|
11584
11602
|
},
|
|
11585
11603
|
beginBatchRemove() {
|
|
11586
11604
|
for (let i = 0; i < partitions.length; i++) {
|
|
@@ -11600,6 +11618,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11600
11618
|
const updateOpts = resolveInsertOptions(options);
|
|
11601
11619
|
partitions[pid].update(docId, document2, config.schema, language, updateOpts);
|
|
11602
11620
|
docPartitionMap.set(docId, pid);
|
|
11621
|
+
writeLog.documentWritten(docId);
|
|
11603
11622
|
},
|
|
11604
11623
|
get(docId, projection) {
|
|
11605
11624
|
const pid = locateDocument(docId);
|
|
@@ -11650,6 +11669,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11650
11669
|
partition.endBatch();
|
|
11651
11670
|
for (const docId of payload.docIds) {
|
|
11652
11671
|
docPartitionMap.set(docId, partitionId);
|
|
11672
|
+
writeLog.documentWritten(docId);
|
|
11653
11673
|
}
|
|
11654
11674
|
},
|
|
11655
11675
|
attachFrozenSegment(partitionId, segment) {
|
|
@@ -11664,6 +11684,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11664
11684
|
}
|
|
11665
11685
|
asCompositePartition(partitionId).attachFrozenSegment(segment);
|
|
11666
11686
|
registerSegmentDocuments(partitionId, segment);
|
|
11687
|
+
writeLog.segmentWritten(segment);
|
|
11667
11688
|
},
|
|
11668
11689
|
swapFrozenSegments(partitionId, dropSegmentIds, replacement) {
|
|
11669
11690
|
validatePartitionId(partitionId);
|
|
@@ -11681,42 +11702,19 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11681
11702
|
},
|
|
11682
11703
|
deserializePartition(partitionId, data) {
|
|
11683
11704
|
validatePartitionId(partitionId);
|
|
11705
|
+
writeLog.partitionsReplaced();
|
|
11684
11706
|
partitions[partitionId].deserialize(data, config.schema);
|
|
11685
11707
|
preparePartialWords(partitionId);
|
|
11686
11708
|
rebuildDocPartitionMapForPartition(partitionId);
|
|
11687
11709
|
},
|
|
11688
11710
|
getAggregateStats() {
|
|
11689
|
-
|
|
11690
|
-
const docFrequencies = /* @__PURE__ */ Object.create(null);
|
|
11691
|
-
const totalFieldLengths = /* @__PURE__ */ Object.create(null);
|
|
11692
|
-
for (let i = 0; i < partitions.length; i++) {
|
|
11693
|
-
const stats = partitions[i].stats;
|
|
11694
|
-
totalDocuments += stats.totalDocuments;
|
|
11695
|
-
for (const [term, freq] of Object.entries(stats.docFrequencies)) {
|
|
11696
|
-
docFrequencies[term] = (docFrequencies[term] ?? 0) + freq;
|
|
11697
|
-
}
|
|
11698
|
-
for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
|
|
11699
|
-
totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
|
|
11700
|
-
}
|
|
11701
|
-
}
|
|
11702
|
-
return { totalDocuments, docFrequencies, totalFieldLengths };
|
|
11711
|
+
return aggregateStatsOf(partitions);
|
|
11703
11712
|
},
|
|
11704
11713
|
estimateMemoryBytes() {
|
|
11705
|
-
|
|
11706
|
-
for (let i = 0; i < partitions.length; i++) {
|
|
11707
|
-
total += partitions[i].estimateMemoryBytes();
|
|
11708
|
-
}
|
|
11709
|
-
for (const [, vecIndex] of vecIndexes) {
|
|
11710
|
-
total += vecIndex.estimateMemoryBytes();
|
|
11711
|
-
}
|
|
11712
|
-
return total;
|
|
11714
|
+
return estimateBytesOf(partitions, vecIndexes);
|
|
11713
11715
|
},
|
|
11714
11716
|
getPartitionStats() {
|
|
11715
|
-
return partitions
|
|
11716
|
-
partitionId: i,
|
|
11717
|
-
documentCount: p.count(),
|
|
11718
|
-
estimatedMemoryBytes: p.estimateMemoryBytes()
|
|
11719
|
-
}));
|
|
11717
|
+
return partitionStatsOf(partitions);
|
|
11720
11718
|
},
|
|
11721
11719
|
getVectorIndexes() {
|
|
11722
11720
|
return vecIndexes;
|
|
@@ -12920,9 +12918,17 @@ function createOsqQuantizer(dimension, bits, metric, store) {
|
|
|
12920
12918
|
if (view === null || ordinal === void 0 || !view.holdsOrdinal(ordinal)) return void 0;
|
|
12921
12919
|
return view.recordAt(ordinal);
|
|
12922
12920
|
},
|
|
12921
|
+
recordAtOrdinal(ordinal) {
|
|
12922
|
+
const view = shared2();
|
|
12923
|
+
if (view === null || !view.holdsOrdinal(ordinal)) return void 0;
|
|
12924
|
+
return view.recordAt(ordinal);
|
|
12925
|
+
},
|
|
12923
12926
|
restoreRecord(docId, record) {
|
|
12924
12927
|
requireShared().restoreRecord(ordinalOf(docId), record);
|
|
12925
12928
|
},
|
|
12929
|
+
restoreRecordAtOrdinal(ordinal, record) {
|
|
12930
|
+
requireShared().restoreRecord(ordinal, record);
|
|
12931
|
+
},
|
|
12926
12932
|
restoreCentroid(centroid) {
|
|
12927
12933
|
requireShared().writeCentroid(centroid);
|
|
12928
12934
|
},
|
|
@@ -12955,6 +12961,43 @@ function createOsqQuantizer(dimension, bits, metric, store) {
|
|
|
12955
12961
|
};
|
|
12956
12962
|
}
|
|
12957
12963
|
|
|
12964
|
+
// src/vector/similarity.ts
|
|
12965
|
+
function magnitude(v) {
|
|
12966
|
+
const result = simdMagnitude(v);
|
|
12967
|
+
if (result !== null) return result;
|
|
12968
|
+
let sum = 0;
|
|
12969
|
+
for (let i = 0; i < v.length; i++) {
|
|
12970
|
+
sum += v[i] * v[i];
|
|
12971
|
+
}
|
|
12972
|
+
return Math.sqrt(sum);
|
|
12973
|
+
}
|
|
12974
|
+
function cosineSimilarityWithMagnitudes(a, b, magA, magB) {
|
|
12975
|
+
if (magA === 0 || magB === 0) return 0;
|
|
12976
|
+
return dotProduct(a, b) / (magA * magB);
|
|
12977
|
+
}
|
|
12978
|
+
function dotProduct(a, b) {
|
|
12979
|
+
const result = simdDotProduct(a, b);
|
|
12980
|
+
if (result !== null) return result;
|
|
12981
|
+
let sum = 0;
|
|
12982
|
+
for (let i = 0; i < a.length; i++) {
|
|
12983
|
+
sum += a[i] * b[i];
|
|
12984
|
+
}
|
|
12985
|
+
return sum;
|
|
12986
|
+
}
|
|
12987
|
+
function euclideanDistance(a, b) {
|
|
12988
|
+
return Math.sqrt(squaredEuclideanDistance(a, b));
|
|
12989
|
+
}
|
|
12990
|
+
function squaredEuclideanDistance(a, b) {
|
|
12991
|
+
const result = simdSquaredEuclideanDistance(a, b);
|
|
12992
|
+
if (result !== null) return result;
|
|
12993
|
+
let sum = 0;
|
|
12994
|
+
for (let i = 0; i < a.length; i++) {
|
|
12995
|
+
const diff = a[i] - b[i];
|
|
12996
|
+
sum += diff * diff;
|
|
12997
|
+
}
|
|
12998
|
+
return sum;
|
|
12999
|
+
}
|
|
13000
|
+
|
|
12958
13001
|
// src/vector/vector-store/doc-ids.ts
|
|
12959
13002
|
var encoder2 = new TextEncoder();
|
|
12960
13003
|
function appendDocId(handles, ordinal, docId) {
|
|
@@ -12974,6 +13017,78 @@ function resetDocIds(handles) {
|
|
|
12974
13017
|
Atomics.store(handles.header, STORE_DOC_ID_BYTES, 0);
|
|
12975
13018
|
}
|
|
12976
13019
|
|
|
13020
|
+
// src/vector/vector-store/ordinal-reads.ts
|
|
13021
|
+
function entryAt(store, ordinal) {
|
|
13022
|
+
return { vector: store.view.vectorAt(ordinal), magnitude: store.magnitudes[ordinal] };
|
|
13023
|
+
}
|
|
13024
|
+
function createOrdinalReads(openStore, readable) {
|
|
13025
|
+
function pairDistance(metric) {
|
|
13026
|
+
const open = openStore();
|
|
13027
|
+
if (open === null) return () => Number.POSITIVE_INFINITY;
|
|
13028
|
+
const inner = open.view.pairDistance(metric);
|
|
13029
|
+
return (ordA, ordB) => readable(ordA) && readable(ordB) ? inner(ordA, ordB) : Number.POSITIVE_INFINITY;
|
|
13030
|
+
}
|
|
13031
|
+
return {
|
|
13032
|
+
entryForOrdinal(ordinal) {
|
|
13033
|
+
const open = openStore();
|
|
13034
|
+
if (open === null || !readable(ordinal)) return void 0;
|
|
13035
|
+
return entryAt(open, ordinal);
|
|
13036
|
+
},
|
|
13037
|
+
distanceByOrdinal(ordA, ordB, metric) {
|
|
13038
|
+
const open = openStore();
|
|
13039
|
+
if (open === null || !readable(ordA) || !readable(ordB)) return Number.POSITIVE_INFINITY;
|
|
13040
|
+
return open.view.distanceByOrdinal(ordA, ordB, metric);
|
|
13041
|
+
},
|
|
13042
|
+
distanceFromArena(prepared, ordinal, metric) {
|
|
13043
|
+
const open = openStore();
|
|
13044
|
+
if (open === null || !readable(ordinal)) return Number.POSITIVE_INFINITY;
|
|
13045
|
+
return open.view.distanceFromArena(prepared, ordinal, metric);
|
|
13046
|
+
},
|
|
13047
|
+
queryDistance(prepared, metric) {
|
|
13048
|
+
const open = openStore();
|
|
13049
|
+
if (open === null) return () => Number.POSITIVE_INFINITY;
|
|
13050
|
+
const inner = open.view.queryDistance(prepared, metric);
|
|
13051
|
+
return (ordinal) => readable(ordinal) ? inner(ordinal) : Number.POSITIVE_INFINITY;
|
|
13052
|
+
},
|
|
13053
|
+
ordinalDistance(from, metric) {
|
|
13054
|
+
const pair = pairDistance(metric);
|
|
13055
|
+
return (ordinal) => pair(from, ordinal);
|
|
13056
|
+
},
|
|
13057
|
+
pairDistance
|
|
13058
|
+
};
|
|
13059
|
+
}
|
|
13060
|
+
|
|
13061
|
+
// src/vector/vector-store/listing.ts
|
|
13062
|
+
function* entriesOf(open, documents) {
|
|
13063
|
+
if (open === null) return;
|
|
13064
|
+
for (let ordinal = 0; ordinal < documents.length; ordinal++) {
|
|
13065
|
+
const docId = documents[ordinal];
|
|
13066
|
+
if (docId === void 0) continue;
|
|
13067
|
+
yield [docId, entryAt(open, ordinal)];
|
|
13068
|
+
}
|
|
13069
|
+
}
|
|
13070
|
+
function* docIdsOf(open, documents) {
|
|
13071
|
+
if (open === null) return;
|
|
13072
|
+
for (let ordinal = 0; ordinal < documents.length; ordinal++) {
|
|
13073
|
+
const docId = documents[ordinal];
|
|
13074
|
+
if (docId !== void 0) yield docId;
|
|
13075
|
+
}
|
|
13076
|
+
}
|
|
13077
|
+
function snapshotOf(open, documents, replacedOrdinals) {
|
|
13078
|
+
const slots = documents.length;
|
|
13079
|
+
const dimension = open === null ? 0 : open.handles.dimension;
|
|
13080
|
+
const vectors = new Float32Array(slots * dimension);
|
|
13081
|
+
const magnitudes = new Float64Array(slots);
|
|
13082
|
+
const docIds = new Array(slots);
|
|
13083
|
+
for (let ordinal = 0; ordinal < slots; ordinal++) {
|
|
13084
|
+
docIds[ordinal] = documents[ordinal] ?? null;
|
|
13085
|
+
if (open === null || documents[ordinal] === void 0 && !replacedOrdinals.has(ordinal)) continue;
|
|
13086
|
+
vectors.set(open.view.vectorAt(ordinal), ordinal * dimension);
|
|
13087
|
+
magnitudes[ordinal] = open.magnitudes[ordinal];
|
|
13088
|
+
}
|
|
13089
|
+
return { dimension, slots, vectors, magnitudes, docIds, replacedOrdinals: [...replacedOrdinals] };
|
|
13090
|
+
}
|
|
13091
|
+
|
|
12977
13092
|
// src/vector/ordinal-filter.ts
|
|
12978
13093
|
function createOrdinalFilter(slots) {
|
|
12979
13094
|
return { bits: new Uint8Array(slots + 7 >>> 3), count: 0 };
|
|
@@ -13010,43 +13125,6 @@ function* ordinalFilterValues(filter) {
|
|
|
13010
13125
|
}
|
|
13011
13126
|
}
|
|
13012
13127
|
|
|
13013
|
-
// src/vector/similarity.ts
|
|
13014
|
-
function magnitude(v) {
|
|
13015
|
-
const result = simdMagnitude(v);
|
|
13016
|
-
if (result !== null) return result;
|
|
13017
|
-
let sum = 0;
|
|
13018
|
-
for (let i = 0; i < v.length; i++) {
|
|
13019
|
-
sum += v[i] * v[i];
|
|
13020
|
-
}
|
|
13021
|
-
return Math.sqrt(sum);
|
|
13022
|
-
}
|
|
13023
|
-
function cosineSimilarityWithMagnitudes(a, b, magA, magB) {
|
|
13024
|
-
if (magA === 0 || magB === 0) return 0;
|
|
13025
|
-
return dotProduct(a, b) / (magA * magB);
|
|
13026
|
-
}
|
|
13027
|
-
function dotProduct(a, b) {
|
|
13028
|
-
const result = simdDotProduct(a, b);
|
|
13029
|
-
if (result !== null) return result;
|
|
13030
|
-
let sum = 0;
|
|
13031
|
-
for (let i = 0; i < a.length; i++) {
|
|
13032
|
-
sum += a[i] * b[i];
|
|
13033
|
-
}
|
|
13034
|
-
return sum;
|
|
13035
|
-
}
|
|
13036
|
-
function euclideanDistance(a, b) {
|
|
13037
|
-
return Math.sqrt(squaredEuclideanDistance(a, b));
|
|
13038
|
-
}
|
|
13039
|
-
function squaredEuclideanDistance(a, b) {
|
|
13040
|
-
const result = simdSquaredEuclideanDistance(a, b);
|
|
13041
|
-
if (result !== null) return result;
|
|
13042
|
-
let sum = 0;
|
|
13043
|
-
for (let i = 0; i < a.length; i++) {
|
|
13044
|
-
const diff = a[i] - b[i];
|
|
13045
|
-
sum += diff * diff;
|
|
13046
|
-
}
|
|
13047
|
-
return sum;
|
|
13048
|
-
}
|
|
13049
|
-
|
|
13050
13128
|
// src/vector/vector-store/vector-file.browser.ts
|
|
13051
13129
|
function filesystemUnavailable() {
|
|
13052
13130
|
throw new NarsilError(
|
|
@@ -13480,13 +13558,14 @@ function writeVector(store, ordinal, vector) {
|
|
|
13480
13558
|
block.float32(base + vector.length).set(vector, base);
|
|
13481
13559
|
store.magnitudes[ordinal] = magnitude(vector);
|
|
13482
13560
|
}
|
|
13483
|
-
function blockFullyCold(store, blockIndex, documents) {
|
|
13561
|
+
function blockFullyCold(store, blockIndex, documents, replacedOrdinals) {
|
|
13484
13562
|
const capacity = store.handles.layout.capacity;
|
|
13485
13563
|
const slots = Atomics.load(store.handles.header, STORE_SLOTS);
|
|
13486
13564
|
const end = Math.min(slots, (blockIndex + 1) * capacity);
|
|
13487
13565
|
if (end < (blockIndex + 1) * capacity) return false;
|
|
13488
13566
|
for (let ordinal = blockIndex * capacity; ordinal < end; ordinal++) {
|
|
13489
|
-
|
|
13567
|
+
const held = documents[ordinal] !== void 0 || replacedOrdinals.has(ordinal);
|
|
13568
|
+
if (held && store.diskFile[ordinal] === IN_MEMORY) return false;
|
|
13490
13569
|
}
|
|
13491
13570
|
return true;
|
|
13492
13571
|
}
|
|
@@ -13499,10 +13578,11 @@ function fileHoldsSameBytes(store, ordinal, location) {
|
|
|
13499
13578
|
for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
|
|
13500
13579
|
return true;
|
|
13501
13580
|
}
|
|
13502
|
-
function forgetUnreferencedFiles(store, documents) {
|
|
13581
|
+
function forgetUnreferencedFiles(store, documents, replacedOrdinals) {
|
|
13503
13582
|
const referenced = /* @__PURE__ */ new Set();
|
|
13504
13583
|
for (let ordinal = 0; ordinal < documents.length; ordinal++) {
|
|
13505
|
-
|
|
13584
|
+
const held = documents[ordinal] !== void 0 || replacedOrdinals.has(ordinal);
|
|
13585
|
+
if (held && store.diskFile[ordinal] !== IN_MEMORY) {
|
|
13506
13586
|
referenced.add(store.diskFile[ordinal]);
|
|
13507
13587
|
}
|
|
13508
13588
|
}
|
|
@@ -13543,15 +13623,20 @@ function createVectorStore(options) {
|
|
|
13543
13623
|
const blockBytes2 = options?.blockBytes ?? BLOCK_MAX_BYTES;
|
|
13544
13624
|
const docToOrd = /* @__PURE__ */ new Map();
|
|
13545
13625
|
const ordToDoc = [];
|
|
13626
|
+
const replacedOrdinals = /* @__PURE__ */ new Set();
|
|
13546
13627
|
let unknownPartitions = 0;
|
|
13547
13628
|
let liveCount = 0;
|
|
13548
13629
|
let open = options?.dimension === void 0 ? null : openHandles(options.dimension, codeBits, blockBytes2);
|
|
13549
13630
|
function forgetDocuments() {
|
|
13550
13631
|
docToOrd.clear();
|
|
13551
13632
|
ordToDoc.length = 0;
|
|
13633
|
+
replacedOrdinals.clear();
|
|
13552
13634
|
unknownPartitions = 0;
|
|
13553
13635
|
liveCount = 0;
|
|
13554
13636
|
}
|
|
13637
|
+
function readable(ordinal) {
|
|
13638
|
+
return ordToDoc[ordinal] !== void 0 || replacedOrdinals.has(ordinal);
|
|
13639
|
+
}
|
|
13555
13640
|
function requireOpen() {
|
|
13556
13641
|
if (open === null) {
|
|
13557
13642
|
throw new NarsilError(ErrorCodes.VECTOR_DIMENSION_MISMATCH, "The vector store holds no vector yet");
|
|
@@ -13568,11 +13653,27 @@ function createVectorStore(options) {
|
|
|
13568
13653
|
Atomics.store(store.present, ordinal, 0);
|
|
13569
13654
|
if (store.partitions[ordinal] === UNKNOWN_PARTITION) unknownPartitions -= 1;
|
|
13570
13655
|
}
|
|
13571
|
-
function
|
|
13656
|
+
function keepReplacedOrdinal(store, ordinal) {
|
|
13657
|
+
ordToDoc[ordinal] = void 0;
|
|
13658
|
+
if (store.partitions[ordinal] === UNKNOWN_PARTITION) unknownPartitions -= 1;
|
|
13659
|
+
store.partitions[ordinal] = NO_PARTITION;
|
|
13660
|
+
replacedOrdinals.add(ordinal);
|
|
13661
|
+
}
|
|
13662
|
+
function restoreReplacedVector(store, ordinal, vector) {
|
|
13663
|
+
store.diskFile[ordinal] = IN_MEMORY;
|
|
13664
|
+
writeVector(store, ordinal, vector);
|
|
13665
|
+
Atomics.store(store.present, ordinal, 1);
|
|
13666
|
+
replacedOrdinals.add(ordinal);
|
|
13667
|
+
}
|
|
13668
|
+
function claimSlot(store, docId, cold) {
|
|
13572
13669
|
const ordinal = Atomics.load(store.handles.header, STORE_SLOTS);
|
|
13573
13670
|
ensureOrdinal(store, ordinal, cold);
|
|
13574
13671
|
store.diskFile[ordinal] = IN_MEMORY;
|
|
13575
13672
|
appendDocId(store.handles, ordinal, docId);
|
|
13673
|
+
return ordinal;
|
|
13674
|
+
}
|
|
13675
|
+
function claimOrdinal(store, docId, cold, partitionId) {
|
|
13676
|
+
const ordinal = claimSlot(store, docId, cold);
|
|
13576
13677
|
recordPartition(store, ordinal, partitionId);
|
|
13577
13678
|
ordToDoc[ordinal] = docId;
|
|
13578
13679
|
docToOrd.set(docId, ordinal);
|
|
@@ -13582,25 +13683,30 @@ function createVectorStore(options) {
|
|
|
13582
13683
|
Atomics.store(store.handles.header, STORE_SLOTS, ordinal + 1);
|
|
13583
13684
|
Atomics.store(store.present, ordinal, 1);
|
|
13584
13685
|
}
|
|
13686
|
+
function publishOnDisk(store, ordinal, vectorMagnitude, location) {
|
|
13687
|
+
store.magnitudes[ordinal] = vectorMagnitude;
|
|
13688
|
+
store.diskOffset[ordinal] = location.offset;
|
|
13689
|
+
store.diskFile[ordinal] = location.fileIndex;
|
|
13690
|
+
publishOrdinal(store, ordinal);
|
|
13691
|
+
}
|
|
13585
13692
|
function appendOrdinal(store, docId, vector, partitionId) {
|
|
13586
13693
|
const ordinal = claimOrdinal(store, docId, false, partitionId);
|
|
13587
13694
|
writeVector(store, ordinal, vector);
|
|
13588
13695
|
publishOrdinal(store, ordinal);
|
|
13589
13696
|
return ordinal;
|
|
13590
13697
|
}
|
|
13591
|
-
function settleReplacement(store, ordinal, previous) {
|
|
13698
|
+
function settleReplacement(store, ordinal, previous, keepsReplacedVector) {
|
|
13592
13699
|
if (previous !== void 0) {
|
|
13593
|
-
|
|
13700
|
+
if (keepsReplacedVector) keepReplacedOrdinal(store, previous);
|
|
13701
|
+
else retireOrdinal(store, previous);
|
|
13594
13702
|
return ordinal;
|
|
13595
13703
|
}
|
|
13596
13704
|
liveCount++;
|
|
13597
13705
|
Atomics.add(store.handles.header, STORE_LIVE_COUNT, 1);
|
|
13598
13706
|
return ordinal;
|
|
13599
13707
|
}
|
|
13600
|
-
function entryAt(store, ordinal) {
|
|
13601
|
-
return { vector: store.view.vectorAt(ordinal), magnitude: store.magnitudes[ordinal] };
|
|
13602
|
-
}
|
|
13603
13708
|
return {
|
|
13709
|
+
...createOrdinalReads(() => open, readable),
|
|
13604
13710
|
get size() {
|
|
13605
13711
|
return liveCount;
|
|
13606
13712
|
},
|
|
@@ -13619,7 +13725,10 @@ function createVectorStore(options) {
|
|
|
13619
13725
|
get view() {
|
|
13620
13726
|
return requireOpen().view;
|
|
13621
13727
|
},
|
|
13622
|
-
|
|
13728
|
+
get replacedVectorCount() {
|
|
13729
|
+
return replacedOrdinals.size;
|
|
13730
|
+
},
|
|
13731
|
+
insert(docId, vector, partitionId, keepsReplacedVector = false) {
|
|
13623
13732
|
if (open === null) open = openHandles(vector.length, codeBits, blockBytes2);
|
|
13624
13733
|
const store = open;
|
|
13625
13734
|
if (vector.length !== store.handles.dimension) {
|
|
@@ -13631,18 +13740,37 @@ function createVectorStore(options) {
|
|
|
13631
13740
|
}
|
|
13632
13741
|
const previous = docToOrd.get(docId);
|
|
13633
13742
|
const ordinal = appendOrdinal(store, docId, vector, partitionId);
|
|
13634
|
-
return settleReplacement(store, ordinal, previous);
|
|
13743
|
+
return settleReplacement(store, ordinal, previous, keepsReplacedVector);
|
|
13744
|
+
},
|
|
13745
|
+
releaseReplacedVectors() {
|
|
13746
|
+
const released = [...replacedOrdinals];
|
|
13747
|
+
replacedOrdinals.clear();
|
|
13748
|
+
if (open !== null) for (const ordinal of released) Atomics.store(open.present, ordinal, 0);
|
|
13749
|
+
return released;
|
|
13635
13750
|
},
|
|
13636
13751
|
insertCold(docId, vectorMagnitude, location, partitionId) {
|
|
13637
13752
|
const store = requireOpen();
|
|
13638
13753
|
const previous = docToOrd.get(docId);
|
|
13639
13754
|
Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
|
|
13640
13755
|
const ordinal = claimOrdinal(store, docId, true, partitionId);
|
|
13641
|
-
store
|
|
13642
|
-
store
|
|
13643
|
-
|
|
13756
|
+
publishOnDisk(store, ordinal, vectorMagnitude, location);
|
|
13757
|
+
return settleReplacement(store, ordinal, previous, false);
|
|
13758
|
+
},
|
|
13759
|
+
insertReplaced(docId, vector, location) {
|
|
13760
|
+
if (open === null) open = openHandles(vector.length, codeBits, blockBytes2);
|
|
13761
|
+
const store = open;
|
|
13762
|
+
if (location !== void 0) Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
|
|
13763
|
+
const ordinal = claimSlot(store, docId, location !== void 0);
|
|
13764
|
+
store.partitions[ordinal] = NO_PARTITION;
|
|
13765
|
+
ordToDoc[ordinal] = void 0;
|
|
13766
|
+
replacedOrdinals.add(ordinal);
|
|
13767
|
+
if (location !== void 0) {
|
|
13768
|
+
publishOnDisk(store, ordinal, magnitude(vector), location);
|
|
13769
|
+
return ordinal;
|
|
13770
|
+
}
|
|
13771
|
+
writeVector(store, ordinal, vector);
|
|
13644
13772
|
publishOrdinal(store, ordinal);
|
|
13645
|
-
return
|
|
13773
|
+
return ordinal;
|
|
13646
13774
|
},
|
|
13647
13775
|
addVectorFile(path) {
|
|
13648
13776
|
const store = requireOpen();
|
|
@@ -13669,11 +13797,11 @@ function createVectorStore(options) {
|
|
|
13669
13797
|
const store = requireOpen();
|
|
13670
13798
|
let released = 0;
|
|
13671
13799
|
for (let index = 0; index < store.handles.blocks.length; index++) {
|
|
13672
|
-
if (store.handles.blocks[index] === null || !blockFullyCold(store, index, ordToDoc)) continue;
|
|
13800
|
+
if (store.handles.blocks[index] === null || !blockFullyCold(store, index, ordToDoc, replacedOrdinals)) continue;
|
|
13673
13801
|
store.handles.blocks[index] = null;
|
|
13674
13802
|
released += 1;
|
|
13675
13803
|
}
|
|
13676
|
-
if (released + forgetUnreferencedFiles(store, ordToDoc) > 0) relayout(store);
|
|
13804
|
+
if (released + forgetUnreferencedFiles(store, ordToDoc, replacedOrdinals) > 0) relayout(store);
|
|
13677
13805
|
return released;
|
|
13678
13806
|
},
|
|
13679
13807
|
setPartition(docId, partitionId) {
|
|
@@ -13717,21 +13845,8 @@ function createVectorStore(options) {
|
|
|
13717
13845
|
holdsOrdinal(ordinal) {
|
|
13718
13846
|
return ordinal >= 0 && ordinal < ordToDoc.length && ordToDoc[ordinal] !== void 0;
|
|
13719
13847
|
},
|
|
13720
|
-
|
|
13721
|
-
|
|
13722
|
-
for (let ordinal = 0; ordinal < ordToDoc.length; ordinal++) {
|
|
13723
|
-
const docId = ordToDoc[ordinal];
|
|
13724
|
-
if (docId === void 0) continue;
|
|
13725
|
-
yield [docId, entryAt(open, ordinal)];
|
|
13726
|
-
}
|
|
13727
|
-
},
|
|
13728
|
-
*docIds() {
|
|
13729
|
-
if (open === null) return;
|
|
13730
|
-
for (let ordinal = 0; ordinal < ordToDoc.length; ordinal++) {
|
|
13731
|
-
const docId = ordToDoc[ordinal];
|
|
13732
|
-
if (docId !== void 0) yield docId;
|
|
13733
|
-
}
|
|
13734
|
-
},
|
|
13848
|
+
entries: () => entriesOf(open, ordToDoc),
|
|
13849
|
+
docIds: () => docIdsOf(open, ordToDoc),
|
|
13735
13850
|
clear() {
|
|
13736
13851
|
forgetDocuments();
|
|
13737
13852
|
if (open === null) return;
|
|
@@ -13763,59 +13878,16 @@ function createVectorStore(options) {
|
|
|
13763
13878
|
docIdForOrdinal(ordinal) {
|
|
13764
13879
|
return ordToDoc[ordinal];
|
|
13765
13880
|
},
|
|
13766
|
-
entryForOrdinal(ordinal) {
|
|
13767
|
-
if (open === null || ordinal < 0 || ordinal >= ordToDoc.length || ordToDoc[ordinal] === void 0) {
|
|
13768
|
-
return void 0;
|
|
13769
|
-
}
|
|
13770
|
-
return entryAt(open, ordinal);
|
|
13771
|
-
},
|
|
13772
|
-
distanceByOrdinal(ordA, ordB, metric) {
|
|
13773
|
-
if (open === null || ordToDoc[ordA] === void 0 || ordToDoc[ordB] === void 0) {
|
|
13774
|
-
return Number.POSITIVE_INFINITY;
|
|
13775
|
-
}
|
|
13776
|
-
return open.view.distanceByOrdinal(ordA, ordB, metric);
|
|
13777
|
-
},
|
|
13778
13881
|
prepareQueryArena(query) {
|
|
13779
13882
|
return open === null ? null : open.view.prepareQueryArena(query);
|
|
13780
13883
|
},
|
|
13781
|
-
|
|
13782
|
-
if (open === null || ordToDoc[ordinal] === void 0) return Number.POSITIVE_INFINITY;
|
|
13783
|
-
return open.view.distanceFromArena(prepared, ordinal, metric);
|
|
13784
|
-
},
|
|
13785
|
-
queryDistance(prepared, metric) {
|
|
13786
|
-
if (open === null) return () => Number.POSITIVE_INFINITY;
|
|
13787
|
-
const inner = open.view.queryDistance(prepared, metric);
|
|
13788
|
-
return (ordinal) => ordToDoc[ordinal] === void 0 ? Number.POSITIVE_INFINITY : inner(ordinal);
|
|
13789
|
-
},
|
|
13790
|
-
ordinalDistance(from, metric) {
|
|
13791
|
-
const pair = this.pairDistance(metric);
|
|
13792
|
-
return (ordinal) => pair(from, ordinal);
|
|
13793
|
-
},
|
|
13794
|
-
pairDistance(metric) {
|
|
13795
|
-
if (open === null) return () => Number.POSITIVE_INFINITY;
|
|
13796
|
-
const inner = open.view.pairDistance(metric);
|
|
13797
|
-
return (ordA, ordB) => ordToDoc[ordA] === void 0 || ordToDoc[ordB] === void 0 ? Number.POSITIVE_INFINITY : inner(ordA, ordB);
|
|
13798
|
-
},
|
|
13799
|
-
exportSnapshot() {
|
|
13800
|
-
const slots = ordToDoc.length;
|
|
13801
|
-
const dimension = open === null ? 0 : open.handles.dimension;
|
|
13802
|
-
const vectors = new Float32Array(slots * dimension);
|
|
13803
|
-
const magnitudes = new Float64Array(slots);
|
|
13804
|
-
const docIds = new Array(slots);
|
|
13805
|
-
for (let ordinal = 0; ordinal < slots; ordinal++) {
|
|
13806
|
-
const docId = ordToDoc[ordinal];
|
|
13807
|
-
docIds[ordinal] = docId ?? null;
|
|
13808
|
-
if (docId === void 0 || open === null) continue;
|
|
13809
|
-
vectors.set(open.view.vectorAt(ordinal), ordinal * dimension);
|
|
13810
|
-
magnitudes[ordinal] = open.magnitudes[ordinal];
|
|
13811
|
-
}
|
|
13812
|
-
return { dimension, slots, vectors, magnitudes, docIds };
|
|
13813
|
-
},
|
|
13884
|
+
exportSnapshot: () => snapshotOf(open, ordToDoc, replacedOrdinals),
|
|
13814
13885
|
restoreSnapshot(snapshot) {
|
|
13815
13886
|
this.clear();
|
|
13816
13887
|
if (snapshot.dimension === 0 || snapshot.slots === 0) return;
|
|
13817
13888
|
if (open === null) open = openHandles(snapshot.dimension, codeBits, blockBytes2);
|
|
13818
13889
|
const store = open;
|
|
13890
|
+
const replaced = new Set(snapshot.replacedOrdinals);
|
|
13819
13891
|
for (let ordinal = 0; ordinal < snapshot.slots; ordinal++) {
|
|
13820
13892
|
const docId = snapshot.docIds[ordinal];
|
|
13821
13893
|
const vector = snapshot.vectors.subarray(ordinal * snapshot.dimension, (ordinal + 1) * snapshot.dimension);
|
|
@@ -13823,6 +13895,7 @@ function createVectorStore(options) {
|
|
|
13823
13895
|
ensureOrdinal(store, ordinal, false);
|
|
13824
13896
|
ordToDoc[ordinal] = void 0;
|
|
13825
13897
|
store.partitions[ordinal] = NO_PARTITION;
|
|
13898
|
+
if (replaced.has(ordinal)) restoreReplacedVector(store, ordinal, vector);
|
|
13826
13899
|
Atomics.store(store.handles.header, STORE_SLOTS, ordinal + 1);
|
|
13827
13900
|
continue;
|
|
13828
13901
|
}
|
|
@@ -14647,7 +14720,7 @@ function readNeighborsLocked(state, ord, layer) {
|
|
|
14647
14720
|
if (nodeReadHeld(state.locks, ord, version)) return count;
|
|
14648
14721
|
}
|
|
14649
14722
|
}
|
|
14650
|
-
function searchLayer(state, qVec, qMag, ef, layer, metric,
|
|
14723
|
+
function searchLayer(state, qVec, qMag, ef, layer, metric, excludeTombstones, distFn, results) {
|
|
14651
14724
|
const getDistance = distFn ?? ((ord) => queryDistanceByOrd(state, qVec, qMag, ord, metric));
|
|
14652
14725
|
const workspace = state.workspace;
|
|
14653
14726
|
const frontier = workspace.frontier;
|
|
@@ -14663,10 +14736,10 @@ function searchLayer(state, qVec, qMag, ef, layer, metric, skipTombstones, distF
|
|
|
14663
14736
|
if (state.visited[epOrd] === stamp) continue;
|
|
14664
14737
|
state.visited[epOrd] = stamp;
|
|
14665
14738
|
if (!nodeExists(state, epOrd)) continue;
|
|
14666
|
-
if (skipTombstones && isTombstoned(state, epOrd)) continue;
|
|
14667
14739
|
const dist = getDistance(epOrd);
|
|
14668
14740
|
if (dist === Number.POSITIVE_INFINITY) continue;
|
|
14669
14741
|
pushHeap(frontier, epOrd, dist);
|
|
14742
|
+
if (excludeTombstones && isTombstoned(state, epOrd)) continue;
|
|
14670
14743
|
pushHeap(found, epOrd, dist);
|
|
14671
14744
|
if (found.size > ef) {
|
|
14672
14745
|
popHeap(found);
|
|
@@ -14674,19 +14747,19 @@ function searchLayer(state, qVec, qMag, ef, layer, metric, skipTombstones, distF
|
|
|
14674
14747
|
}
|
|
14675
14748
|
if (found.size > 0) furthestDist = found.distances[0];
|
|
14676
14749
|
while (popHeap(frontier)) {
|
|
14677
|
-
if (frontier.topDistance > furthestDist) break;
|
|
14750
|
+
if (frontier.topDistance > furthestDist && found.size >= ef) break;
|
|
14678
14751
|
const count = readNeighborsLocked(state, frontier.topOrd, layer);
|
|
14679
14752
|
for (let i = 0; i < count; i++) {
|
|
14680
14753
|
const neighborOrd = scratch[i];
|
|
14681
14754
|
ensureVisited(state, neighborOrd + 1);
|
|
14682
14755
|
if (state.visited[neighborOrd] === stamp) continue;
|
|
14683
14756
|
state.visited[neighborOrd] = stamp;
|
|
14684
|
-
if (skipTombstones && isTombstoned(state, neighborOrd)) continue;
|
|
14685
14757
|
if (!nodeExists(state, neighborOrd)) continue;
|
|
14686
14758
|
const dist = getDistance(neighborOrd);
|
|
14687
14759
|
if (dist === Number.POSITIVE_INFINITY) continue;
|
|
14688
14760
|
if (dist < furthestDist || found.size < ef) {
|
|
14689
14761
|
pushHeap(frontier, neighborOrd, dist);
|
|
14762
|
+
if (excludeTombstones && isTombstoned(state, neighborOrd)) continue;
|
|
14690
14763
|
pushHeap(found, neighborOrd, dist);
|
|
14691
14764
|
if (found.size > ef) {
|
|
14692
14765
|
popHeap(found);
|
|
@@ -14962,13 +15035,6 @@ function markTombstone(state, ord) {
|
|
|
14962
15035
|
if (Atomics.compareExchange(state.tombstones, ord, 0, 1) === 0) {
|
|
14963
15036
|
Atomics.add(state.header, GRAPH_TOMBSTONE_COUNT, 1);
|
|
14964
15037
|
}
|
|
14965
|
-
if (entryPointOf(state) !== ord) return;
|
|
14966
|
-
lockEntry(state.locks);
|
|
14967
|
-
try {
|
|
14968
|
-
if (entryPointOf(state) === ord) setEntryPoint(state, highestNode(state, false));
|
|
14969
|
-
} finally {
|
|
14970
|
-
unlockEntry(state.locks);
|
|
14971
|
-
}
|
|
14972
15038
|
}
|
|
14973
15039
|
function compactTombstones(state) {
|
|
14974
15040
|
if (Atomics.load(state.header, GRAPH_TOMBSTONE_COUNT) === 0) return;
|
|
@@ -15042,7 +15108,7 @@ function valuesOf(bytes) {
|
|
|
15042
15108
|
return new Uint32Array(copy.buffer, 0, copy.byteLength / BYTES_PER_VALUE);
|
|
15043
15109
|
}
|
|
15044
15110
|
function savedTopLayer(state, ordinal) {
|
|
15045
|
-
if (ordinal === NO_ORDINAL
|
|
15111
|
+
if (ordinal === NO_ORDINAL) return NO_LAYER;
|
|
15046
15112
|
return Math.min(nodeMaxLayer(state, ordinal), MAX_LAYER_CAP);
|
|
15047
15113
|
}
|
|
15048
15114
|
function neighbourValueCount(state, topLayers, start, end) {
|
|
@@ -15108,6 +15174,25 @@ function serializeNumberedGraphParts(state, numberOfOrdinal, ordinalOfNumber, pa
|
|
|
15108
15174
|
}
|
|
15109
15175
|
return parts;
|
|
15110
15176
|
}
|
|
15177
|
+
function graphWithoutNodes(part, firstNumber, numbers) {
|
|
15178
|
+
const values = valuesOf(part.neighbours);
|
|
15179
|
+
const levels = part.levels.slice();
|
|
15180
|
+
const kept = new Uint32Array(values.length);
|
|
15181
|
+
let cursor = 0;
|
|
15182
|
+
let length = 0;
|
|
15183
|
+
for (let at = 0; at < part.levels.length; at++) {
|
|
15184
|
+
const start = cursor;
|
|
15185
|
+
for (let layer = 0; layer < part.levels[at]; layer++) cursor += values[cursor] + 1;
|
|
15186
|
+
if (numbers.has(firstNumber + at)) {
|
|
15187
|
+
levels[at] = 0;
|
|
15188
|
+
continue;
|
|
15189
|
+
}
|
|
15190
|
+
kept.set(values.subarray(start, cursor), length);
|
|
15191
|
+
length += cursor - start;
|
|
15192
|
+
}
|
|
15193
|
+
const entryPoint = part.entryPoint !== null && numbers.has(part.entryPoint) ? null : part.entryPoint;
|
|
15194
|
+
return { ...part, entryPoint, levels, neighbours: littleEndianBytesOf(kept.subarray(0, length)) };
|
|
15195
|
+
}
|
|
15111
15196
|
function endsInsideAList() {
|
|
15112
15197
|
throw new NarsilError(
|
|
15113
15198
|
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
@@ -15374,7 +15459,7 @@ function traverse(state, query, k, searchMetric, options) {
|
|
|
15374
15459
|
if (entryPoint === -1) return { candidates, depth: useQuantized ? depth : 0, arenaQuery, qMag, native: null };
|
|
15375
15460
|
setSingleEntryPoint(workspace, entryPoint);
|
|
15376
15461
|
for (let layer = topLayerOf(state); layer >= 1; layer--) {
|
|
15377
|
-
searchLayer(state, query, qMag, 1, layer, searchMetric,
|
|
15462
|
+
searchLayer(state, query, qMag, 1, layer, searchMetric, false, distFn, candidates);
|
|
15378
15463
|
if (candidates.size > 0) {
|
|
15379
15464
|
setSingleEntryPoint(workspace, candidates.ords[0]);
|
|
15380
15465
|
}
|
|
@@ -15682,6 +15767,8 @@ function createHNSWIndex(dimension, store, config, quantizer, handles) {
|
|
|
15682
15767
|
if (ord !== void 0) markTombstone(state, ord);
|
|
15683
15768
|
},
|
|
15684
15769
|
markTombstoneOrdinal: (ordinal) => markTombstone(state, ordinal),
|
|
15770
|
+
holdsNode: (ordinal) => hasNode(state.adjacency, ordinal),
|
|
15771
|
+
holdsTombstone: (ordinal) => hasNode(state.adjacency, ordinal) && isTombstoned(state, ordinal),
|
|
15685
15772
|
has: (docId) => {
|
|
15686
15773
|
const ord = store.getOrdinal(docId);
|
|
15687
15774
|
return ord !== void 0 && hasNode(state.adjacency, ord) && !isTombstoned(state, ord);
|
|
@@ -15751,6 +15838,9 @@ function emptyFieldBeforeRestore(state) {
|
|
|
15751
15838
|
state.savedFiles = [];
|
|
15752
15839
|
state.savedSignature = null;
|
|
15753
15840
|
}
|
|
15841
|
+
function releaseReplacedVectors(state) {
|
|
15842
|
+
for (const ordinal of state.store.releaseReplacedVectors()) state.osq?.removeOrdinal(ordinal);
|
|
15843
|
+
}
|
|
15754
15844
|
function adoptGraph(state, graph) {
|
|
15755
15845
|
if (graph === null && state.hnsw !== null) {
|
|
15756
15846
|
state.hnsw.clear();
|
|
@@ -16310,9 +16400,7 @@ function captureCloneCodes(state) {
|
|
|
16310
16400
|
const records = new Uint8Array(slots * layout.slotStride);
|
|
16311
16401
|
const present = new Uint8Array(slots);
|
|
16312
16402
|
for (let ordinal = 0; ordinal < slots; ordinal++) {
|
|
16313
|
-
const
|
|
16314
|
-
if (docId === void 0) continue;
|
|
16315
|
-
const record = quantizer.recordOf(docId);
|
|
16403
|
+
const record = quantizer.recordAtOrdinal(ordinal);
|
|
16316
16404
|
if (record === void 0) continue;
|
|
16317
16405
|
records.set(record, ordinal * layout.slotStride);
|
|
16318
16406
|
present[ordinal] = 1;
|
|
@@ -16449,7 +16537,7 @@ async function dispatcherFor(state, graph) {
|
|
|
16449
16537
|
workerCount: Math.max(1, host.workerCount)
|
|
16450
16538
|
};
|
|
16451
16539
|
}
|
|
16452
|
-
if (liveSize(state) < WORKER_COPY_MIN_VECTORS) return null;
|
|
16540
|
+
if (liveSize(state) < WORKER_COPY_MIN_VECTORS || !sharedMemoryAvailable()) return null;
|
|
16453
16541
|
const handle = await shareGraph(state, graph, graph === state.hnsw);
|
|
16454
16542
|
const pool = state.workerCopyPool;
|
|
16455
16543
|
if (handle === null || pool === null || state.sharedHandles.get(graph)?.mode !== "shared") return null;
|
|
@@ -16539,12 +16627,40 @@ async function insertLocally(state, graph, docIds, admit) {
|
|
|
16539
16627
|
state.buffer.delete(docId);
|
|
16540
16628
|
count += 1;
|
|
16541
16629
|
if (count % BUILD_CHUNK_SIZE === 0) {
|
|
16630
|
+
noteWrite(state);
|
|
16542
16631
|
await yieldToEventLoop2();
|
|
16543
16632
|
}
|
|
16544
16633
|
}
|
|
16634
|
+
if (count > 0) noteWrite(state);
|
|
16545
16635
|
return !state.disposed;
|
|
16546
16636
|
}
|
|
16547
16637
|
|
|
16638
|
+
// src/vector/vector-index/dead-bits.ts
|
|
16639
|
+
var NO_LIVE_VECTOR_AT_POSITION = -1;
|
|
16640
|
+
function deadBytesFor(count) {
|
|
16641
|
+
return Math.ceil(count / 8);
|
|
16642
|
+
}
|
|
16643
|
+
function isDead(dead, position) {
|
|
16644
|
+
return dead !== null && (dead[position >> 3] >> (position & 7) & 1) === 1;
|
|
16645
|
+
}
|
|
16646
|
+
function countDead(dead, count) {
|
|
16647
|
+
if (dead === null) return 0;
|
|
16648
|
+
let found = 0;
|
|
16649
|
+
for (let position = 0; position < count; position++) {
|
|
16650
|
+
if (isDead(dead, position)) found += 1;
|
|
16651
|
+
}
|
|
16652
|
+
return found;
|
|
16653
|
+
}
|
|
16654
|
+
function deadBitsWhere(count, isDeadAt) {
|
|
16655
|
+
let dead = null;
|
|
16656
|
+
for (let position = 0; position < count; position++) {
|
|
16657
|
+
if (!isDeadAt(position)) continue;
|
|
16658
|
+
dead ??= new Uint8Array(deadBytesFor(count));
|
|
16659
|
+
dead[position >> 3] |= 1 << (position & 7);
|
|
16660
|
+
}
|
|
16661
|
+
return dead;
|
|
16662
|
+
}
|
|
16663
|
+
|
|
16548
16664
|
// src/vector/vector-index/disk.ts
|
|
16549
16665
|
function readsFromDisk(state) {
|
|
16550
16666
|
return state.storage === "disk";
|
|
@@ -16555,6 +16671,7 @@ function holdsVectorsOnDisk(state) {
|
|
|
16555
16671
|
function* locationsOf(state, layout) {
|
|
16556
16672
|
const stride = state.dimension * 4;
|
|
16557
16673
|
for (let i = 0; i < layout.docIds.length; i++) {
|
|
16674
|
+
if (isDead(layout.dead, i)) continue;
|
|
16558
16675
|
yield [layout.docIds[i], { path: layout.path, offset: layout.vectorsOffset + i * stride }];
|
|
16559
16676
|
}
|
|
16560
16677
|
}
|
|
@@ -16945,32 +17062,6 @@ function graphPartsOfEarlierPayload(payload) {
|
|
|
16945
17062
|
};
|
|
16946
17063
|
}
|
|
16947
17064
|
|
|
16948
|
-
// src/vector/vector-index/dead-bits.ts
|
|
16949
|
-
var NO_LIVE_VECTOR_AT_POSITION = -1;
|
|
16950
|
-
function deadBytesFor(count) {
|
|
16951
|
-
return Math.ceil(count / 8);
|
|
16952
|
-
}
|
|
16953
|
-
function isDead(dead, position) {
|
|
16954
|
-
return dead !== null && (dead[position >> 3] >> (position & 7) & 1) === 1;
|
|
16955
|
-
}
|
|
16956
|
-
function countDead(dead, count) {
|
|
16957
|
-
if (dead === null) return 0;
|
|
16958
|
-
let found = 0;
|
|
16959
|
-
for (let position = 0; position < count; position++) {
|
|
16960
|
-
if (isDead(dead, position)) found += 1;
|
|
16961
|
-
}
|
|
16962
|
-
return found;
|
|
16963
|
-
}
|
|
16964
|
-
function deadBitsWhere(count, isDeadAt) {
|
|
16965
|
-
let dead = null;
|
|
16966
|
-
for (let position = 0; position < count; position++) {
|
|
16967
|
-
if (!isDeadAt(position)) continue;
|
|
16968
|
-
dead ??= new Uint8Array(deadBytesFor(count));
|
|
16969
|
-
dead[position >> 3] |= 1 << (position & 7);
|
|
16970
|
-
}
|
|
16971
|
-
return dead;
|
|
16972
|
-
}
|
|
16973
|
-
|
|
16974
17065
|
// src/vector/vector-index/persistence.ts
|
|
16975
17066
|
function recordFor(state, quantizer, docId) {
|
|
16976
17067
|
const record = quantizer.recordOf(docId);
|
|
@@ -16992,7 +17083,7 @@ function sameCentroid(state, planned) {
|
|
|
16992
17083
|
}
|
|
16993
17084
|
return true;
|
|
16994
17085
|
}
|
|
16995
|
-
function partCodes(state, docIds, centroid) {
|
|
17086
|
+
function partCodes(state, docIds, centroid, tombstonedAt) {
|
|
16996
17087
|
const quantizer = state.osq;
|
|
16997
17088
|
const layout = state.store.handles.codeLayout;
|
|
16998
17089
|
if (centroid === null || quantizer === null || layout === null) return null;
|
|
@@ -17005,7 +17096,8 @@ function partCodes(state, docIds, centroid) {
|
|
|
17005
17096
|
}
|
|
17006
17097
|
const records = new Uint8Array(docIds.length * layout.slotStride);
|
|
17007
17098
|
for (let i = 0; i < docIds.length; i++) {
|
|
17008
|
-
const
|
|
17099
|
+
const tombstoned = tombstonedAt.get(i);
|
|
17100
|
+
const record = tombstoned === void 0 ? recordFor(state, quantizer, docIds[i]) : quantizer.recordAtOrdinal(tombstoned);
|
|
17009
17101
|
if (record !== void 0) records.set(record, i * layout.slotStride);
|
|
17010
17102
|
}
|
|
17011
17103
|
return { bits: quantizer.bits, centroid, records };
|
|
@@ -17162,33 +17254,39 @@ function isLive(state, ordinal) {
|
|
|
17162
17254
|
const docId = state.store.docIdForOrdinal(ordinal);
|
|
17163
17255
|
return docId !== void 0 && !state.tombstones.has(docId);
|
|
17164
17256
|
}
|
|
17165
|
-
function
|
|
17166
|
-
|
|
17167
|
-
|
|
17168
|
-
|
|
17257
|
+
function isGraphTombstone(state, ordinal) {
|
|
17258
|
+
return ordinal !== NO_LIVE_VECTOR_AT_POSITION && state.hnsw !== null && state.hnsw.holdsTombstone(ordinal);
|
|
17259
|
+
}
|
|
17260
|
+
function isCarried(state, ordinal) {
|
|
17261
|
+
return isLive(state, ordinal) || isGraphTombstone(state, ordinal);
|
|
17262
|
+
}
|
|
17263
|
+
function carriedCountOf(state, file) {
|
|
17264
|
+
let carried = 0;
|
|
17265
|
+
for (let i = 0; i < file.ordinals.length; i++) if (isCarried(state, file.ordinals[i])) carried += 1;
|
|
17266
|
+
return carried;
|
|
17169
17267
|
}
|
|
17170
17268
|
function deadBitsOf(state, file) {
|
|
17171
17269
|
return deadBitsWhere(file.ordinals.length, (position) => !isLive(state, file.ordinals[position]));
|
|
17172
17270
|
}
|
|
17173
|
-
function filesWorthKeeping(state, listed, calibration,
|
|
17271
|
+
function filesWorthKeeping(state, listed, calibration, carriedCounts) {
|
|
17174
17272
|
const kept = [];
|
|
17175
17273
|
for (const file of listed) {
|
|
17176
|
-
const
|
|
17177
|
-
const
|
|
17178
|
-
if (file.calibration !== calibration ||
|
|
17274
|
+
const carried = carriedCountOf(state, file);
|
|
17275
|
+
const deadWithoutNodeShare = file.ordinals.length === 0 ? 1 : 1 - carried / file.ordinals.length;
|
|
17276
|
+
if (file.calibration !== calibration || deadWithoutNodeShare > VECTOR_FILE_REPLACED_ABOVE_DEAD_SHARE) continue;
|
|
17179
17277
|
kept.push(file);
|
|
17180
|
-
|
|
17278
|
+
carriedCounts.push(carried);
|
|
17181
17279
|
}
|
|
17182
17280
|
return kept;
|
|
17183
17281
|
}
|
|
17184
|
-
function absorbSmallerPartialFiles(kept,
|
|
17282
|
+
function absorbSmallerPartialFiles(kept, carriedCounts, vectorsToWrite, capacity) {
|
|
17185
17283
|
let pending = vectorsToWrite;
|
|
17186
17284
|
for (let index = kept.length - 1; index >= 0 && pending > 0; index--) {
|
|
17187
17285
|
if (kept[index].ordinals.length >= capacity) continue;
|
|
17188
|
-
if (
|
|
17189
|
-
pending +=
|
|
17286
|
+
if (carriedCounts[index] > pending) return;
|
|
17287
|
+
pending += carriedCounts[index];
|
|
17190
17288
|
kept.splice(index, 1);
|
|
17191
|
-
|
|
17289
|
+
carriedCounts.splice(index, 1);
|
|
17192
17290
|
}
|
|
17193
17291
|
}
|
|
17194
17292
|
function ordinalsOutsideOf(state, kept) {
|
|
@@ -17202,14 +17300,15 @@ function ordinalsOutsideOf(state, kept) {
|
|
|
17202
17300
|
}
|
|
17203
17301
|
const outside = [];
|
|
17204
17302
|
for (let ordinal = 0; ordinal < slots; ordinal++) {
|
|
17205
|
-
if (saved[ordinal] === 0 &&
|
|
17303
|
+
if (saved[ordinal] === 0 && isCarried(state, ordinal)) outside.push(ordinal);
|
|
17206
17304
|
}
|
|
17207
17305
|
return outside;
|
|
17208
17306
|
}
|
|
17209
|
-
function countOutside(state,
|
|
17307
|
+
function countOutside(state, carriedCounts) {
|
|
17210
17308
|
let held = 0;
|
|
17211
|
-
for (const
|
|
17212
|
-
|
|
17309
|
+
for (const carried of carriedCounts) held += carried;
|
|
17310
|
+
const tombstonedNodes = state.hnsw?.tombstoneCount ?? 0;
|
|
17311
|
+
return Math.max(0, state.store.size - state.tombstones.size + tombstonedNodes - held);
|
|
17213
17312
|
}
|
|
17214
17313
|
function numberEveryVector(state, kept, written) {
|
|
17215
17314
|
let total = written.length;
|
|
@@ -17220,7 +17319,7 @@ function numberEveryVector(state, kept, written) {
|
|
|
17220
17319
|
for (const file of kept) {
|
|
17221
17320
|
for (let i = 0; i < file.ordinals.length; i++, number++) {
|
|
17222
17321
|
const ordinal = file.ordinals[i];
|
|
17223
|
-
if (!
|
|
17322
|
+
if (!isCarried(state, ordinal)) continue;
|
|
17224
17323
|
numberOfOrdinal[ordinal] = number;
|
|
17225
17324
|
ordinalOfNumber[number] = ordinal;
|
|
17226
17325
|
}
|
|
@@ -17241,16 +17340,30 @@ function graphPartsOf(state, numbers) {
|
|
|
17241
17340
|
);
|
|
17242
17341
|
return parts.map((graph) => ({ v: VECTOR_GRAPH_PART_PAYLOAD_VERSION, fieldName: state.fieldName, graphs: [graph] }));
|
|
17243
17342
|
}
|
|
17244
|
-
function
|
|
17343
|
+
function graphPartsWithoutNodes(parts, numbers) {
|
|
17344
|
+
return parts.map((part, index) => ({
|
|
17345
|
+
...part,
|
|
17346
|
+
graphs: part.graphs.map((graph) => graphWithoutNodes(graph, index * VECTOR_GRAPH_PART_VECTORS, numbers))
|
|
17347
|
+
}));
|
|
17348
|
+
}
|
|
17349
|
+
function plannedVectorOf(state, ordinal) {
|
|
17350
|
+
const docId = state.store.docIdForOrdinal(ordinal) ?? state.store.view.docIdAt(ordinal) ?? "";
|
|
17351
|
+
return { ordinal, docId, tombstoned: !isLive(state, ordinal) };
|
|
17352
|
+
}
|
|
17353
|
+
function readFile(state, planned, centroid) {
|
|
17245
17354
|
const dimension = state.dimension;
|
|
17246
17355
|
const vectors = new Float32Array(planned.length * dimension);
|
|
17247
17356
|
const ordinals = new Int32Array(planned.length).fill(NO_LIVE_VECTOR_AT_POSITION);
|
|
17357
|
+
const tombstonedAt = /* @__PURE__ */ new Map();
|
|
17358
|
+
const docIds = planned.map((vector) => vector.docId);
|
|
17248
17359
|
for (let i = 0; i < planned.length; i++) {
|
|
17249
|
-
const
|
|
17360
|
+
const { ordinal, docId, tombstoned } = planned[i];
|
|
17361
|
+
const held = tombstoned || state.store.holdsOrdinal(ordinal) ? ordinal : state.store.getOrdinal(docId);
|
|
17250
17362
|
const entry = held === void 0 ? void 0 : state.store.entryForOrdinal(held);
|
|
17251
17363
|
if (held === void 0 || entry === void 0) continue;
|
|
17252
17364
|
vectors.set(entry.vector, i * dimension);
|
|
17253
17365
|
ordinals[i] = held;
|
|
17366
|
+
if (tombstoned) tombstonedAt.set(i, held);
|
|
17254
17367
|
}
|
|
17255
17368
|
return {
|
|
17256
17369
|
payload: {
|
|
@@ -17258,10 +17371,11 @@ function readFile(state, planned, docIds, centroid) {
|
|
|
17258
17371
|
fieldName: state.fieldName,
|
|
17259
17372
|
dimension,
|
|
17260
17373
|
docIds,
|
|
17261
|
-
codes: partCodes(state, docIds, centroid),
|
|
17374
|
+
codes: partCodes(state, docIds, centroid, tombstonedAt),
|
|
17262
17375
|
vectors: vectorsToBytes(vectors)
|
|
17263
17376
|
},
|
|
17264
|
-
ordinals
|
|
17377
|
+
ordinals,
|
|
17378
|
+
dead: deadBitsWhere(planned.length, (i) => planned[i].tombstoned || ordinals[i] === NO_LIVE_VECTOR_AT_POSITION)
|
|
17265
17379
|
};
|
|
17266
17380
|
}
|
|
17267
17381
|
function planCheckpoint(state, listedKeys, rewritesGraph = false) {
|
|
@@ -17269,14 +17383,20 @@ function planCheckpoint(state, listedKeys, rewritesGraph = false) {
|
|
|
17269
17383
|
const calibration = signature.calibration;
|
|
17270
17384
|
const trusted = registryMatches(state, listedKeys);
|
|
17271
17385
|
const capacity = fileCapacityOf(state);
|
|
17272
|
-
const
|
|
17273
|
-
const keptFiles = filesWorthKeeping(state, trusted ? state.savedFiles : [], calibration,
|
|
17274
|
-
absorbSmallerPartialFiles(keptFiles,
|
|
17386
|
+
const carriedCounts = [];
|
|
17387
|
+
const keptFiles = filesWorthKeeping(state, trusted ? state.savedFiles : [], calibration, carriedCounts);
|
|
17388
|
+
absorbSmallerPartialFiles(keptFiles, carriedCounts, countOutside(state, carriedCounts), capacity);
|
|
17275
17389
|
const written = ordinalsOutsideOf(state, keptFiles);
|
|
17276
|
-
const
|
|
17390
|
+
const planned = written.map((ordinal) => plannedVectorOf(state, ordinal));
|
|
17277
17391
|
const centroid = codedCentroid(state);
|
|
17278
17392
|
const listedBefore = trusted && listedKeys !== null;
|
|
17279
17393
|
const unchanged = !rewritesGraph && listedBefore && sameSignature(state.savedSignature, signature) && written.length === 0;
|
|
17394
|
+
let firstWrittenNumber = 0;
|
|
17395
|
+
for (const file of keptFiles) firstWrittenNumber += file.ordinals.length;
|
|
17396
|
+
const graphParts = unchanged ? null : graphPartsOf(state, numberEveryVector(state, keptFiles, written));
|
|
17397
|
+
const vanished = /* @__PURE__ */ new Set();
|
|
17398
|
+
let vanishedWhenStripped = 0;
|
|
17399
|
+
let graphPartsToWrite = graphParts;
|
|
17280
17400
|
return {
|
|
17281
17401
|
unchanged,
|
|
17282
17402
|
keptFiles,
|
|
@@ -17287,12 +17407,22 @@ function planCheckpoint(state, listedKeys, rewritesGraph = false) {
|
|
|
17287
17407
|
dead: deadBitsOf(state, file)
|
|
17288
17408
|
})),
|
|
17289
17409
|
newFiles: Math.ceil(written.length / capacity),
|
|
17290
|
-
graphParts
|
|
17410
|
+
get graphParts() {
|
|
17411
|
+
if (graphParts !== null && vanished.size !== vanishedWhenStripped) {
|
|
17412
|
+
graphPartsToWrite = graphPartsWithoutNodes(graphParts, vanished);
|
|
17413
|
+
vanishedWhenStripped = vanished.size;
|
|
17414
|
+
}
|
|
17415
|
+
return graphPartsToWrite;
|
|
17416
|
+
},
|
|
17291
17417
|
signature,
|
|
17292
17418
|
readNewFile(index) {
|
|
17293
17419
|
const start = index * capacity;
|
|
17294
17420
|
const end = start + capacity;
|
|
17295
|
-
|
|
17421
|
+
const file = readFile(state, planned.slice(start, end), centroid);
|
|
17422
|
+
for (let i = 0; i < file.ordinals.length; i++) {
|
|
17423
|
+
if (file.ordinals[i] === NO_LIVE_VECTOR_AT_POSITION) vanished.add(firstWrittenNumber + start + i);
|
|
17424
|
+
}
|
|
17425
|
+
return file;
|
|
17296
17426
|
}
|
|
17297
17427
|
};
|
|
17298
17428
|
}
|
|
@@ -17325,9 +17455,10 @@ function insertNodesOfLaterGraph(state, graph, later, ordinals) {
|
|
|
17325
17455
|
if (docId !== void 0 && !graph.has(docId)) graph.insertOrdinal(ordinal);
|
|
17326
17456
|
}
|
|
17327
17457
|
}
|
|
17328
|
-
function restoreGraph(state, payload, ordinalOfNumber) {
|
|
17458
|
+
function restoreGraph(state, payload, ordinalOfNumber, tombstoned) {
|
|
17329
17459
|
const first = payload?.graphs[0]?.parts[0];
|
|
17330
17460
|
if (payload === null || first === void 0) {
|
|
17461
|
+
releaseReplacedVectors(state);
|
|
17331
17462
|
bufferEveryVectorOutsideTheGraph(state);
|
|
17332
17463
|
return;
|
|
17333
17464
|
}
|
|
@@ -17338,11 +17469,13 @@ function restoreGraph(state, payload, ordinalOfNumber) {
|
|
|
17338
17469
|
state.osq ?? void 0
|
|
17339
17470
|
);
|
|
17340
17471
|
graph.deserializeNumbered(payload.graphs[0], ordinalOfNumber);
|
|
17472
|
+
for (const ordinal of tombstoned) graph.markTombstoneOrdinal(ordinal);
|
|
17341
17473
|
adoptGraph(state, graph);
|
|
17342
17474
|
for (const later of payload.graphs.slice(1)) insertNodesOfLaterGraph(state, graph, later, ordinalOfNumber);
|
|
17343
17475
|
bufferEveryVectorOutsideTheGraph(state);
|
|
17344
17476
|
if (state.buffer.size <= graph.size) return;
|
|
17345
17477
|
adoptGraph(state, null);
|
|
17478
|
+
releaseReplacedVectors(state);
|
|
17346
17479
|
state.buffer.clear();
|
|
17347
17480
|
bufferEveryVectorOutsideTheGraph(state);
|
|
17348
17481
|
}
|
|
@@ -17351,6 +17484,9 @@ function beginCheckpointRestore(state, shape) {
|
|
|
17351
17484
|
const cold = readsFromDisk(state) && (shape.holdsGraph || shape.liveVectors >= state.promotionThreshold);
|
|
17352
17485
|
const restoresCodes = state.osq !== null && osqBitsOf(state.quantizationMode) !== null && shape.holdsGraph;
|
|
17353
17486
|
const files = [];
|
|
17487
|
+
const graphNodes = shape.graph?.graphs[0] ?? null;
|
|
17488
|
+
const tombstoned = /* @__PURE__ */ new Set();
|
|
17489
|
+
let nextNumber = 0;
|
|
17354
17490
|
let sharedCodes = null;
|
|
17355
17491
|
let codesAgree = true;
|
|
17356
17492
|
function restoreRecords(payload, ordinals) {
|
|
@@ -17372,9 +17508,14 @@ function beginCheckpointRestore(state, shape) {
|
|
|
17372
17508
|
const stride = layout.slotStride;
|
|
17373
17509
|
for (let i = 0; i < ordinals.length; i++) {
|
|
17374
17510
|
if (ordinals[i] === NO_LIVE_VECTOR_AT_POSITION) continue;
|
|
17375
|
-
|
|
17511
|
+
const record = codes.records.subarray(i * stride, (i + 1) * stride);
|
|
17512
|
+
if (tombstoned.has(ordinals[i])) quantizer.restoreRecordAtOrdinal(ordinals[i], record);
|
|
17513
|
+
else quantizer.restoreRecord(payload.docIds[i], record);
|
|
17376
17514
|
}
|
|
17377
17515
|
}
|
|
17516
|
+
function graphHoldsNodeFor(number) {
|
|
17517
|
+
return graphNodes !== null && topLayerInParts(graphNodes, number) >= 0;
|
|
17518
|
+
}
|
|
17378
17519
|
function addFile(file) {
|
|
17379
17520
|
const { payload, location } = file;
|
|
17380
17521
|
if (payload.dimension !== state.dimension) {
|
|
@@ -17389,13 +17530,21 @@ function beginCheckpointRestore(state, shape) {
|
|
|
17389
17530
|
const vectors = bytesToVectors(payload.vectors);
|
|
17390
17531
|
const fileIndex = onDisk && cold ? state.store.addVectorFile(location.path) : null;
|
|
17391
17532
|
const ordinals = new Int32Array(payload.docIds.length).fill(NO_LIVE_VECTOR_AT_POSITION);
|
|
17533
|
+
const firstNumber = nextNumber;
|
|
17534
|
+
nextNumber += payload.docIds.length;
|
|
17392
17535
|
for (let i = 0; i < payload.docIds.length; i++) {
|
|
17393
|
-
|
|
17536
|
+
const dead = isDead(file.dead, i);
|
|
17537
|
+
if (dead && !graphHoldsNodeFor(firstNumber + i)) continue;
|
|
17394
17538
|
const docId = payload.docIds[i];
|
|
17395
|
-
const replaced = state.store.getOrdinal(docId);
|
|
17396
|
-
if (replaced !== void 0) state.osq?.removeOrdinal(replaced);
|
|
17397
17539
|
const vector = vectors.subarray(i * dimension, (i + 1) * dimension);
|
|
17398
17540
|
const offset = onDisk ? location.vectorsOffset + i * dimension * 4 : 0;
|
|
17541
|
+
if (dead) {
|
|
17542
|
+
ordinals[i] = state.store.insertReplaced(docId, vector, fileIndex === null ? void 0 : { fileIndex, offset });
|
|
17543
|
+
tombstoned.add(ordinals[i]);
|
|
17544
|
+
continue;
|
|
17545
|
+
}
|
|
17546
|
+
const replaced = state.store.getOrdinal(docId);
|
|
17547
|
+
if (replaced !== void 0) state.osq?.removeOrdinal(replaced);
|
|
17399
17548
|
if (fileIndex !== null) {
|
|
17400
17549
|
ordinals[i] = state.store.insertCold(docId, magnitude(vector), { fileIndex, offset });
|
|
17401
17550
|
continue;
|
|
@@ -17406,21 +17555,27 @@ function beginCheckpointRestore(state, shape) {
|
|
|
17406
17555
|
restoreRecords(payload, ordinals);
|
|
17407
17556
|
files.push({ id: file.id, key: file.key, ordinals, coded: payload.codes !== null });
|
|
17408
17557
|
}
|
|
17409
|
-
function finish(
|
|
17558
|
+
function finish() {
|
|
17410
17559
|
const codesRestored = restoresCodes && codesAgree && sharedCodes !== null;
|
|
17411
|
-
if (restoresCodes && !codesRestored)
|
|
17560
|
+
if (restoresCodes && !codesRestored) {
|
|
17561
|
+
recalibrateFromStore(state);
|
|
17562
|
+
for (const ordinal of tombstoned) state.osq?.writeCodes(ordinal);
|
|
17563
|
+
}
|
|
17412
17564
|
let total = 0;
|
|
17413
17565
|
for (const file of files) total += file.ordinals.length;
|
|
17414
17566
|
const ordinalOfNumber = new Int32Array(total);
|
|
17415
17567
|
let number = 0;
|
|
17416
17568
|
for (const file of files) {
|
|
17417
17569
|
for (let i = 0; i < file.ordinals.length; i++) {
|
|
17418
|
-
|
|
17570
|
+
const ordinal = file.ordinals[i];
|
|
17571
|
+
if (!state.store.holdsOrdinal(ordinal) && !tombstoned.has(ordinal)) {
|
|
17572
|
+
file.ordinals[i] = NO_LIVE_VECTOR_AT_POSITION;
|
|
17573
|
+
}
|
|
17419
17574
|
}
|
|
17420
17575
|
ordinalOfNumber.set(file.ordinals, number);
|
|
17421
17576
|
number += file.ordinals.length;
|
|
17422
17577
|
}
|
|
17423
|
-
restoreGraph(state, graph, ordinalOfNumber);
|
|
17578
|
+
restoreGraph(state, shape.graph, ordinalOfNumber, tombstoned);
|
|
17424
17579
|
const current = signatureOf(state).calibration;
|
|
17425
17580
|
const codedUnder = codesRestored && current !== null ? current : CALIBRATION_NEVER_CURRENT;
|
|
17426
17581
|
const saved = files.map((file) => ({
|
|
@@ -17437,11 +17592,12 @@ function beginCheckpointRestore(state, shape) {
|
|
|
17437
17592
|
|
|
17438
17593
|
// src/vector/vector-index/maintenance.ts
|
|
17439
17594
|
function compact(state) {
|
|
17440
|
-
if (state.tombstones.size === 0) return;
|
|
17595
|
+
if (state.tombstones.size === 0 && state.store.replacedVectorCount === 0) return;
|
|
17441
17596
|
if (state.hnsw) {
|
|
17442
17597
|
state.compactedNodeCount += state.hnsw.tombstoneCount;
|
|
17443
17598
|
state.hnsw.compactTombstones();
|
|
17444
17599
|
}
|
|
17600
|
+
state.freshGraph?.compactTombstones();
|
|
17445
17601
|
for (const docId of state.tombstones) {
|
|
17446
17602
|
state.store.remove(docId);
|
|
17447
17603
|
state.buffer.delete(docId);
|
|
@@ -17449,6 +17605,7 @@ function compact(state) {
|
|
|
17449
17605
|
state.osq.remove(docId);
|
|
17450
17606
|
}
|
|
17451
17607
|
}
|
|
17608
|
+
releaseReplacedVectors(state);
|
|
17452
17609
|
state.tombstones.clear();
|
|
17453
17610
|
}
|
|
17454
17611
|
function recalibrateWhileNoThreadSearches(state, graph) {
|
|
@@ -17480,6 +17637,7 @@ async function foldIntoGraph(state) {
|
|
|
17480
17637
|
const graph = state.hnsw;
|
|
17481
17638
|
if (graph === null || rebuildNeeded) {
|
|
17482
17639
|
if (state.osq?.isCalibrated()) recalibrateWhileNoThreadSearches(state, graph);
|
|
17640
|
+
else calibrateQuantizer(state);
|
|
17483
17641
|
await buildGraphFromStore(state);
|
|
17484
17642
|
} else {
|
|
17485
17643
|
await insertMissing(state, graph);
|
|
@@ -17526,8 +17684,9 @@ function maintenanceStatus(state) {
|
|
|
17526
17684
|
const storeSize = state.store.size;
|
|
17527
17685
|
const graphCount = state.hnsw ? 1 : 0;
|
|
17528
17686
|
const graphNodes = state.hnsw === null ? storeSize : state.hnsw.size + state.hnsw.tombstoneCount;
|
|
17529
|
-
const
|
|
17530
|
-
const
|
|
17687
|
+
const tombstoned = state.tombstones.size + state.store.replacedVectorCount;
|
|
17688
|
+
const tombstoneRatio = graphNodes > 0 ? tombstoned / graphNodes : 0;
|
|
17689
|
+
const estimatedCompactMs = Math.round(tombstoned * ESTIMATED_MS_PER_TOMBSTONE * state.dimensionScale);
|
|
17531
17690
|
const optimizeVectorCount = state.hnsw === null || graphNeedsRebuild(state) ? storeSize : state.buffer.size;
|
|
17532
17691
|
const estimatedOptimizeMs = Math.round(optimizeVectorCount * ESTIMATED_MS_PER_VECTOR_REBUILD * state.dimensionScale);
|
|
17533
17692
|
return {
|
|
@@ -17787,19 +17946,19 @@ function searchWithFilter(state, query, k, options, filter) {
|
|
|
17787
17946
|
return scannedOutcome(scan2);
|
|
17788
17947
|
}
|
|
17789
17948
|
}
|
|
17790
|
-
const
|
|
17791
|
-
|
|
17792
|
-
|
|
17793
|
-
|
|
17794
|
-
|
|
17795
|
-
|
|
17796
|
-
|
|
17797
|
-
|
|
17798
|
-
|
|
17799
|
-
|
|
17800
|
-
const
|
|
17801
|
-
const buffered = bruteForceSearch(state, query, k, metric, minSimilarity,
|
|
17802
|
-
return
|
|
17949
|
+
const hnswResults = state.hnsw.search(query, k, metric, minSimilarity, { filter, efSearch, oversample }).map((r) => ({ docId: r.docId, score: r.score }));
|
|
17950
|
+
return fetchedOutcome(
|
|
17951
|
+
state,
|
|
17952
|
+
withBufferedVectors(state, query, k, options, filter, hnswResults),
|
|
17953
|
+
minSimilarity,
|
|
17954
|
+
filter
|
|
17955
|
+
);
|
|
17956
|
+
}
|
|
17957
|
+
function withBufferedVectors(state, query, k, options, filter, graphResults) {
|
|
17958
|
+
if (state.buffer.size === 0) return graphResults;
|
|
17959
|
+
const candidates = bufferCandidates(state, filter);
|
|
17960
|
+
const buffered = bruteForceSearch(state, query, k, options.metric, options.minSimilarity, candidates);
|
|
17961
|
+
return mergeResults(graphResults, buffered.results, k);
|
|
17803
17962
|
}
|
|
17804
17963
|
|
|
17805
17964
|
// src/vector/vector-index/index.ts
|
|
@@ -17900,11 +18059,12 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
17900
18059
|
}
|
|
17901
18060
|
noteWrite(state);
|
|
17902
18061
|
state.tombstones.delete(docId);
|
|
17903
|
-
state.
|
|
18062
|
+
const graphHoldsPrevious = previous !== void 0 && ((state.hnsw?.holdsNode(previous) ?? false) || (state.freshGraph?.holdsNode(previous) ?? false));
|
|
18063
|
+
state.store.insert(docId, vector, partitionId, graphHoldsPrevious);
|
|
17904
18064
|
if (previous !== void 0) {
|
|
17905
18065
|
state.hnsw?.markTombstoneOrdinal(previous);
|
|
17906
18066
|
state.freshGraph?.markTombstoneOrdinal(previous);
|
|
17907
|
-
state.osq?.removeOrdinal(previous);
|
|
18067
|
+
if (!graphHoldsPrevious) state.osq?.removeOrdinal(previous);
|
|
17908
18068
|
}
|
|
17909
18069
|
state.buffer.add(docId);
|
|
17910
18070
|
state.pendingLocations.delete(docId);
|
|
@@ -17974,10 +18134,13 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
17974
18134
|
efSearch: options.efSearch,
|
|
17975
18135
|
oversample: options.oversample
|
|
17976
18136
|
});
|
|
17977
|
-
if (viaWorkerCopy !== null) return fetchedOutcome(state, viaWorkerCopy, options.minSimilarity, filter);
|
|
17978
18137
|
if (confined && state.revision !== filterRevision) {
|
|
17979
18138
|
filter = filterForOptions(state, options);
|
|
17980
18139
|
}
|
|
18140
|
+
if (viaWorkerCopy !== null) {
|
|
18141
|
+
const results = withBufferedVectors(state, query, k, options, filter, viaWorkerCopy);
|
|
18142
|
+
return fetchedOutcome(state, results, options.minSimilarity, filter);
|
|
18143
|
+
}
|
|
17981
18144
|
return searchWithFilter(state, query, k, options, filter);
|
|
17982
18145
|
}
|
|
17983
18146
|
return {
|
|
@@ -19380,10 +19543,10 @@ function indexByFieldPath(entries) {
|
|
|
19380
19543
|
for (const entry of entries) byFieldPath.set(entry.fieldPath, entry);
|
|
19381
19544
|
return byFieldPath;
|
|
19382
19545
|
}
|
|
19383
|
-
function rowsByFieldPath(inputs,
|
|
19546
|
+
function rowsByFieldPath(inputs, entriesOf2, valuesOf2) {
|
|
19384
19547
|
const byFieldPath = /* @__PURE__ */ new Map();
|
|
19385
19548
|
for (const input of inputs) {
|
|
19386
|
-
for (const entry of
|
|
19549
|
+
for (const entry of entriesOf2(input)) {
|
|
19387
19550
|
let rows = byFieldPath.get(entry.fieldPath);
|
|
19388
19551
|
if (rows === void 0) {
|
|
19389
19552
|
rows = [];
|
|
@@ -20839,6 +21002,74 @@ function createDirectExecutor(options) {
|
|
|
20839
21002
|
};
|
|
20840
21003
|
}
|
|
20841
21004
|
|
|
21005
|
+
// src/partitioning/text-rebuild.ts
|
|
21006
|
+
function interrupted(pass) {
|
|
21007
|
+
return pass.log.partitionsReplaced || !pass.hooks.isCurrent();
|
|
21008
|
+
}
|
|
21009
|
+
async function buildPartition(pass, source, partitionId) {
|
|
21010
|
+
const { manager, options } = pass;
|
|
21011
|
+
const target = createPartitionIndex(partitionId, manager.config.trackPositions ?? true);
|
|
21012
|
+
const docIds = [...source.docIds()];
|
|
21013
|
+
target.beginBatch();
|
|
21014
|
+
for (let start = 0; start < docIds.length; start += TEXT_REBUILD_CHUNK_DOCUMENTS) {
|
|
21015
|
+
const end = Math.min(start + TEXT_REBUILD_CHUNK_DOCUMENTS, docIds.length);
|
|
21016
|
+
for (let i = start; i < end; i++) {
|
|
21017
|
+
const document2 = manager.getRef(docIds[i]);
|
|
21018
|
+
if (document2 !== void 0) target.insert(docIds[i], document2, manager.schema, manager.language, options);
|
|
21019
|
+
}
|
|
21020
|
+
await yieldToEventLoop();
|
|
21021
|
+
if (interrupted(pass)) return null;
|
|
21022
|
+
}
|
|
21023
|
+
target.endBatch();
|
|
21024
|
+
if (manager.partialWordAnalysis !== null) target.preparePartialWords(manager.partialWordAnalysis);
|
|
21025
|
+
return target;
|
|
21026
|
+
}
|
|
21027
|
+
function applyWrittenDocuments(pass, built) {
|
|
21028
|
+
const { manager, options } = pass;
|
|
21029
|
+
for (const docId of pass.log.takeWrittenDocIds()) {
|
|
21030
|
+
for (const partition of built) {
|
|
21031
|
+
if (!partition.has(docId)) continue;
|
|
21032
|
+
partition.remove(docId, manager.schema, manager.language, options);
|
|
21033
|
+
break;
|
|
21034
|
+
}
|
|
21035
|
+
const partitionId = manager.partitionIdOf(docId);
|
|
21036
|
+
const document2 = manager.getRef(docId);
|
|
21037
|
+
if (partitionId === void 0 || document2 === void 0) continue;
|
|
21038
|
+
built[partitionId].insert(docId, document2, manager.schema, manager.language, options);
|
|
21039
|
+
}
|
|
21040
|
+
}
|
|
21041
|
+
async function rebuildOnce(pass) {
|
|
21042
|
+
const partitionCount = pass.manager.partitionCount;
|
|
21043
|
+
const built = [];
|
|
21044
|
+
for (let partitionId = 0; partitionId < partitionCount; partitionId++) {
|
|
21045
|
+
const partition = await buildPartition(pass, pass.manager.getPartition(partitionId), partitionId);
|
|
21046
|
+
if (partition === null) return false;
|
|
21047
|
+
built.push(partition);
|
|
21048
|
+
pass.hooks.onPartitionBuilt(built.length);
|
|
21049
|
+
}
|
|
21050
|
+
while (pass.log.pendingDocuments > TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS) {
|
|
21051
|
+
applyWrittenDocuments(pass, built);
|
|
21052
|
+
await yieldToEventLoop();
|
|
21053
|
+
if (interrupted(pass)) return false;
|
|
21054
|
+
}
|
|
21055
|
+
applyWrittenDocuments(pass, built);
|
|
21056
|
+
pass.manager.adoptRebuiltPartitions(built);
|
|
21057
|
+
pass.hooks.onSwitch();
|
|
21058
|
+
return true;
|
|
21059
|
+
}
|
|
21060
|
+
async function rebuildTextIndexes(manager, hooks) {
|
|
21061
|
+
const options = resolvePartitionInsertOptions(manager.config, manager.analysis, { validate: false, skipClone: true });
|
|
21062
|
+
while (hooks.isCurrent()) {
|
|
21063
|
+
const log = manager.openWriteLog();
|
|
21064
|
+
try {
|
|
21065
|
+
if (await rebuildOnce({ manager, log, hooks, options })) return true;
|
|
21066
|
+
} finally {
|
|
21067
|
+
log.close();
|
|
21068
|
+
}
|
|
21069
|
+
}
|
|
21070
|
+
return false;
|
|
21071
|
+
}
|
|
21072
|
+
|
|
20842
21073
|
// src/engine/events.ts
|
|
20843
21074
|
function eventHandlersOf(events) {
|
|
20844
21075
|
const handlers = /* @__PURE__ */ new Map();
|
|
@@ -20861,11 +21092,6 @@ function emitEngineEvent(eventHandlers, event, payload) {
|
|
|
20861
21092
|
}
|
|
20862
21093
|
|
|
20863
21094
|
// src/engine/analysis-rebuild.ts
|
|
20864
|
-
function yieldToEventLoop4() {
|
|
20865
|
-
return new Promise((resolve) => {
|
|
20866
|
-
setTimeout(resolve, 0);
|
|
20867
|
-
});
|
|
20868
|
-
}
|
|
20869
21095
|
function toError(value) {
|
|
20870
21096
|
return value instanceof Error ? value : new Error(String(value));
|
|
20871
21097
|
}
|
|
@@ -20876,9 +21102,13 @@ function wireAnalysisRebuild(wiring) {
|
|
|
20876
21102
|
resyncIndex: wiring.resyncIndex,
|
|
20877
21103
|
async persistAnalysisRevision(indexName) {
|
|
20878
21104
|
const durability = wiring.durabilityManager;
|
|
20879
|
-
if (durability === null)
|
|
20880
|
-
|
|
21105
|
+
if (durability === null) {
|
|
21106
|
+
wiring.recordCurrentAnalysis(indexName);
|
|
21107
|
+
return;
|
|
21108
|
+
}
|
|
20881
21109
|
await (durability.checkpointFromMemory?.(indexName) ?? durability.checkpoint(indexName));
|
|
21110
|
+
wiring.recordCurrentAnalysis(indexName);
|
|
21111
|
+
await durability.persistMetadata(indexName);
|
|
20882
21112
|
},
|
|
20883
21113
|
emit(payload) {
|
|
20884
21114
|
emitEngineEvent(wiring.eventHandlers, "analysisRebuild", payload);
|
|
@@ -20889,9 +21119,10 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
20889
21119
|
const stale = /* @__PURE__ */ new Map();
|
|
20890
21120
|
const rebuildMode = config?.rebuild ?? "auto";
|
|
20891
21121
|
let queue = Promise.resolve();
|
|
21122
|
+
let active = null;
|
|
20892
21123
|
async function rebuildNow(indexName, entry) {
|
|
20893
21124
|
const manager = deps.getManager(indexName);
|
|
20894
|
-
if (manager === void 0) {
|
|
21125
|
+
if (manager === void 0 || entry.abandoned) {
|
|
20895
21126
|
entry.run = null;
|
|
20896
21127
|
return;
|
|
20897
21128
|
}
|
|
@@ -20899,12 +21130,20 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
20899
21130
|
deps.emit({ indexName, status: "started", partitionsRebuilt: 0, partitionCount });
|
|
20900
21131
|
const wasPromoted = deps.desyncIndex(indexName);
|
|
20901
21132
|
try {
|
|
20902
|
-
|
|
20903
|
-
manager.
|
|
20904
|
-
|
|
20905
|
-
|
|
21133
|
+
const rebuilt = await rebuildTextIndexes(manager, {
|
|
21134
|
+
isCurrent: () => deps.getManager(indexName) === manager && stale.get(indexName) === entry && !entry.abandoned,
|
|
21135
|
+
onPartitionBuilt(partitionsBuilt) {
|
|
21136
|
+
entry.partitionsRebuilt = partitionsBuilt;
|
|
21137
|
+
},
|
|
21138
|
+
onSwitch() {
|
|
21139
|
+
stale.delete(indexName);
|
|
21140
|
+
}
|
|
21141
|
+
});
|
|
21142
|
+
if (!rebuilt) {
|
|
21143
|
+
entry.run = null;
|
|
21144
|
+
if (deps.getManager(indexName) === manager) await deps.resyncIndex(indexName, wasPromoted);
|
|
21145
|
+
return;
|
|
20906
21146
|
}
|
|
20907
|
-
stale.delete(indexName);
|
|
20908
21147
|
await deps.persistAnalysisRevision(indexName);
|
|
20909
21148
|
await deps.resyncIndex(indexName, wasPromoted);
|
|
20910
21149
|
deps.emit({
|
|
@@ -20915,7 +21154,6 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
20915
21154
|
});
|
|
20916
21155
|
} catch (err) {
|
|
20917
21156
|
const error = toError(err);
|
|
20918
|
-
stale.set(indexName, entry);
|
|
20919
21157
|
entry.run = null;
|
|
20920
21158
|
await deps.resyncIndex(indexName, wasPromoted).catch(() => void 0);
|
|
20921
21159
|
deps.emit({
|
|
@@ -20928,11 +21166,20 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
20928
21166
|
throw error;
|
|
20929
21167
|
}
|
|
20930
21168
|
}
|
|
21169
|
+
function runTracked(indexName, entry) {
|
|
21170
|
+
const done = rebuildNow(indexName, entry);
|
|
21171
|
+
const tracked = { indexName, entry, done };
|
|
21172
|
+
active = tracked;
|
|
21173
|
+
return done.finally(() => {
|
|
21174
|
+
if (active === tracked) active = null;
|
|
21175
|
+
});
|
|
21176
|
+
}
|
|
20931
21177
|
function enqueue(indexName, entry) {
|
|
20932
21178
|
if (entry.run !== null) {
|
|
20933
21179
|
return entry.run;
|
|
20934
21180
|
}
|
|
20935
|
-
|
|
21181
|
+
entry.abandoned = false;
|
|
21182
|
+
const run = queue.then(() => runTracked(indexName, entry));
|
|
20936
21183
|
entry.run = run;
|
|
20937
21184
|
queue = run.then(
|
|
20938
21185
|
() => void 0,
|
|
@@ -20967,7 +21214,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
20967
21214
|
if (existing !== void 0 && existing.run !== null) {
|
|
20968
21215
|
return;
|
|
20969
21216
|
}
|
|
20970
|
-
stale.set(index.indexName, { index, partitionsRebuilt: 0, run: null });
|
|
21217
|
+
stale.set(index.indexName, { index, partitionsRebuilt: 0, run: null, abandoned: false });
|
|
20971
21218
|
},
|
|
20972
21219
|
clearStale(indexName) {
|
|
20973
21220
|
stale.delete(indexName);
|
|
@@ -20978,6 +21225,14 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
20978
21225
|
isRunning(indexName) {
|
|
20979
21226
|
return stale.get(indexName)?.run !== null && stale.get(indexName)?.run !== void 0;
|
|
20980
21227
|
},
|
|
21228
|
+
async abandon(indexName) {
|
|
21229
|
+
const entry = stale.get(indexName);
|
|
21230
|
+
if (entry !== void 0 && entry.run !== null) entry.abandoned = true;
|
|
21231
|
+
const running = active;
|
|
21232
|
+
if (running === null || running.indexName !== indexName) return;
|
|
21233
|
+
running.entry.abandoned = true;
|
|
21234
|
+
await running.done.catch(() => void 0);
|
|
21235
|
+
},
|
|
20981
21236
|
review(indexName) {
|
|
20982
21237
|
const entry = stale.get(indexName);
|
|
20983
21238
|
return entry === void 0 ? Promise.resolve() : review(indexName, entry);
|
|
@@ -21644,7 +21899,7 @@ function buildEntry(fields) {
|
|
|
21644
21899
|
}
|
|
21645
21900
|
|
|
21646
21901
|
// src/serialization/chunk-checksum.ts
|
|
21647
|
-
function
|
|
21902
|
+
function yieldToEventLoop4() {
|
|
21648
21903
|
if (typeof setImmediate === "function") {
|
|
21649
21904
|
return new Promise((resolve) => setImmediate(resolve));
|
|
21650
21905
|
}
|
|
@@ -21656,7 +21911,7 @@ async function advanceChecksum(start, payload) {
|
|
|
21656
21911
|
const end = Math.min(offset + CHECKSUM_YIELD_CHUNK_BYTES, payload.length);
|
|
21657
21912
|
state = crc32Update(state, payload.subarray(offset, end));
|
|
21658
21913
|
if (end < payload.length) {
|
|
21659
|
-
await
|
|
21914
|
+
await yieldToEventLoop4();
|
|
21660
21915
|
}
|
|
21661
21916
|
}
|
|
21662
21917
|
return state;
|
|
@@ -23782,7 +24037,8 @@ async function readVectorFile(store, file, fieldPath) {
|
|
|
23782
24037
|
async function loadVectorField(store, ref, vectorIndex) {
|
|
23783
24038
|
const restore = vectorIndex.restoreCheckpoint({
|
|
23784
24039
|
liveVectors: liveVectorsOf(ref),
|
|
23785
|
-
holdsGraph: ref.graphParts.length > 0
|
|
24040
|
+
holdsGraph: ref.graphParts.length > 0,
|
|
24041
|
+
graph: await readGraphParts(store, ref)
|
|
23786
24042
|
});
|
|
23787
24043
|
for (const file of ref.files) {
|
|
23788
24044
|
const { payload, payloadLength } = await readVectorFile(store, file, ref.fieldPath);
|
|
@@ -23797,7 +24053,7 @@ async function loadVectorField(store, ref, vectorIndex) {
|
|
|
23797
24053
|
}
|
|
23798
24054
|
});
|
|
23799
24055
|
}
|
|
23800
|
-
restore.finish(
|
|
24056
|
+
restore.finish();
|
|
23801
24057
|
}
|
|
23802
24058
|
var EMPTY_BIN = new Uint8Array(0);
|
|
23803
24059
|
var EMPTY_BIN_BYTES = 2;
|
|
@@ -23861,7 +24117,7 @@ async function writeField(input, fieldPath, plan, outcome) {
|
|
|
23861
24117
|
const files = plan.kept.map((kept) => keptFileRef(input, prior, kept));
|
|
23862
24118
|
const written = [];
|
|
23863
24119
|
for (let index = 0; index < plan.newFiles; index++) {
|
|
23864
|
-
const { payload, ordinals } = plan.readNewFile(index);
|
|
24120
|
+
const { payload, ordinals, dead } = plan.readNewFile(index);
|
|
23865
24121
|
const id = nextFileId;
|
|
23866
24122
|
nextFileId += 1;
|
|
23867
24123
|
const key = vectorFileKey(input.indexName, fieldPath, id);
|
|
@@ -23871,13 +24127,13 @@ async function writeField(input, fieldPath, plan, outcome) {
|
|
|
23871
24127
|
`vector file "${key}"`
|
|
23872
24128
|
);
|
|
23873
24129
|
await input.store.write(key, packed.chunks);
|
|
23874
|
-
const dead = deadBitsWhere(ordinals.length, (position) => ordinals[position] === NO_LIVE_VECTOR_AT_POSITION);
|
|
23875
24130
|
files.push({ id, key, count: payload.docIds.length, dead, bytes: packed.byteLength, checksum: packed.checksum });
|
|
23876
24131
|
written.push({ id, key, ordinals });
|
|
23877
24132
|
outcome.layouts.push({
|
|
23878
24133
|
fieldPath,
|
|
23879
24134
|
key,
|
|
23880
24135
|
docIds: payload.docIds,
|
|
24136
|
+
dead,
|
|
23881
24137
|
vectorsOffset: packed.byteLength - payload.vectors.byteLength
|
|
23882
24138
|
});
|
|
23883
24139
|
}
|
|
@@ -25664,7 +25920,7 @@ function removeWhatTheSegmentReplaces(manager, segment, contents) {
|
|
|
25664
25920
|
manager.endBatchRemove();
|
|
25665
25921
|
}
|
|
25666
25922
|
}
|
|
25667
|
-
function
|
|
25923
|
+
function yieldToEventLoop5() {
|
|
25668
25924
|
return new Promise((resolve) => setTimeout(resolve, 0));
|
|
25669
25925
|
}
|
|
25670
25926
|
async function loadPartitionSegments(store, partition, manager, options) {
|
|
@@ -25675,7 +25931,7 @@ async function loadPartitionSegments(store, partition, manager, options) {
|
|
|
25675
25931
|
const segment = createSharedFrozenSegment(shareSegmentArrays(contents.arrays, manager.partialWordAnalysis));
|
|
25676
25932
|
removeWhatTheSegmentReplaces(manager, segment, contents);
|
|
25677
25933
|
if (contents.arrays.documentCount > 0) manager.attachFrozenSegment(partition.partitionId, segment);
|
|
25678
|
-
await
|
|
25934
|
+
await yieldToEventLoop5();
|
|
25679
25935
|
}
|
|
25680
25936
|
}
|
|
25681
25937
|
|
|
@@ -25748,7 +26004,12 @@ async function adoptVectorLayouts(store, vectorIndexes, layouts) {
|
|
|
25748
26004
|
const vecIndex = vectorIndexes.get(layout.fieldPath);
|
|
25749
26005
|
if (vecIndex === void 0 || vecIndex.storage !== "disk") continue;
|
|
25750
26006
|
const path = await pathOf.call(store, layout.key);
|
|
25751
|
-
await vecIndex.adoptDiskLayout({
|
|
26007
|
+
await vecIndex.adoptDiskLayout({
|
|
26008
|
+
path,
|
|
26009
|
+
docIds: layout.docIds,
|
|
26010
|
+
dead: layout.dead,
|
|
26011
|
+
vectorsOffset: layout.vectorsOffset
|
|
26012
|
+
});
|
|
25752
26013
|
}
|
|
25753
26014
|
}
|
|
25754
26015
|
async function garbageThatNoFieldStillReads(store, vectorIndexes, keys) {
|
|
@@ -26703,7 +26964,11 @@ async function readGraph(store, ref) {
|
|
|
26703
26964
|
async function loadVectorField2(store, ref, vectorIndex, fileRefs) {
|
|
26704
26965
|
let liveVectors = 0;
|
|
26705
26966
|
for (const file of ref.files) liveVectors += file.count - countDead(file.dead, file.count);
|
|
26706
|
-
const restore = vectorIndex.restoreCheckpoint({
|
|
26967
|
+
const restore = vectorIndex.restoreCheckpoint({
|
|
26968
|
+
liveVectors,
|
|
26969
|
+
holdsGraph: ref.graphKey !== null,
|
|
26970
|
+
graph: await readGraph(store, ref)
|
|
26971
|
+
});
|
|
26707
26972
|
for (const file of ref.files) {
|
|
26708
26973
|
const bytes = await store.read(file.key);
|
|
26709
26974
|
if (bytes === null) {
|
|
@@ -26736,7 +27001,7 @@ async function loadVectorField2(store, ref, vectorIndex, fileRefs) {
|
|
|
26736
27001
|
}
|
|
26737
27002
|
});
|
|
26738
27003
|
}
|
|
26739
|
-
restore.finish(
|
|
27004
|
+
restore.finish();
|
|
26740
27005
|
}
|
|
26741
27006
|
async function loadEarlierLayout(store, indexName, manifest, deps) {
|
|
26742
27007
|
let highestPartitionId = manifest.partitions.length - 1;
|
|
@@ -28094,7 +28359,7 @@ function metadataConfigOf(entry) {
|
|
|
28094
28359
|
b: entry.config.bm25?.b ?? 0.75,
|
|
28095
28360
|
...embedding !== void 0 ? { embedding } : {},
|
|
28096
28361
|
surfaceForms: entry.config.surfaceForms !== false,
|
|
28097
|
-
analysisRevision: entry.
|
|
28362
|
+
...entry.analysisRevision !== null ? { analysisRevision: entry.analysisRevision } : {},
|
|
28098
28363
|
...typeof entry.config.tokenizer === "string" ? { tokenizer: entry.config.tokenizer } : {},
|
|
28099
28364
|
...typeof entry.config.stopWords === "string" ? { stopWords: entry.config.stopWords } : {},
|
|
28100
28365
|
...entry.config.stopWords instanceof Set ? { stopWordList: [...entry.config.stopWords].sort(compareCodePoints) } : {},
|
|
@@ -28335,7 +28600,8 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
28335
28600
|
indexUuid: metadata.indexUuid ?? null,
|
|
28336
28601
|
heldPartitions: metadata.heldPartitions ?? null,
|
|
28337
28602
|
documentCount: metadata.documentCount ?? 0,
|
|
28338
|
-
partitionCount: metadata.partitionCount
|
|
28603
|
+
partitionCount: metadata.partitionCount,
|
|
28604
|
+
analysisRevision: metadata.analysisRevision ?? null
|
|
28339
28605
|
});
|
|
28340
28606
|
if (loadData) await deps.indexState().registerOpen(metadata.indexName);
|
|
28341
28607
|
else deps.indexState().registerClosed(metadata.indexName);
|
|
@@ -28458,8 +28724,14 @@ function createIndexStateCoordinator(config, callbacks) {
|
|
|
28458
28724
|
return entry;
|
|
28459
28725
|
}
|
|
28460
28726
|
}
|
|
28727
|
+
function transitionPending(indexName, entry) {
|
|
28728
|
+
if (entries.get(indexName) !== entry) return true;
|
|
28729
|
+
if (entry.state === "dropping" && entry.dropPromise !== null) return true;
|
|
28730
|
+
return entry.state === "closing" && entry.closePromise !== null;
|
|
28731
|
+
}
|
|
28461
28732
|
async function reopen(indexName, explicit) {
|
|
28462
|
-
|
|
28733
|
+
let entry = await awaitPendingTransitions(indexName);
|
|
28734
|
+
while (transitionPending(indexName, entry)) entry = await awaitPendingTransitions(indexName);
|
|
28463
28735
|
if (entry.state === "open") return;
|
|
28464
28736
|
if (explicit) {
|
|
28465
28737
|
entry.reopenFailures = 0;
|
|
@@ -30878,7 +31150,13 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
30878
31150
|
idleSweep: null
|
|
30879
31151
|
};
|
|
30880
31152
|
startIdleSweep(state);
|
|
31153
|
+
function settlingCopyWork(indexName) {
|
|
31154
|
+
return state.copyTransitions.get(indexName)?.done ?? state.compactionsInFlight.get(indexName);
|
|
31155
|
+
}
|
|
30881
31156
|
async function resyncIndex(indexName, wasScaledOut) {
|
|
31157
|
+
for (let pending = settlingCopyWork(indexName); pending !== void 0; pending = settlingCopyWork(indexName)) {
|
|
31158
|
+
await pending;
|
|
31159
|
+
}
|
|
30882
31160
|
state.desyncedIndexes.delete(indexName);
|
|
30883
31161
|
if (!wasScaledOut) return;
|
|
30884
31162
|
await scaleOutIndex(state, indexName, copyThresholdReason(state, indexName));
|
|
@@ -31230,6 +31508,10 @@ function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
31230
31508
|
getManager: (indexName) => executor.getManager(indexName),
|
|
31231
31509
|
desyncIndex: (indexName) => orchestrator.desyncIndex(indexName),
|
|
31232
31510
|
resyncIndex: (indexName, wasPromoted) => orchestrator.resyncIndex(indexName, wasPromoted),
|
|
31511
|
+
recordCurrentAnalysis: (indexName) => {
|
|
31512
|
+
const entry = indexRegistry.get(indexName);
|
|
31513
|
+
if (entry !== void 0 && !analysisRebuild.isStale(indexName)) entry.analysisRevision = entry.language.revision;
|
|
31514
|
+
},
|
|
31233
31515
|
durabilityManager: durability?.manager ?? null
|
|
31234
31516
|
});
|
|
31235
31517
|
const indexState = wireIndexState({
|
|
@@ -31403,11 +31685,16 @@ function refuseCode(entry) {
|
|
|
31403
31685
|
}
|
|
31404
31686
|
function indexConfigurationOf(entry) {
|
|
31405
31687
|
refuseCode(entry);
|
|
31406
|
-
return configurationPayloadOf(entry.config, entry.language, entry.embeddingAdapterName);
|
|
31688
|
+
return configurationPayloadOf(entry.config, entry.language, entry.embeddingAdapterName, entry.analysisRevision);
|
|
31407
31689
|
}
|
|
31408
31690
|
function requireConfigurationWithinSizeLimit(config, language) {
|
|
31409
31691
|
const adapter = config.embedding?.adapter;
|
|
31410
|
-
const payload = configurationPayloadOf(
|
|
31692
|
+
const payload = configurationPayloadOf(
|
|
31693
|
+
config,
|
|
31694
|
+
language,
|
|
31695
|
+
typeof adapter === "string" ? adapter : null,
|
|
31696
|
+
language.revision
|
|
31697
|
+
);
|
|
31411
31698
|
if (messagePackUpperBoundBytes(payload) <= MAX_INDEX_CONFIGURATION_BYTES) return;
|
|
31412
31699
|
const bytes = encode(payload).byteLength;
|
|
31413
31700
|
if (bytes <= MAX_INDEX_CONFIGURATION_BYTES) return;
|
|
@@ -31417,12 +31704,12 @@ function requireConfigurationWithinSizeLimit(config, language) {
|
|
|
31417
31704
|
{ bytes, maximum: MAX_INDEX_CONFIGURATION_BYTES }
|
|
31418
31705
|
);
|
|
31419
31706
|
}
|
|
31420
|
-
function configurationPayloadOf(config, language, adapterName) {
|
|
31707
|
+
function configurationPayloadOf(config, language, adapterName, analysisRevision) {
|
|
31421
31708
|
const bm25 = config.bm25;
|
|
31422
31709
|
return {
|
|
31423
31710
|
schema: config.schema,
|
|
31424
31711
|
language: language.name,
|
|
31425
|
-
analysisRevision:
|
|
31712
|
+
...analysisRevision !== null ? { analysisRevision } : {},
|
|
31426
31713
|
...typeof config.tokenizer === "string" ? { tokenizer: config.tokenizer } : {},
|
|
31427
31714
|
...typeof config.stopWords === "string" ? { stopWords: config.stopWords } : {},
|
|
31428
31715
|
...config.stopWords instanceof Set ? { stopWordList: [...config.stopWords].sort(compareCodePoints) } : {},
|
|
@@ -31581,7 +31868,8 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
31581
31868
|
indexUuid: indexUuid ?? null,
|
|
31582
31869
|
heldPartitions: null,
|
|
31583
31870
|
documentCount: 0,
|
|
31584
|
-
partitionCount: indexConfig.partitions?.maxPartitions ?? 1
|
|
31871
|
+
partitionCount: indexConfig.partitions?.maxPartitions ?? 1,
|
|
31872
|
+
analysisRevision: language.revision
|
|
31585
31873
|
});
|
|
31586
31874
|
await core.indexState.registerOpen(name);
|
|
31587
31875
|
if (core.durability) {
|
|
@@ -31593,6 +31881,7 @@ async function dropEngineIndex(core, name) {
|
|
|
31593
31881
|
core.guardShutdown();
|
|
31594
31882
|
const entry = core.requireIndex(name);
|
|
31595
31883
|
await core.indexState.drop(name, async () => {
|
|
31884
|
+
await core.analysisRebuild.abandon(name);
|
|
31596
31885
|
await core.orchestrator.closeIndex(name);
|
|
31597
31886
|
if (core.durability) {
|
|
31598
31887
|
try {
|
|
@@ -34397,7 +34686,7 @@ function partitionVectorPlan(vectorIndex, docIds) {
|
|
|
34397
34686
|
codes: null,
|
|
34398
34687
|
vectors: vectorsToBytes(packed)
|
|
34399
34688
|
};
|
|
34400
|
-
files.push({ payload, ordinals: new Int32Array(slice.length) });
|
|
34689
|
+
files.push({ payload, ordinals: new Int32Array(slice.length), dead: null });
|
|
34401
34690
|
}
|
|
34402
34691
|
return {
|
|
34403
34692
|
unchanged: false,
|
|
@@ -34893,7 +35182,8 @@ async function createRestoredIndex(indexName, prepared, partitionCount, deps) {
|
|
|
34893
35182
|
indexUuid: null,
|
|
34894
35183
|
heldPartitions: null,
|
|
34895
35184
|
documentCount: 0,
|
|
34896
|
-
partitionCount
|
|
35185
|
+
partitionCount,
|
|
35186
|
+
analysisRevision: prepared.storedRevision
|
|
34897
35187
|
});
|
|
34898
35188
|
await deps.indexState.registerOpen(indexName);
|
|
34899
35189
|
const release = await deps.indexState.acquire(indexName, false);
|
|
@@ -36790,6 +37080,15 @@ async function enterRead(core, indexName) {
|
|
|
36790
37080
|
core.guardShutdown();
|
|
36791
37081
|
if (core.rebalancingIndexes.has(indexName)) await yieldToEventLoop();
|
|
36792
37082
|
}
|
|
37083
|
+
async function readUnderOneAnalysis(core, indexName, read) {
|
|
37084
|
+
let analysisStale = core.analysisRebuild.isStale(indexName);
|
|
37085
|
+
let result = await read();
|
|
37086
|
+
while (core.analysisRebuild.isStale(indexName) !== analysisStale) {
|
|
37087
|
+
analysisStale = core.analysisRebuild.isStale(indexName);
|
|
37088
|
+
result = await read();
|
|
37089
|
+
}
|
|
37090
|
+
return { result, analysisStale };
|
|
37091
|
+
}
|
|
36793
37092
|
function scopedParams(params, options) {
|
|
36794
37093
|
{
|
|
36795
37094
|
return params;
|
|
@@ -36817,7 +37116,7 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
36817
37116
|
entry.embeddingAdapterName
|
|
36818
37117
|
);
|
|
36819
37118
|
const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
|
|
36820
|
-
const
|
|
37119
|
+
const context = {
|
|
36821
37120
|
manager,
|
|
36822
37121
|
language: entry.language,
|
|
36823
37122
|
config: entry.config,
|
|
@@ -36827,7 +37126,15 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
36827
37126
|
partitionIds: options?.partitionIds,
|
|
36828
37127
|
cursorBinding: queryBindingOf(params),
|
|
36829
37128
|
patternWorkCap: core.patternWorkCap
|
|
36830
|
-
}
|
|
37129
|
+
};
|
|
37130
|
+
const { result, analysisStale } = await readUnderOneAnalysis(
|
|
37131
|
+
core,
|
|
37132
|
+
indexName,
|
|
37133
|
+
() => executeQuery(resolvedParams, context)
|
|
37134
|
+
);
|
|
37135
|
+
if (analysisStale) {
|
|
37136
|
+
result.analysisStale = true;
|
|
37137
|
+
}
|
|
36831
37138
|
if (core.pluginRegistry.hasHooks("afterSearch")) {
|
|
36832
37139
|
const observed = { ...result, hits: [...result.hits] };
|
|
36833
37140
|
try {
|
|
@@ -36836,9 +37143,6 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
36836
37143
|
console.warn("afterSearch plugin hook error:", err);
|
|
36837
37144
|
}
|
|
36838
37145
|
}
|
|
36839
|
-
if (core.analysisRebuild.isStale(indexName)) {
|
|
36840
|
-
result.analysisStale = true;
|
|
36841
|
-
}
|
|
36842
37146
|
return result;
|
|
36843
37147
|
} finally {
|
|
36844
37148
|
release();
|
|
@@ -36859,7 +37163,7 @@ async function runEnginePreflight(core, indexName, params, options) {
|
|
|
36859
37163
|
entry.embeddingAdapterName
|
|
36860
37164
|
);
|
|
36861
37165
|
const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
|
|
36862
|
-
const
|
|
37166
|
+
const context = {
|
|
36863
37167
|
manager,
|
|
36864
37168
|
language: entry.language,
|
|
36865
37169
|
config: entry.config,
|
|
@@ -36869,8 +37173,13 @@ async function runEnginePreflight(core, indexName, params, options) {
|
|
|
36869
37173
|
partitionIds: options?.partitionIds,
|
|
36870
37174
|
cursorBinding: queryBindingOf(params),
|
|
36871
37175
|
patternWorkCap: core.patternWorkCap
|
|
36872
|
-
}
|
|
36873
|
-
|
|
37176
|
+
};
|
|
37177
|
+
const { result, analysisStale } = await readUnderOneAnalysis(
|
|
37178
|
+
core,
|
|
37179
|
+
indexName,
|
|
37180
|
+
() => executePreflight(resolvedParams, context)
|
|
37181
|
+
);
|
|
37182
|
+
if (analysisStale) {
|
|
36874
37183
|
result.analysisStale = true;
|
|
36875
37184
|
}
|
|
36876
37185
|
return result;
|