@delali/narsil 0.3.1-next.83 → 0.3.1-next.84
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-CUP2E4OH.mjs → chunk-IA67SOJO.mjs} +102 -106
- package/dist/{chunk-7JZZ3HNV.mjs → chunk-XIQZKKTK.mjs} +177 -39
- package/dist/{direct-executor-6HASZ36Y.mjs → direct-executor-6WWFY5EI.mjs} +1 -1
- package/dist/distribution.mjs +19 -4
- package/dist/index.browser.mjs +277 -143
- package/dist/index.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +2 -2
- package/dist/workers/entry.mjs +1 -1
- package/package.json +7 -7
package/dist/index.browser.mjs
CHANGED
|
@@ -7612,65 +7612,6 @@ function mergeSegmentState(target, source) {
|
|
|
7612
7612
|
target.scoreBuffer = null;
|
|
7613
7613
|
}
|
|
7614
7614
|
|
|
7615
|
-
// src/core/partition/rebuild.ts
|
|
7616
|
-
var EMPTY_STATS = {
|
|
7617
|
-
totalDocuments: 0,
|
|
7618
|
-
totalFieldLengths: {},
|
|
7619
|
-
averageFieldLengths: {},
|
|
7620
|
-
docFrequencies: {}
|
|
7621
|
-
};
|
|
7622
|
-
function textFieldsOf(fields) {
|
|
7623
|
-
return fields.filter((field) => isWordIndexedBase(field.base));
|
|
7624
|
-
}
|
|
7625
|
-
function rebuildTextIndex(state, schema, language, options) {
|
|
7626
|
-
const textFields = textFieldsOf(schemaFieldsOf(schema));
|
|
7627
|
-
state.invertedIdx.clear();
|
|
7628
|
-
state.surfaceRegistry.clear();
|
|
7629
|
-
state.stats.deserialize(EMPTY_STATS);
|
|
7630
|
-
state.invertedIdx.beginBatch();
|
|
7631
|
-
try {
|
|
7632
|
-
for (const [docId, stored] of state.docStore.all()) {
|
|
7633
|
-
const internalId = state.docStore.getInternalId(docId);
|
|
7634
|
-
if (internalId === void 0) continue;
|
|
7635
|
-
const fields = stored.fields;
|
|
7636
|
-
const fieldLengths = {};
|
|
7637
|
-
const tokensByField = {};
|
|
7638
|
-
for (const { path: fieldPath, base } of textFields) {
|
|
7639
|
-
const value = getNestedValue(fields, fieldPath);
|
|
7640
|
-
if (value === void 0 || value === null) continue;
|
|
7641
|
-
if (base === "string") {
|
|
7642
|
-
indexStringField(
|
|
7643
|
-
state,
|
|
7644
|
-
internalId,
|
|
7645
|
-
fieldPath,
|
|
7646
|
-
value,
|
|
7647
|
-
language,
|
|
7648
|
-
options,
|
|
7649
|
-
fieldLengths,
|
|
7650
|
-
tokensByField
|
|
7651
|
-
);
|
|
7652
|
-
} else {
|
|
7653
|
-
indexStringArrayField(
|
|
7654
|
-
state,
|
|
7655
|
-
internalId,
|
|
7656
|
-
fieldPath,
|
|
7657
|
-
value,
|
|
7658
|
-
language,
|
|
7659
|
-
options,
|
|
7660
|
-
fieldLengths,
|
|
7661
|
-
tokensByField
|
|
7662
|
-
);
|
|
7663
|
-
}
|
|
7664
|
-
}
|
|
7665
|
-
state.docStore.storeRef(docId, stored.fields, fieldLengths);
|
|
7666
|
-
state.stats.addDocument(fieldLengths, tokensByField);
|
|
7667
|
-
}
|
|
7668
|
-
} finally {
|
|
7669
|
-
state.invertedIdx.endBatch();
|
|
7670
|
-
state.partialWords.forgetSpellings();
|
|
7671
|
-
}
|
|
7672
|
-
}
|
|
7673
|
-
|
|
7674
7615
|
// src/core/partition/block-bounds.ts
|
|
7675
7616
|
var ABSENT_FIELD_LENGTH_BOUND = 0;
|
|
7676
7617
|
var boundsByList = /* @__PURE__ */ new WeakMap();
|
|
@@ -9163,9 +9104,6 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
9163
9104
|
if (internalId !== void 0) recordSortValues(state, internalId, document2);
|
|
9164
9105
|
}
|
|
9165
9106
|
},
|
|
9166
|
-
rebuildTextIndex(schema, language, options) {
|
|
9167
|
-
rebuildTextIndex(state, schema, language, options);
|
|
9168
|
-
},
|
|
9169
9107
|
get(docId, projection) {
|
|
9170
9108
|
const stored = state.docStore.get(docId);
|
|
9171
9109
|
if (!stored) return void 0;
|
|
@@ -9259,6 +9197,8 @@ function yieldToEventLoop() {
|
|
|
9259
9197
|
// src/partitioning/constants.ts
|
|
9260
9198
|
var REBALANCE_CHUNK_SIZE = 1e3;
|
|
9261
9199
|
var DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE = 1e4;
|
|
9200
|
+
var TEXT_REBUILD_CHUNK_DOCUMENTS = 1e3;
|
|
9201
|
+
var TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS = 1e3;
|
|
9262
9202
|
|
|
9263
9203
|
// src/partitioning/rebalancer.ts
|
|
9264
9204
|
function createRebalancer() {
|
|
@@ -11162,18 +11102,6 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
11162
11102
|
if (frozen.length === 0) return live.encodeSegment();
|
|
11163
11103
|
return composedScratch().encodeSegment();
|
|
11164
11104
|
},
|
|
11165
|
-
rebuildTextIndex(schema, language, options) {
|
|
11166
|
-
invalidateDocFrequencies();
|
|
11167
|
-
if (frozen.length > 0) {
|
|
11168
|
-
for (const segment of frozen) {
|
|
11169
|
-
const payload = encodeSegmentState(segment);
|
|
11170
|
-
if (payload.documentCount === 0) continue;
|
|
11171
|
-
live.mergeSegmentPayload(payload, survivorDocumentsOf(segment, payload));
|
|
11172
|
-
}
|
|
11173
|
-
frozen.length = 0;
|
|
11174
|
-
}
|
|
11175
|
-
live.rebuildTextIndex(schema, language, options);
|
|
11176
|
-
},
|
|
11177
11105
|
serialize(indexName, totalPartitions, language, schema) {
|
|
11178
11106
|
if (frozen.length === 0) return live.serialize(indexName, totalPartitions, language, schema);
|
|
11179
11107
|
return composedScratch().serialize(indexName, totalPartitions, language, schema);
|
|
@@ -11417,6 +11345,81 @@ function setNestedValue(obj, path, value) {
|
|
|
11417
11345
|
current[segments[segments.length - 1]] = value;
|
|
11418
11346
|
}
|
|
11419
11347
|
|
|
11348
|
+
// src/partitioning/manager/stats.ts
|
|
11349
|
+
function aggregateStatsOf(partitions) {
|
|
11350
|
+
let totalDocuments = 0;
|
|
11351
|
+
const docFrequencies = /* @__PURE__ */ Object.create(null);
|
|
11352
|
+
const totalFieldLengths = /* @__PURE__ */ Object.create(null);
|
|
11353
|
+
for (let i = 0; i < partitions.length; i++) {
|
|
11354
|
+
const stats = partitions[i].stats;
|
|
11355
|
+
totalDocuments += stats.totalDocuments;
|
|
11356
|
+
for (const [term, freq] of Object.entries(stats.docFrequencies)) {
|
|
11357
|
+
docFrequencies[term] = (docFrequencies[term] ?? 0) + freq;
|
|
11358
|
+
}
|
|
11359
|
+
for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
|
|
11360
|
+
totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
|
|
11361
|
+
}
|
|
11362
|
+
}
|
|
11363
|
+
return { totalDocuments, docFrequencies, totalFieldLengths };
|
|
11364
|
+
}
|
|
11365
|
+
function estimateBytesOf(partitions, vecIndexes) {
|
|
11366
|
+
let total = 0;
|
|
11367
|
+
for (let i = 0; i < partitions.length; i++) {
|
|
11368
|
+
total += partitions[i].estimateMemoryBytes();
|
|
11369
|
+
}
|
|
11370
|
+
for (const [, vecIndex] of vecIndexes) {
|
|
11371
|
+
total += vecIndex.estimateMemoryBytes();
|
|
11372
|
+
}
|
|
11373
|
+
return total;
|
|
11374
|
+
}
|
|
11375
|
+
function partitionStatsOf(partitions) {
|
|
11376
|
+
return partitions.map((p, i) => ({
|
|
11377
|
+
partitionId: i,
|
|
11378
|
+
documentCount: p.count(),
|
|
11379
|
+
estimatedMemoryBytes: p.estimateMemoryBytes()
|
|
11380
|
+
}));
|
|
11381
|
+
}
|
|
11382
|
+
|
|
11383
|
+
// src/partitioning/manager/write-log.ts
|
|
11384
|
+
function createWriteLogRecorder() {
|
|
11385
|
+
let current = null;
|
|
11386
|
+
return {
|
|
11387
|
+
open() {
|
|
11388
|
+
const log = { docIds: /* @__PURE__ */ new Set(), replaced: false };
|
|
11389
|
+
current = log;
|
|
11390
|
+
return {
|
|
11391
|
+
get partitionsReplaced() {
|
|
11392
|
+
return log.replaced;
|
|
11393
|
+
},
|
|
11394
|
+
get pendingDocuments() {
|
|
11395
|
+
return log.docIds.size;
|
|
11396
|
+
},
|
|
11397
|
+
takeWrittenDocIds() {
|
|
11398
|
+
const taken = log.docIds;
|
|
11399
|
+
log.docIds = /* @__PURE__ */ new Set();
|
|
11400
|
+
return taken;
|
|
11401
|
+
},
|
|
11402
|
+
close() {
|
|
11403
|
+
if (current === log) current = null;
|
|
11404
|
+
}
|
|
11405
|
+
};
|
|
11406
|
+
},
|
|
11407
|
+
documentWritten(docId) {
|
|
11408
|
+
if (current !== null) current.docIds.add(docId);
|
|
11409
|
+
},
|
|
11410
|
+
segmentWritten(segment) {
|
|
11411
|
+
if (current === null) return;
|
|
11412
|
+
for (const internalId of segment.docStore.allInternalIds()) {
|
|
11413
|
+
const docId = segment.docStore.getExternalId(internalId);
|
|
11414
|
+
if (docId !== void 0) current.docIds.add(docId);
|
|
11415
|
+
}
|
|
11416
|
+
},
|
|
11417
|
+
partitionsReplaced() {
|
|
11418
|
+
if (current !== null) current.replaced = true;
|
|
11419
|
+
}
|
|
11420
|
+
};
|
|
11421
|
+
}
|
|
11422
|
+
|
|
11420
11423
|
// src/partitioning/manager/index.ts
|
|
11421
11424
|
function createPartitionManager(indexName, config, language, router, initialPartitionCount, vectorIndexes) {
|
|
11422
11425
|
requirePatternSearchFor(config.schema);
|
|
@@ -11428,6 +11431,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11428
11431
|
const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
|
|
11429
11432
|
let partitions = [];
|
|
11430
11433
|
const docPartitionMap = /* @__PURE__ */ new Map();
|
|
11434
|
+
const writeLog = createWriteLogRecorder();
|
|
11431
11435
|
for (let i = 0; i < count; i++) {
|
|
11432
11436
|
partitions.push(createPartitionIndex(i, trackPositions));
|
|
11433
11437
|
}
|
|
@@ -11515,9 +11519,23 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11515
11519
|
return [...partitions];
|
|
11516
11520
|
},
|
|
11517
11521
|
setPartitions(newPartitions) {
|
|
11522
|
+
writeLog.partitionsReplaced();
|
|
11518
11523
|
partitions = [...newPartitions];
|
|
11519
11524
|
rebuildDocPartitionMap();
|
|
11520
11525
|
},
|
|
11526
|
+
openWriteLog() {
|
|
11527
|
+
return writeLog.open();
|
|
11528
|
+
},
|
|
11529
|
+
adoptRebuiltPartitions(rebuilt) {
|
|
11530
|
+
if (rebuilt.length !== partitions.length) {
|
|
11531
|
+
throw new NarsilError(
|
|
11532
|
+
ErrorCodes.PARTITION_CORRUPTED,
|
|
11533
|
+
`Index "${indexName}" holds ${partitions.length} partitions, but the rebuild produced ${rebuilt.length}`,
|
|
11534
|
+
{ indexName, partitionCount: partitions.length, rebuiltCount: rebuilt.length }
|
|
11535
|
+
);
|
|
11536
|
+
}
|
|
11537
|
+
partitions = [...rebuilt];
|
|
11538
|
+
},
|
|
11521
11539
|
addPartition() {
|
|
11522
11540
|
const currentMaxPartitions = config.partitions?.maxPartitions;
|
|
11523
11541
|
if (currentMaxPartitions !== void 0 && partitions.length >= currentMaxPartitions) {
|
|
@@ -11528,11 +11546,13 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11528
11546
|
);
|
|
11529
11547
|
}
|
|
11530
11548
|
const partition = createPartitionIndex(partitions.length, trackPositions);
|
|
11549
|
+
writeLog.partitionsReplaced();
|
|
11531
11550
|
partitions.push(partition);
|
|
11532
11551
|
return partition;
|
|
11533
11552
|
},
|
|
11534
11553
|
removePartition(partitionId) {
|
|
11535
11554
|
validatePartitionId(partitionId);
|
|
11555
|
+
writeLog.partitionsReplaced();
|
|
11536
11556
|
partitions.splice(partitionId, 1);
|
|
11537
11557
|
rebuildDocPartitionMap();
|
|
11538
11558
|
},
|
|
@@ -11546,6 +11566,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11546
11566
|
if (newCount >= partitions.length) {
|
|
11547
11567
|
return;
|
|
11548
11568
|
}
|
|
11569
|
+
writeLog.partitionsReplaced();
|
|
11549
11570
|
for (const [docId, pid] of docPartitionMap) {
|
|
11550
11571
|
if (pid >= newCount) {
|
|
11551
11572
|
docPartitionMap.delete(docId);
|
|
@@ -11568,6 +11589,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11568
11589
|
const insertOpts = resolveInsertOptions(options);
|
|
11569
11590
|
partitions[pid].insert(docId, document2, config.schema, language, insertOpts);
|
|
11570
11591
|
docPartitionMap.set(docId, pid);
|
|
11592
|
+
writeLog.documentWritten(docId);
|
|
11571
11593
|
},
|
|
11572
11594
|
remove(docId) {
|
|
11573
11595
|
const pid = locateDocument(docId);
|
|
@@ -11576,11 +11598,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11576
11598
|
}
|
|
11577
11599
|
partitions[pid].remove(docId, config.schema, language, resolveInsertOptions());
|
|
11578
11600
|
docPartitionMap.delete(docId);
|
|
11579
|
-
|
|
11580
|
-
rebuildTextIndex(partitionId) {
|
|
11581
|
-
validatePartitionId(partitionId);
|
|
11582
|
-
partitions[partitionId].rebuildTextIndex(config.schema, language, resolveInsertOptions());
|
|
11583
|
-
preparePartialWords(partitionId);
|
|
11601
|
+
writeLog.documentWritten(docId);
|
|
11584
11602
|
},
|
|
11585
11603
|
beginBatchRemove() {
|
|
11586
11604
|
for (let i = 0; i < partitions.length; i++) {
|
|
@@ -11600,6 +11618,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11600
11618
|
const updateOpts = resolveInsertOptions(options);
|
|
11601
11619
|
partitions[pid].update(docId, document2, config.schema, language, updateOpts);
|
|
11602
11620
|
docPartitionMap.set(docId, pid);
|
|
11621
|
+
writeLog.documentWritten(docId);
|
|
11603
11622
|
},
|
|
11604
11623
|
get(docId, projection) {
|
|
11605
11624
|
const pid = locateDocument(docId);
|
|
@@ -11650,6 +11669,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11650
11669
|
partition.endBatch();
|
|
11651
11670
|
for (const docId of payload.docIds) {
|
|
11652
11671
|
docPartitionMap.set(docId, partitionId);
|
|
11672
|
+
writeLog.documentWritten(docId);
|
|
11653
11673
|
}
|
|
11654
11674
|
},
|
|
11655
11675
|
attachFrozenSegment(partitionId, segment) {
|
|
@@ -11664,6 +11684,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11664
11684
|
}
|
|
11665
11685
|
asCompositePartition(partitionId).attachFrozenSegment(segment);
|
|
11666
11686
|
registerSegmentDocuments(partitionId, segment);
|
|
11687
|
+
writeLog.segmentWritten(segment);
|
|
11667
11688
|
},
|
|
11668
11689
|
swapFrozenSegments(partitionId, dropSegmentIds, replacement) {
|
|
11669
11690
|
validatePartitionId(partitionId);
|
|
@@ -11681,42 +11702,19 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11681
11702
|
},
|
|
11682
11703
|
deserializePartition(partitionId, data) {
|
|
11683
11704
|
validatePartitionId(partitionId);
|
|
11705
|
+
writeLog.partitionsReplaced();
|
|
11684
11706
|
partitions[partitionId].deserialize(data, config.schema);
|
|
11685
11707
|
preparePartialWords(partitionId);
|
|
11686
11708
|
rebuildDocPartitionMapForPartition(partitionId);
|
|
11687
11709
|
},
|
|
11688
11710
|
getAggregateStats() {
|
|
11689
|
-
|
|
11690
|
-
const docFrequencies = /* @__PURE__ */ Object.create(null);
|
|
11691
|
-
const totalFieldLengths = /* @__PURE__ */ Object.create(null);
|
|
11692
|
-
for (let i = 0; i < partitions.length; i++) {
|
|
11693
|
-
const stats = partitions[i].stats;
|
|
11694
|
-
totalDocuments += stats.totalDocuments;
|
|
11695
|
-
for (const [term, freq] of Object.entries(stats.docFrequencies)) {
|
|
11696
|
-
docFrequencies[term] = (docFrequencies[term] ?? 0) + freq;
|
|
11697
|
-
}
|
|
11698
|
-
for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
|
|
11699
|
-
totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
|
|
11700
|
-
}
|
|
11701
|
-
}
|
|
11702
|
-
return { totalDocuments, docFrequencies, totalFieldLengths };
|
|
11711
|
+
return aggregateStatsOf(partitions);
|
|
11703
11712
|
},
|
|
11704
11713
|
estimateMemoryBytes() {
|
|
11705
|
-
|
|
11706
|
-
for (let i = 0; i < partitions.length; i++) {
|
|
11707
|
-
total += partitions[i].estimateMemoryBytes();
|
|
11708
|
-
}
|
|
11709
|
-
for (const [, vecIndex] of vecIndexes) {
|
|
11710
|
-
total += vecIndex.estimateMemoryBytes();
|
|
11711
|
-
}
|
|
11712
|
-
return total;
|
|
11714
|
+
return estimateBytesOf(partitions, vecIndexes);
|
|
11713
11715
|
},
|
|
11714
11716
|
getPartitionStats() {
|
|
11715
|
-
return partitions
|
|
11716
|
-
partitionId: i,
|
|
11717
|
-
documentCount: p.count(),
|
|
11718
|
-
estimatedMemoryBytes: p.estimateMemoryBytes()
|
|
11719
|
-
}));
|
|
11717
|
+
return partitionStatsOf(partitions);
|
|
11720
11718
|
},
|
|
11721
11719
|
getVectorIndexes() {
|
|
11722
11720
|
return vecIndexes;
|
|
@@ -21004,6 +21002,74 @@ function createDirectExecutor(options) {
|
|
|
21004
21002
|
};
|
|
21005
21003
|
}
|
|
21006
21004
|
|
|
21005
|
+
// src/partitioning/text-rebuild.ts
|
|
21006
|
+
function interrupted(pass) {
|
|
21007
|
+
return pass.log.partitionsReplaced || !pass.hooks.isCurrent();
|
|
21008
|
+
}
|
|
21009
|
+
async function buildPartition(pass, source, partitionId) {
|
|
21010
|
+
const { manager, options } = pass;
|
|
21011
|
+
const target = createPartitionIndex(partitionId, manager.config.trackPositions ?? true);
|
|
21012
|
+
const docIds = [...source.docIds()];
|
|
21013
|
+
target.beginBatch();
|
|
21014
|
+
for (let start = 0; start < docIds.length; start += TEXT_REBUILD_CHUNK_DOCUMENTS) {
|
|
21015
|
+
const end = Math.min(start + TEXT_REBUILD_CHUNK_DOCUMENTS, docIds.length);
|
|
21016
|
+
for (let i = start; i < end; i++) {
|
|
21017
|
+
const document2 = manager.getRef(docIds[i]);
|
|
21018
|
+
if (document2 !== void 0) target.insert(docIds[i], document2, manager.schema, manager.language, options);
|
|
21019
|
+
}
|
|
21020
|
+
await yieldToEventLoop();
|
|
21021
|
+
if (interrupted(pass)) return null;
|
|
21022
|
+
}
|
|
21023
|
+
target.endBatch();
|
|
21024
|
+
if (manager.partialWordAnalysis !== null) target.preparePartialWords(manager.partialWordAnalysis);
|
|
21025
|
+
return target;
|
|
21026
|
+
}
|
|
21027
|
+
function applyWrittenDocuments(pass, built) {
|
|
21028
|
+
const { manager, options } = pass;
|
|
21029
|
+
for (const docId of pass.log.takeWrittenDocIds()) {
|
|
21030
|
+
for (const partition of built) {
|
|
21031
|
+
if (!partition.has(docId)) continue;
|
|
21032
|
+
partition.remove(docId, manager.schema, manager.language, options);
|
|
21033
|
+
break;
|
|
21034
|
+
}
|
|
21035
|
+
const partitionId = manager.partitionIdOf(docId);
|
|
21036
|
+
const document2 = manager.getRef(docId);
|
|
21037
|
+
if (partitionId === void 0 || document2 === void 0) continue;
|
|
21038
|
+
built[partitionId].insert(docId, document2, manager.schema, manager.language, options);
|
|
21039
|
+
}
|
|
21040
|
+
}
|
|
21041
|
+
async function rebuildOnce(pass) {
|
|
21042
|
+
const partitionCount = pass.manager.partitionCount;
|
|
21043
|
+
const built = [];
|
|
21044
|
+
for (let partitionId = 0; partitionId < partitionCount; partitionId++) {
|
|
21045
|
+
const partition = await buildPartition(pass, pass.manager.getPartition(partitionId), partitionId);
|
|
21046
|
+
if (partition === null) return false;
|
|
21047
|
+
built.push(partition);
|
|
21048
|
+
pass.hooks.onPartitionBuilt(built.length);
|
|
21049
|
+
}
|
|
21050
|
+
while (pass.log.pendingDocuments > TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS) {
|
|
21051
|
+
applyWrittenDocuments(pass, built);
|
|
21052
|
+
await yieldToEventLoop();
|
|
21053
|
+
if (interrupted(pass)) return false;
|
|
21054
|
+
}
|
|
21055
|
+
applyWrittenDocuments(pass, built);
|
|
21056
|
+
pass.manager.adoptRebuiltPartitions(built);
|
|
21057
|
+
pass.hooks.onSwitch();
|
|
21058
|
+
return true;
|
|
21059
|
+
}
|
|
21060
|
+
async function rebuildTextIndexes(manager, hooks) {
|
|
21061
|
+
const options = resolvePartitionInsertOptions(manager.config, manager.analysis, { validate: false, skipClone: true });
|
|
21062
|
+
while (hooks.isCurrent()) {
|
|
21063
|
+
const log = manager.openWriteLog();
|
|
21064
|
+
try {
|
|
21065
|
+
if (await rebuildOnce({ manager, log, hooks, options })) return true;
|
|
21066
|
+
} finally {
|
|
21067
|
+
log.close();
|
|
21068
|
+
}
|
|
21069
|
+
}
|
|
21070
|
+
return false;
|
|
21071
|
+
}
|
|
21072
|
+
|
|
21007
21073
|
// src/engine/events.ts
|
|
21008
21074
|
function eventHandlersOf(events) {
|
|
21009
21075
|
const handlers = /* @__PURE__ */ new Map();
|
|
@@ -21026,11 +21092,6 @@ function emitEngineEvent(eventHandlers, event, payload) {
|
|
|
21026
21092
|
}
|
|
21027
21093
|
|
|
21028
21094
|
// src/engine/analysis-rebuild.ts
|
|
21029
|
-
function yieldToEventLoop4() {
|
|
21030
|
-
return new Promise((resolve) => {
|
|
21031
|
-
setTimeout(resolve, 0);
|
|
21032
|
-
});
|
|
21033
|
-
}
|
|
21034
21095
|
function toError(value) {
|
|
21035
21096
|
return value instanceof Error ? value : new Error(String(value));
|
|
21036
21097
|
}
|
|
@@ -21041,9 +21102,13 @@ function wireAnalysisRebuild(wiring) {
|
|
|
21041
21102
|
resyncIndex: wiring.resyncIndex,
|
|
21042
21103
|
async persistAnalysisRevision(indexName) {
|
|
21043
21104
|
const durability = wiring.durabilityManager;
|
|
21044
|
-
if (durability === null)
|
|
21045
|
-
|
|
21105
|
+
if (durability === null) {
|
|
21106
|
+
wiring.recordCurrentAnalysis(indexName);
|
|
21107
|
+
return;
|
|
21108
|
+
}
|
|
21046
21109
|
await (durability.checkpointFromMemory?.(indexName) ?? durability.checkpoint(indexName));
|
|
21110
|
+
wiring.recordCurrentAnalysis(indexName);
|
|
21111
|
+
await durability.persistMetadata(indexName);
|
|
21047
21112
|
},
|
|
21048
21113
|
emit(payload) {
|
|
21049
21114
|
emitEngineEvent(wiring.eventHandlers, "analysisRebuild", payload);
|
|
@@ -21054,9 +21119,10 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
21054
21119
|
const stale = /* @__PURE__ */ new Map();
|
|
21055
21120
|
const rebuildMode = config?.rebuild ?? "auto";
|
|
21056
21121
|
let queue = Promise.resolve();
|
|
21122
|
+
let active = null;
|
|
21057
21123
|
async function rebuildNow(indexName, entry) {
|
|
21058
21124
|
const manager = deps.getManager(indexName);
|
|
21059
|
-
if (manager === void 0) {
|
|
21125
|
+
if (manager === void 0 || entry.abandoned) {
|
|
21060
21126
|
entry.run = null;
|
|
21061
21127
|
return;
|
|
21062
21128
|
}
|
|
@@ -21064,12 +21130,20 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
21064
21130
|
deps.emit({ indexName, status: "started", partitionsRebuilt: 0, partitionCount });
|
|
21065
21131
|
const wasPromoted = deps.desyncIndex(indexName);
|
|
21066
21132
|
try {
|
|
21067
|
-
|
|
21068
|
-
manager.
|
|
21069
|
-
|
|
21070
|
-
|
|
21133
|
+
const rebuilt = await rebuildTextIndexes(manager, {
|
|
21134
|
+
isCurrent: () => deps.getManager(indexName) === manager && stale.get(indexName) === entry && !entry.abandoned,
|
|
21135
|
+
onPartitionBuilt(partitionsBuilt) {
|
|
21136
|
+
entry.partitionsRebuilt = partitionsBuilt;
|
|
21137
|
+
},
|
|
21138
|
+
onSwitch() {
|
|
21139
|
+
stale.delete(indexName);
|
|
21140
|
+
}
|
|
21141
|
+
});
|
|
21142
|
+
if (!rebuilt) {
|
|
21143
|
+
entry.run = null;
|
|
21144
|
+
if (deps.getManager(indexName) === manager) await deps.resyncIndex(indexName, wasPromoted);
|
|
21145
|
+
return;
|
|
21071
21146
|
}
|
|
21072
|
-
stale.delete(indexName);
|
|
21073
21147
|
await deps.persistAnalysisRevision(indexName);
|
|
21074
21148
|
await deps.resyncIndex(indexName, wasPromoted);
|
|
21075
21149
|
deps.emit({
|
|
@@ -21080,7 +21154,6 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
21080
21154
|
});
|
|
21081
21155
|
} catch (err) {
|
|
21082
21156
|
const error = toError(err);
|
|
21083
|
-
stale.set(indexName, entry);
|
|
21084
21157
|
entry.run = null;
|
|
21085
21158
|
await deps.resyncIndex(indexName, wasPromoted).catch(() => void 0);
|
|
21086
21159
|
deps.emit({
|
|
@@ -21093,11 +21166,20 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
21093
21166
|
throw error;
|
|
21094
21167
|
}
|
|
21095
21168
|
}
|
|
21169
|
+
function runTracked(indexName, entry) {
|
|
21170
|
+
const done = rebuildNow(indexName, entry);
|
|
21171
|
+
const tracked = { indexName, entry, done };
|
|
21172
|
+
active = tracked;
|
|
21173
|
+
return done.finally(() => {
|
|
21174
|
+
if (active === tracked) active = null;
|
|
21175
|
+
});
|
|
21176
|
+
}
|
|
21096
21177
|
function enqueue(indexName, entry) {
|
|
21097
21178
|
if (entry.run !== null) {
|
|
21098
21179
|
return entry.run;
|
|
21099
21180
|
}
|
|
21100
|
-
|
|
21181
|
+
entry.abandoned = false;
|
|
21182
|
+
const run = queue.then(() => runTracked(indexName, entry));
|
|
21101
21183
|
entry.run = run;
|
|
21102
21184
|
queue = run.then(
|
|
21103
21185
|
() => void 0,
|
|
@@ -21132,7 +21214,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
21132
21214
|
if (existing !== void 0 && existing.run !== null) {
|
|
21133
21215
|
return;
|
|
21134
21216
|
}
|
|
21135
|
-
stale.set(index.indexName, { index, partitionsRebuilt: 0, run: null });
|
|
21217
|
+
stale.set(index.indexName, { index, partitionsRebuilt: 0, run: null, abandoned: false });
|
|
21136
21218
|
},
|
|
21137
21219
|
clearStale(indexName) {
|
|
21138
21220
|
stale.delete(indexName);
|
|
@@ -21143,6 +21225,14 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
21143
21225
|
isRunning(indexName) {
|
|
21144
21226
|
return stale.get(indexName)?.run !== null && stale.get(indexName)?.run !== void 0;
|
|
21145
21227
|
},
|
|
21228
|
+
async abandon(indexName) {
|
|
21229
|
+
const entry = stale.get(indexName);
|
|
21230
|
+
if (entry !== void 0 && entry.run !== null) entry.abandoned = true;
|
|
21231
|
+
const running = active;
|
|
21232
|
+
if (running === null || running.indexName !== indexName) return;
|
|
21233
|
+
running.entry.abandoned = true;
|
|
21234
|
+
await running.done.catch(() => void 0);
|
|
21235
|
+
},
|
|
21146
21236
|
review(indexName) {
|
|
21147
21237
|
const entry = stale.get(indexName);
|
|
21148
21238
|
return entry === void 0 ? Promise.resolve() : review(indexName, entry);
|
|
@@ -21809,7 +21899,7 @@ function buildEntry(fields) {
|
|
|
21809
21899
|
}
|
|
21810
21900
|
|
|
21811
21901
|
// src/serialization/chunk-checksum.ts
|
|
21812
|
-
function
|
|
21902
|
+
function yieldToEventLoop4() {
|
|
21813
21903
|
if (typeof setImmediate === "function") {
|
|
21814
21904
|
return new Promise((resolve) => setImmediate(resolve));
|
|
21815
21905
|
}
|
|
@@ -21821,7 +21911,7 @@ async function advanceChecksum(start, payload) {
|
|
|
21821
21911
|
const end = Math.min(offset + CHECKSUM_YIELD_CHUNK_BYTES, payload.length);
|
|
21822
21912
|
state = crc32Update(state, payload.subarray(offset, end));
|
|
21823
21913
|
if (end < payload.length) {
|
|
21824
|
-
await
|
|
21914
|
+
await yieldToEventLoop4();
|
|
21825
21915
|
}
|
|
21826
21916
|
}
|
|
21827
21917
|
return state;
|
|
@@ -25830,7 +25920,7 @@ function removeWhatTheSegmentReplaces(manager, segment, contents) {
|
|
|
25830
25920
|
manager.endBatchRemove();
|
|
25831
25921
|
}
|
|
25832
25922
|
}
|
|
25833
|
-
function
|
|
25923
|
+
function yieldToEventLoop5() {
|
|
25834
25924
|
return new Promise((resolve) => setTimeout(resolve, 0));
|
|
25835
25925
|
}
|
|
25836
25926
|
async function loadPartitionSegments(store, partition, manager, options) {
|
|
@@ -25841,7 +25931,7 @@ async function loadPartitionSegments(store, partition, manager, options) {
|
|
|
25841
25931
|
const segment = createSharedFrozenSegment(shareSegmentArrays(contents.arrays, manager.partialWordAnalysis));
|
|
25842
25932
|
removeWhatTheSegmentReplaces(manager, segment, contents);
|
|
25843
25933
|
if (contents.arrays.documentCount > 0) manager.attachFrozenSegment(partition.partitionId, segment);
|
|
25844
|
-
await
|
|
25934
|
+
await yieldToEventLoop5();
|
|
25845
25935
|
}
|
|
25846
25936
|
}
|
|
25847
25937
|
|
|
@@ -28269,7 +28359,7 @@ function metadataConfigOf(entry) {
|
|
|
28269
28359
|
b: entry.config.bm25?.b ?? 0.75,
|
|
28270
28360
|
...embedding !== void 0 ? { embedding } : {},
|
|
28271
28361
|
surfaceForms: entry.config.surfaceForms !== false,
|
|
28272
|
-
analysisRevision: entry.
|
|
28362
|
+
...entry.analysisRevision !== null ? { analysisRevision: entry.analysisRevision } : {},
|
|
28273
28363
|
...typeof entry.config.tokenizer === "string" ? { tokenizer: entry.config.tokenizer } : {},
|
|
28274
28364
|
...typeof entry.config.stopWords === "string" ? { stopWords: entry.config.stopWords } : {},
|
|
28275
28365
|
...entry.config.stopWords instanceof Set ? { stopWordList: [...entry.config.stopWords].sort(compareCodePoints) } : {},
|
|
@@ -28510,7 +28600,8 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
28510
28600
|
indexUuid: metadata.indexUuid ?? null,
|
|
28511
28601
|
heldPartitions: metadata.heldPartitions ?? null,
|
|
28512
28602
|
documentCount: metadata.documentCount ?? 0,
|
|
28513
|
-
partitionCount: metadata.partitionCount
|
|
28603
|
+
partitionCount: metadata.partitionCount,
|
|
28604
|
+
analysisRevision: metadata.analysisRevision ?? null
|
|
28514
28605
|
});
|
|
28515
28606
|
if (loadData) await deps.indexState().registerOpen(metadata.indexName);
|
|
28516
28607
|
else deps.indexState().registerClosed(metadata.indexName);
|
|
@@ -28633,8 +28724,14 @@ function createIndexStateCoordinator(config, callbacks) {
|
|
|
28633
28724
|
return entry;
|
|
28634
28725
|
}
|
|
28635
28726
|
}
|
|
28727
|
+
function transitionPending(indexName, entry) {
|
|
28728
|
+
if (entries.get(indexName) !== entry) return true;
|
|
28729
|
+
if (entry.state === "dropping" && entry.dropPromise !== null) return true;
|
|
28730
|
+
return entry.state === "closing" && entry.closePromise !== null;
|
|
28731
|
+
}
|
|
28636
28732
|
async function reopen(indexName, explicit) {
|
|
28637
|
-
|
|
28733
|
+
let entry = await awaitPendingTransitions(indexName);
|
|
28734
|
+
while (transitionPending(indexName, entry)) entry = await awaitPendingTransitions(indexName);
|
|
28638
28735
|
if (entry.state === "open") return;
|
|
28639
28736
|
if (explicit) {
|
|
28640
28737
|
entry.reopenFailures = 0;
|
|
@@ -31053,7 +31150,13 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
31053
31150
|
idleSweep: null
|
|
31054
31151
|
};
|
|
31055
31152
|
startIdleSweep(state);
|
|
31153
|
+
function settlingCopyWork(indexName) {
|
|
31154
|
+
return state.copyTransitions.get(indexName)?.done ?? state.compactionsInFlight.get(indexName);
|
|
31155
|
+
}
|
|
31056
31156
|
async function resyncIndex(indexName, wasScaledOut) {
|
|
31157
|
+
for (let pending = settlingCopyWork(indexName); pending !== void 0; pending = settlingCopyWork(indexName)) {
|
|
31158
|
+
await pending;
|
|
31159
|
+
}
|
|
31057
31160
|
state.desyncedIndexes.delete(indexName);
|
|
31058
31161
|
if (!wasScaledOut) return;
|
|
31059
31162
|
await scaleOutIndex(state, indexName, copyThresholdReason(state, indexName));
|
|
@@ -31405,6 +31508,10 @@ function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
31405
31508
|
getManager: (indexName) => executor.getManager(indexName),
|
|
31406
31509
|
desyncIndex: (indexName) => orchestrator.desyncIndex(indexName),
|
|
31407
31510
|
resyncIndex: (indexName, wasPromoted) => orchestrator.resyncIndex(indexName, wasPromoted),
|
|
31511
|
+
recordCurrentAnalysis: (indexName) => {
|
|
31512
|
+
const entry = indexRegistry.get(indexName);
|
|
31513
|
+
if (entry !== void 0 && !analysisRebuild.isStale(indexName)) entry.analysisRevision = entry.language.revision;
|
|
31514
|
+
},
|
|
31408
31515
|
durabilityManager: durability?.manager ?? null
|
|
31409
31516
|
});
|
|
31410
31517
|
const indexState = wireIndexState({
|
|
@@ -31578,11 +31685,16 @@ function refuseCode(entry) {
|
|
|
31578
31685
|
}
|
|
31579
31686
|
function indexConfigurationOf(entry) {
|
|
31580
31687
|
refuseCode(entry);
|
|
31581
|
-
return configurationPayloadOf(entry.config, entry.language, entry.embeddingAdapterName);
|
|
31688
|
+
return configurationPayloadOf(entry.config, entry.language, entry.embeddingAdapterName, entry.analysisRevision);
|
|
31582
31689
|
}
|
|
31583
31690
|
function requireConfigurationWithinSizeLimit(config, language) {
|
|
31584
31691
|
const adapter = config.embedding?.adapter;
|
|
31585
|
-
const payload = configurationPayloadOf(
|
|
31692
|
+
const payload = configurationPayloadOf(
|
|
31693
|
+
config,
|
|
31694
|
+
language,
|
|
31695
|
+
typeof adapter === "string" ? adapter : null,
|
|
31696
|
+
language.revision
|
|
31697
|
+
);
|
|
31586
31698
|
if (messagePackUpperBoundBytes(payload) <= MAX_INDEX_CONFIGURATION_BYTES) return;
|
|
31587
31699
|
const bytes = encode(payload).byteLength;
|
|
31588
31700
|
if (bytes <= MAX_INDEX_CONFIGURATION_BYTES) return;
|
|
@@ -31592,12 +31704,12 @@ function requireConfigurationWithinSizeLimit(config, language) {
|
|
|
31592
31704
|
{ bytes, maximum: MAX_INDEX_CONFIGURATION_BYTES }
|
|
31593
31705
|
);
|
|
31594
31706
|
}
|
|
31595
|
-
function configurationPayloadOf(config, language, adapterName) {
|
|
31707
|
+
function configurationPayloadOf(config, language, adapterName, analysisRevision) {
|
|
31596
31708
|
const bm25 = config.bm25;
|
|
31597
31709
|
return {
|
|
31598
31710
|
schema: config.schema,
|
|
31599
31711
|
language: language.name,
|
|
31600
|
-
analysisRevision:
|
|
31712
|
+
...analysisRevision !== null ? { analysisRevision } : {},
|
|
31601
31713
|
...typeof config.tokenizer === "string" ? { tokenizer: config.tokenizer } : {},
|
|
31602
31714
|
...typeof config.stopWords === "string" ? { stopWords: config.stopWords } : {},
|
|
31603
31715
|
...config.stopWords instanceof Set ? { stopWordList: [...config.stopWords].sort(compareCodePoints) } : {},
|
|
@@ -31756,7 +31868,8 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
31756
31868
|
indexUuid: indexUuid ?? null,
|
|
31757
31869
|
heldPartitions: null,
|
|
31758
31870
|
documentCount: 0,
|
|
31759
|
-
partitionCount: indexConfig.partitions?.maxPartitions ?? 1
|
|
31871
|
+
partitionCount: indexConfig.partitions?.maxPartitions ?? 1,
|
|
31872
|
+
analysisRevision: language.revision
|
|
31760
31873
|
});
|
|
31761
31874
|
await core.indexState.registerOpen(name);
|
|
31762
31875
|
if (core.durability) {
|
|
@@ -31768,6 +31881,7 @@ async function dropEngineIndex(core, name) {
|
|
|
31768
31881
|
core.guardShutdown();
|
|
31769
31882
|
const entry = core.requireIndex(name);
|
|
31770
31883
|
await core.indexState.drop(name, async () => {
|
|
31884
|
+
await core.analysisRebuild.abandon(name);
|
|
31771
31885
|
await core.orchestrator.closeIndex(name);
|
|
31772
31886
|
if (core.durability) {
|
|
31773
31887
|
try {
|
|
@@ -35068,7 +35182,8 @@ async function createRestoredIndex(indexName, prepared, partitionCount, deps) {
|
|
|
35068
35182
|
indexUuid: null,
|
|
35069
35183
|
heldPartitions: null,
|
|
35070
35184
|
documentCount: 0,
|
|
35071
|
-
partitionCount
|
|
35185
|
+
partitionCount,
|
|
35186
|
+
analysisRevision: prepared.storedRevision
|
|
35072
35187
|
});
|
|
35073
35188
|
await deps.indexState.registerOpen(indexName);
|
|
35074
35189
|
const release = await deps.indexState.acquire(indexName, false);
|
|
@@ -36965,6 +37080,15 @@ async function enterRead(core, indexName) {
|
|
|
36965
37080
|
core.guardShutdown();
|
|
36966
37081
|
if (core.rebalancingIndexes.has(indexName)) await yieldToEventLoop();
|
|
36967
37082
|
}
|
|
37083
|
+
async function readUnderOneAnalysis(core, indexName, read) {
|
|
37084
|
+
let analysisStale = core.analysisRebuild.isStale(indexName);
|
|
37085
|
+
let result = await read();
|
|
37086
|
+
while (core.analysisRebuild.isStale(indexName) !== analysisStale) {
|
|
37087
|
+
analysisStale = core.analysisRebuild.isStale(indexName);
|
|
37088
|
+
result = await read();
|
|
37089
|
+
}
|
|
37090
|
+
return { result, analysisStale };
|
|
37091
|
+
}
|
|
36968
37092
|
function scopedParams(params, options) {
|
|
36969
37093
|
{
|
|
36970
37094
|
return params;
|
|
@@ -36992,7 +37116,7 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
36992
37116
|
entry.embeddingAdapterName
|
|
36993
37117
|
);
|
|
36994
37118
|
const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
|
|
36995
|
-
const
|
|
37119
|
+
const context = {
|
|
36996
37120
|
manager,
|
|
36997
37121
|
language: entry.language,
|
|
36998
37122
|
config: entry.config,
|
|
@@ -37002,7 +37126,15 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
37002
37126
|
partitionIds: options?.partitionIds,
|
|
37003
37127
|
cursorBinding: queryBindingOf(params),
|
|
37004
37128
|
patternWorkCap: core.patternWorkCap
|
|
37005
|
-
}
|
|
37129
|
+
};
|
|
37130
|
+
const { result, analysisStale } = await readUnderOneAnalysis(
|
|
37131
|
+
core,
|
|
37132
|
+
indexName,
|
|
37133
|
+
() => executeQuery(resolvedParams, context)
|
|
37134
|
+
);
|
|
37135
|
+
if (analysisStale) {
|
|
37136
|
+
result.analysisStale = true;
|
|
37137
|
+
}
|
|
37006
37138
|
if (core.pluginRegistry.hasHooks("afterSearch")) {
|
|
37007
37139
|
const observed = { ...result, hits: [...result.hits] };
|
|
37008
37140
|
try {
|
|
@@ -37011,9 +37143,6 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
37011
37143
|
console.warn("afterSearch plugin hook error:", err);
|
|
37012
37144
|
}
|
|
37013
37145
|
}
|
|
37014
|
-
if (core.analysisRebuild.isStale(indexName)) {
|
|
37015
|
-
result.analysisStale = true;
|
|
37016
|
-
}
|
|
37017
37146
|
return result;
|
|
37018
37147
|
} finally {
|
|
37019
37148
|
release();
|
|
@@ -37034,7 +37163,7 @@ async function runEnginePreflight(core, indexName, params, options) {
|
|
|
37034
37163
|
entry.embeddingAdapterName
|
|
37035
37164
|
);
|
|
37036
37165
|
const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
|
|
37037
|
-
const
|
|
37166
|
+
const context = {
|
|
37038
37167
|
manager,
|
|
37039
37168
|
language: entry.language,
|
|
37040
37169
|
config: entry.config,
|
|
@@ -37044,8 +37173,13 @@ async function runEnginePreflight(core, indexName, params, options) {
|
|
|
37044
37173
|
partitionIds: options?.partitionIds,
|
|
37045
37174
|
cursorBinding: queryBindingOf(params),
|
|
37046
37175
|
patternWorkCap: core.patternWorkCap
|
|
37047
|
-
}
|
|
37048
|
-
|
|
37176
|
+
};
|
|
37177
|
+
const { result, analysisStale } = await readUnderOneAnalysis(
|
|
37178
|
+
core,
|
|
37179
|
+
indexName,
|
|
37180
|
+
() => executePreflight(resolvedParams, context)
|
|
37181
|
+
);
|
|
37182
|
+
if (analysisStale) {
|
|
37049
37183
|
result.analysisStale = true;
|
|
37050
37184
|
}
|
|
37051
37185
|
return result;
|