@delali/narsil 0.3.1-next.83 → 0.3.1-next.84
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-CUP2E4OH.mjs → chunk-IA67SOJO.mjs} +102 -106
- package/dist/{chunk-7JZZ3HNV.mjs → chunk-XIQZKKTK.mjs} +177 -39
- package/dist/{direct-executor-6HASZ36Y.mjs → direct-executor-6WWFY5EI.mjs} +1 -1
- package/dist/distribution.mjs +19 -4
- package/dist/index.browser.mjs +277 -143
- package/dist/index.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +2 -2
- package/dist/workers/entry.mjs +1 -1
- package/package.json +7 -7
|
@@ -3394,65 +3394,6 @@ function mergeSegmentState(target, source) {
|
|
|
3394
3394
|
target.scoreBuffer = null;
|
|
3395
3395
|
}
|
|
3396
3396
|
|
|
3397
|
-
// src/core/partition/rebuild.ts
|
|
3398
|
-
var EMPTY_STATS = {
|
|
3399
|
-
totalDocuments: 0,
|
|
3400
|
-
totalFieldLengths: {},
|
|
3401
|
-
averageFieldLengths: {},
|
|
3402
|
-
docFrequencies: {}
|
|
3403
|
-
};
|
|
3404
|
-
function textFieldsOf(fields) {
|
|
3405
|
-
return fields.filter((field) => isWordIndexedBase(field.base));
|
|
3406
|
-
}
|
|
3407
|
-
function rebuildTextIndex(state, schema, language, options) {
|
|
3408
|
-
const textFields = textFieldsOf(schemaFieldsOf(schema));
|
|
3409
|
-
state.invertedIdx.clear();
|
|
3410
|
-
state.surfaceRegistry.clear();
|
|
3411
|
-
state.stats.deserialize(EMPTY_STATS);
|
|
3412
|
-
state.invertedIdx.beginBatch();
|
|
3413
|
-
try {
|
|
3414
|
-
for (const [docId, stored] of state.docStore.all()) {
|
|
3415
|
-
const internalId = state.docStore.getInternalId(docId);
|
|
3416
|
-
if (internalId === void 0) continue;
|
|
3417
|
-
const fields = stored.fields;
|
|
3418
|
-
const fieldLengths = {};
|
|
3419
|
-
const tokensByField = {};
|
|
3420
|
-
for (const { path: fieldPath, base } of textFields) {
|
|
3421
|
-
const value = getNestedValue(fields, fieldPath);
|
|
3422
|
-
if (value === void 0 || value === null) continue;
|
|
3423
|
-
if (base === "string") {
|
|
3424
|
-
indexStringField(
|
|
3425
|
-
state,
|
|
3426
|
-
internalId,
|
|
3427
|
-
fieldPath,
|
|
3428
|
-
value,
|
|
3429
|
-
language,
|
|
3430
|
-
options,
|
|
3431
|
-
fieldLengths,
|
|
3432
|
-
tokensByField
|
|
3433
|
-
);
|
|
3434
|
-
} else {
|
|
3435
|
-
indexStringArrayField(
|
|
3436
|
-
state,
|
|
3437
|
-
internalId,
|
|
3438
|
-
fieldPath,
|
|
3439
|
-
value,
|
|
3440
|
-
language,
|
|
3441
|
-
options,
|
|
3442
|
-
fieldLengths,
|
|
3443
|
-
tokensByField
|
|
3444
|
-
);
|
|
3445
|
-
}
|
|
3446
|
-
}
|
|
3447
|
-
state.docStore.storeRef(docId, stored.fields, fieldLengths);
|
|
3448
|
-
state.stats.addDocument(fieldLengths, tokensByField);
|
|
3449
|
-
}
|
|
3450
|
-
} finally {
|
|
3451
|
-
state.invertedIdx.endBatch();
|
|
3452
|
-
state.partialWords.forgetSpellings();
|
|
3453
|
-
}
|
|
3454
|
-
}
|
|
3455
|
-
|
|
3456
3397
|
// src/core/partition/block-bounds.ts
|
|
3457
3398
|
var ABSENT_FIELD_LENGTH_BOUND = 0;
|
|
3458
3399
|
var boundsByList = /* @__PURE__ */ new WeakMap();
|
|
@@ -4873,9 +4814,6 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
4873
4814
|
if (internalId !== void 0) recordSortValues(state, internalId, document);
|
|
4874
4815
|
}
|
|
4875
4816
|
},
|
|
4876
|
-
rebuildTextIndex(schema, language, options) {
|
|
4877
|
-
rebuildTextIndex(state, schema, language, options);
|
|
4878
|
-
},
|
|
4879
4817
|
get(docId, projection) {
|
|
4880
4818
|
const stored = state.docStore.get(docId);
|
|
4881
4819
|
if (!stored) return void 0;
|
|
@@ -5442,18 +5380,6 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
5442
5380
|
if (frozen.length === 0) return live.encodeSegment();
|
|
5443
5381
|
return composedScratch().encodeSegment();
|
|
5444
5382
|
},
|
|
5445
|
-
rebuildTextIndex(schema, language, options) {
|
|
5446
|
-
invalidateDocFrequencies();
|
|
5447
|
-
if (frozen.length > 0) {
|
|
5448
|
-
for (const segment of frozen) {
|
|
5449
|
-
const payload = encodeSegmentState(segment);
|
|
5450
|
-
if (payload.documentCount === 0) continue;
|
|
5451
|
-
live.mergeSegmentPayload(payload, survivorDocumentsOf(segment, payload));
|
|
5452
|
-
}
|
|
5453
|
-
frozen.length = 0;
|
|
5454
|
-
}
|
|
5455
|
-
live.rebuildTextIndex(schema, language, options);
|
|
5456
|
-
},
|
|
5457
5383
|
serialize(indexName, totalPartitions, language, schema) {
|
|
5458
5384
|
if (frozen.length === 0) return live.serialize(indexName, totalPartitions, language, schema);
|
|
5459
5385
|
return composedScratch().serialize(indexName, totalPartitions, language, schema);
|
|
@@ -5678,6 +5604,81 @@ function createCapacityChecks(source) {
|
|
|
5678
5604
|
};
|
|
5679
5605
|
}
|
|
5680
5606
|
|
|
5607
|
+
// src/partitioning/manager/stats.ts
|
|
5608
|
+
function aggregateStatsOf(partitions) {
|
|
5609
|
+
let totalDocuments = 0;
|
|
5610
|
+
const docFrequencies = /* @__PURE__ */ Object.create(null);
|
|
5611
|
+
const totalFieldLengths = /* @__PURE__ */ Object.create(null);
|
|
5612
|
+
for (let i = 0; i < partitions.length; i++) {
|
|
5613
|
+
const stats = partitions[i].stats;
|
|
5614
|
+
totalDocuments += stats.totalDocuments;
|
|
5615
|
+
for (const [term, freq] of Object.entries(stats.docFrequencies)) {
|
|
5616
|
+
docFrequencies[term] = (docFrequencies[term] ?? 0) + freq;
|
|
5617
|
+
}
|
|
5618
|
+
for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
|
|
5619
|
+
totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
|
|
5620
|
+
}
|
|
5621
|
+
}
|
|
5622
|
+
return { totalDocuments, docFrequencies, totalFieldLengths };
|
|
5623
|
+
}
|
|
5624
|
+
function estimateBytesOf(partitions, vecIndexes) {
|
|
5625
|
+
let total = 0;
|
|
5626
|
+
for (let i = 0; i < partitions.length; i++) {
|
|
5627
|
+
total += partitions[i].estimateMemoryBytes();
|
|
5628
|
+
}
|
|
5629
|
+
for (const [, vecIndex] of vecIndexes) {
|
|
5630
|
+
total += vecIndex.estimateMemoryBytes();
|
|
5631
|
+
}
|
|
5632
|
+
return total;
|
|
5633
|
+
}
|
|
5634
|
+
function partitionStatsOf(partitions) {
|
|
5635
|
+
return partitions.map((p, i) => ({
|
|
5636
|
+
partitionId: i,
|
|
5637
|
+
documentCount: p.count(),
|
|
5638
|
+
estimatedMemoryBytes: p.estimateMemoryBytes()
|
|
5639
|
+
}));
|
|
5640
|
+
}
|
|
5641
|
+
|
|
5642
|
+
// src/partitioning/manager/write-log.ts
|
|
5643
|
+
function createWriteLogRecorder() {
|
|
5644
|
+
let current = null;
|
|
5645
|
+
return {
|
|
5646
|
+
open() {
|
|
5647
|
+
const log = { docIds: /* @__PURE__ */ new Set(), replaced: false };
|
|
5648
|
+
current = log;
|
|
5649
|
+
return {
|
|
5650
|
+
get partitionsReplaced() {
|
|
5651
|
+
return log.replaced;
|
|
5652
|
+
},
|
|
5653
|
+
get pendingDocuments() {
|
|
5654
|
+
return log.docIds.size;
|
|
5655
|
+
},
|
|
5656
|
+
takeWrittenDocIds() {
|
|
5657
|
+
const taken = log.docIds;
|
|
5658
|
+
log.docIds = /* @__PURE__ */ new Set();
|
|
5659
|
+
return taken;
|
|
5660
|
+
},
|
|
5661
|
+
close() {
|
|
5662
|
+
if (current === log) current = null;
|
|
5663
|
+
}
|
|
5664
|
+
};
|
|
5665
|
+
},
|
|
5666
|
+
documentWritten(docId) {
|
|
5667
|
+
if (current !== null) current.docIds.add(docId);
|
|
5668
|
+
},
|
|
5669
|
+
segmentWritten(segment) {
|
|
5670
|
+
if (current === null) return;
|
|
5671
|
+
for (const internalId of segment.docStore.allInternalIds()) {
|
|
5672
|
+
const docId = segment.docStore.getExternalId(internalId);
|
|
5673
|
+
if (docId !== void 0) current.docIds.add(docId);
|
|
5674
|
+
}
|
|
5675
|
+
},
|
|
5676
|
+
partitionsReplaced() {
|
|
5677
|
+
if (current !== null) current.replaced = true;
|
|
5678
|
+
}
|
|
5679
|
+
};
|
|
5680
|
+
}
|
|
5681
|
+
|
|
5681
5682
|
// src/partitioning/manager/index.ts
|
|
5682
5683
|
function createPartitionManager(indexName, config, language, router, initialPartitionCount, vectorIndexes) {
|
|
5683
5684
|
requirePatternSearchFor(config.schema);
|
|
@@ -5689,6 +5690,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
5689
5690
|
const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
|
|
5690
5691
|
let partitions = [];
|
|
5691
5692
|
const docPartitionMap = /* @__PURE__ */ new Map();
|
|
5693
|
+
const writeLog = createWriteLogRecorder();
|
|
5692
5694
|
for (let i = 0; i < count; i++) {
|
|
5693
5695
|
partitions.push(createPartitionIndex(i, trackPositions));
|
|
5694
5696
|
}
|
|
@@ -5776,9 +5778,23 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
5776
5778
|
return [...partitions];
|
|
5777
5779
|
},
|
|
5778
5780
|
setPartitions(newPartitions) {
|
|
5781
|
+
writeLog.partitionsReplaced();
|
|
5779
5782
|
partitions = [...newPartitions];
|
|
5780
5783
|
rebuildDocPartitionMap();
|
|
5781
5784
|
},
|
|
5785
|
+
openWriteLog() {
|
|
5786
|
+
return writeLog.open();
|
|
5787
|
+
},
|
|
5788
|
+
adoptRebuiltPartitions(rebuilt) {
|
|
5789
|
+
if (rebuilt.length !== partitions.length) {
|
|
5790
|
+
throw new NarsilError(
|
|
5791
|
+
ErrorCodes.PARTITION_CORRUPTED,
|
|
5792
|
+
`Index "${indexName}" holds ${partitions.length} partitions, but the rebuild produced ${rebuilt.length}`,
|
|
5793
|
+
{ indexName, partitionCount: partitions.length, rebuiltCount: rebuilt.length }
|
|
5794
|
+
);
|
|
5795
|
+
}
|
|
5796
|
+
partitions = [...rebuilt];
|
|
5797
|
+
},
|
|
5782
5798
|
addPartition() {
|
|
5783
5799
|
const currentMaxPartitions = config.partitions?.maxPartitions;
|
|
5784
5800
|
if (currentMaxPartitions !== void 0 && partitions.length >= currentMaxPartitions) {
|
|
@@ -5789,11 +5805,13 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
5789
5805
|
);
|
|
5790
5806
|
}
|
|
5791
5807
|
const partition = createPartitionIndex(partitions.length, trackPositions);
|
|
5808
|
+
writeLog.partitionsReplaced();
|
|
5792
5809
|
partitions.push(partition);
|
|
5793
5810
|
return partition;
|
|
5794
5811
|
},
|
|
5795
5812
|
removePartition(partitionId) {
|
|
5796
5813
|
validatePartitionId(partitionId);
|
|
5814
|
+
writeLog.partitionsReplaced();
|
|
5797
5815
|
partitions.splice(partitionId, 1);
|
|
5798
5816
|
rebuildDocPartitionMap();
|
|
5799
5817
|
},
|
|
@@ -5807,6 +5825,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
5807
5825
|
if (newCount >= partitions.length) {
|
|
5808
5826
|
return;
|
|
5809
5827
|
}
|
|
5828
|
+
writeLog.partitionsReplaced();
|
|
5810
5829
|
for (const [docId, pid] of docPartitionMap) {
|
|
5811
5830
|
if (pid >= newCount) {
|
|
5812
5831
|
docPartitionMap.delete(docId);
|
|
@@ -5829,6 +5848,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
5829
5848
|
const insertOpts = resolveInsertOptions(options);
|
|
5830
5849
|
partitions[pid].insert(docId, document, config.schema, language, insertOpts);
|
|
5831
5850
|
docPartitionMap.set(docId, pid);
|
|
5851
|
+
writeLog.documentWritten(docId);
|
|
5832
5852
|
},
|
|
5833
5853
|
remove(docId) {
|
|
5834
5854
|
const pid = locateDocument(docId);
|
|
@@ -5837,11 +5857,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
5837
5857
|
}
|
|
5838
5858
|
partitions[pid].remove(docId, config.schema, language, resolveInsertOptions());
|
|
5839
5859
|
docPartitionMap.delete(docId);
|
|
5840
|
-
|
|
5841
|
-
rebuildTextIndex(partitionId) {
|
|
5842
|
-
validatePartitionId(partitionId);
|
|
5843
|
-
partitions[partitionId].rebuildTextIndex(config.schema, language, resolveInsertOptions());
|
|
5844
|
-
preparePartialWords(partitionId);
|
|
5860
|
+
writeLog.documentWritten(docId);
|
|
5845
5861
|
},
|
|
5846
5862
|
beginBatchRemove() {
|
|
5847
5863
|
for (let i = 0; i < partitions.length; i++) {
|
|
@@ -5861,6 +5877,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
5861
5877
|
const updateOpts = resolveInsertOptions(options);
|
|
5862
5878
|
partitions[pid].update(docId, document, config.schema, language, updateOpts);
|
|
5863
5879
|
docPartitionMap.set(docId, pid);
|
|
5880
|
+
writeLog.documentWritten(docId);
|
|
5864
5881
|
},
|
|
5865
5882
|
get(docId, projection) {
|
|
5866
5883
|
const pid = locateDocument(docId);
|
|
@@ -5911,6 +5928,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
5911
5928
|
partition.endBatch();
|
|
5912
5929
|
for (const docId of payload.docIds) {
|
|
5913
5930
|
docPartitionMap.set(docId, partitionId);
|
|
5931
|
+
writeLog.documentWritten(docId);
|
|
5914
5932
|
}
|
|
5915
5933
|
},
|
|
5916
5934
|
attachFrozenSegment(partitionId, segment) {
|
|
@@ -5925,6 +5943,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
5925
5943
|
}
|
|
5926
5944
|
asCompositePartition(partitionId).attachFrozenSegment(segment);
|
|
5927
5945
|
registerSegmentDocuments(partitionId, segment);
|
|
5946
|
+
writeLog.segmentWritten(segment);
|
|
5928
5947
|
},
|
|
5929
5948
|
swapFrozenSegments(partitionId, dropSegmentIds, replacement) {
|
|
5930
5949
|
validatePartitionId(partitionId);
|
|
@@ -5942,42 +5961,19 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
5942
5961
|
},
|
|
5943
5962
|
deserializePartition(partitionId, data) {
|
|
5944
5963
|
validatePartitionId(partitionId);
|
|
5964
|
+
writeLog.partitionsReplaced();
|
|
5945
5965
|
partitions[partitionId].deserialize(data, config.schema);
|
|
5946
5966
|
preparePartialWords(partitionId);
|
|
5947
5967
|
rebuildDocPartitionMapForPartition(partitionId);
|
|
5948
5968
|
},
|
|
5949
5969
|
getAggregateStats() {
|
|
5950
|
-
|
|
5951
|
-
const docFrequencies = /* @__PURE__ */ Object.create(null);
|
|
5952
|
-
const totalFieldLengths = /* @__PURE__ */ Object.create(null);
|
|
5953
|
-
for (let i = 0; i < partitions.length; i++) {
|
|
5954
|
-
const stats = partitions[i].stats;
|
|
5955
|
-
totalDocuments += stats.totalDocuments;
|
|
5956
|
-
for (const [term, freq] of Object.entries(stats.docFrequencies)) {
|
|
5957
|
-
docFrequencies[term] = (docFrequencies[term] ?? 0) + freq;
|
|
5958
|
-
}
|
|
5959
|
-
for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
|
|
5960
|
-
totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
|
|
5961
|
-
}
|
|
5962
|
-
}
|
|
5963
|
-
return { totalDocuments, docFrequencies, totalFieldLengths };
|
|
5970
|
+
return aggregateStatsOf(partitions);
|
|
5964
5971
|
},
|
|
5965
5972
|
estimateMemoryBytes() {
|
|
5966
|
-
|
|
5967
|
-
for (let i = 0; i < partitions.length; i++) {
|
|
5968
|
-
total += partitions[i].estimateMemoryBytes();
|
|
5969
|
-
}
|
|
5970
|
-
for (const [, vecIndex] of vecIndexes) {
|
|
5971
|
-
total += vecIndex.estimateMemoryBytes();
|
|
5972
|
-
}
|
|
5973
|
-
return total;
|
|
5973
|
+
return estimateBytesOf(partitions, vecIndexes);
|
|
5974
5974
|
},
|
|
5975
5975
|
getPartitionStats() {
|
|
5976
|
-
return partitions
|
|
5977
|
-
partitionId: i,
|
|
5978
|
-
documentCount: p.count(),
|
|
5979
|
-
estimatedMemoryBytes: p.estimateMemoryBytes()
|
|
5980
|
-
}));
|
|
5976
|
+
return partitionStatsOf(partitions);
|
|
5981
5977
|
},
|
|
5982
5978
|
getVectorIndexes() {
|
|
5983
5979
|
return vecIndexes;
|