@delali/narsil 0.3.1-next.83 → 0.3.1-next.84

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3394,65 +3394,6 @@ function mergeSegmentState(target, source) {
3394
3394
  target.scoreBuffer = null;
3395
3395
  }
3396
3396
 
3397
- // src/core/partition/rebuild.ts
3398
- var EMPTY_STATS = {
3399
- totalDocuments: 0,
3400
- totalFieldLengths: {},
3401
- averageFieldLengths: {},
3402
- docFrequencies: {}
3403
- };
3404
- function textFieldsOf(fields) {
3405
- return fields.filter((field) => isWordIndexedBase(field.base));
3406
- }
3407
- function rebuildTextIndex(state, schema, language, options) {
3408
- const textFields = textFieldsOf(schemaFieldsOf(schema));
3409
- state.invertedIdx.clear();
3410
- state.surfaceRegistry.clear();
3411
- state.stats.deserialize(EMPTY_STATS);
3412
- state.invertedIdx.beginBatch();
3413
- try {
3414
- for (const [docId, stored] of state.docStore.all()) {
3415
- const internalId = state.docStore.getInternalId(docId);
3416
- if (internalId === void 0) continue;
3417
- const fields = stored.fields;
3418
- const fieldLengths = {};
3419
- const tokensByField = {};
3420
- for (const { path: fieldPath, base } of textFields) {
3421
- const value = getNestedValue(fields, fieldPath);
3422
- if (value === void 0 || value === null) continue;
3423
- if (base === "string") {
3424
- indexStringField(
3425
- state,
3426
- internalId,
3427
- fieldPath,
3428
- value,
3429
- language,
3430
- options,
3431
- fieldLengths,
3432
- tokensByField
3433
- );
3434
- } else {
3435
- indexStringArrayField(
3436
- state,
3437
- internalId,
3438
- fieldPath,
3439
- value,
3440
- language,
3441
- options,
3442
- fieldLengths,
3443
- tokensByField
3444
- );
3445
- }
3446
- }
3447
- state.docStore.storeRef(docId, stored.fields, fieldLengths);
3448
- state.stats.addDocument(fieldLengths, tokensByField);
3449
- }
3450
- } finally {
3451
- state.invertedIdx.endBatch();
3452
- state.partialWords.forgetSpellings();
3453
- }
3454
- }
3455
-
3456
3397
  // src/core/partition/block-bounds.ts
3457
3398
  var ABSENT_FIELD_LENGTH_BOUND = 0;
3458
3399
  var boundsByList = /* @__PURE__ */ new WeakMap();
@@ -4873,9 +4814,6 @@ function createPartitionIndex(partitionId, trackPositions = true) {
4873
4814
  if (internalId !== void 0) recordSortValues(state, internalId, document);
4874
4815
  }
4875
4816
  },
4876
- rebuildTextIndex(schema, language, options) {
4877
- rebuildTextIndex(state, schema, language, options);
4878
- },
4879
4817
  get(docId, projection) {
4880
4818
  const stored = state.docStore.get(docId);
4881
4819
  if (!stored) return void 0;
@@ -5442,18 +5380,6 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
5442
5380
  if (frozen.length === 0) return live.encodeSegment();
5443
5381
  return composedScratch().encodeSegment();
5444
5382
  },
5445
- rebuildTextIndex(schema, language, options) {
5446
- invalidateDocFrequencies();
5447
- if (frozen.length > 0) {
5448
- for (const segment of frozen) {
5449
- const payload = encodeSegmentState(segment);
5450
- if (payload.documentCount === 0) continue;
5451
- live.mergeSegmentPayload(payload, survivorDocumentsOf(segment, payload));
5452
- }
5453
- frozen.length = 0;
5454
- }
5455
- live.rebuildTextIndex(schema, language, options);
5456
- },
5457
5383
  serialize(indexName, totalPartitions, language, schema) {
5458
5384
  if (frozen.length === 0) return live.serialize(indexName, totalPartitions, language, schema);
5459
5385
  return composedScratch().serialize(indexName, totalPartitions, language, schema);
@@ -5678,6 +5604,81 @@ function createCapacityChecks(source) {
5678
5604
  };
5679
5605
  }
5680
5606
 
5607
+ // src/partitioning/manager/stats.ts
5608
+ function aggregateStatsOf(partitions) {
5609
+ let totalDocuments = 0;
5610
+ const docFrequencies = /* @__PURE__ */ Object.create(null);
5611
+ const totalFieldLengths = /* @__PURE__ */ Object.create(null);
5612
+ for (let i = 0; i < partitions.length; i++) {
5613
+ const stats = partitions[i].stats;
5614
+ totalDocuments += stats.totalDocuments;
5615
+ for (const [term, freq] of Object.entries(stats.docFrequencies)) {
5616
+ docFrequencies[term] = (docFrequencies[term] ?? 0) + freq;
5617
+ }
5618
+ for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
5619
+ totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
5620
+ }
5621
+ }
5622
+ return { totalDocuments, docFrequencies, totalFieldLengths };
5623
+ }
5624
+ function estimateBytesOf(partitions, vecIndexes) {
5625
+ let total = 0;
5626
+ for (let i = 0; i < partitions.length; i++) {
5627
+ total += partitions[i].estimateMemoryBytes();
5628
+ }
5629
+ for (const [, vecIndex] of vecIndexes) {
5630
+ total += vecIndex.estimateMemoryBytes();
5631
+ }
5632
+ return total;
5633
+ }
5634
+ function partitionStatsOf(partitions) {
5635
+ return partitions.map((p, i) => ({
5636
+ partitionId: i,
5637
+ documentCount: p.count(),
5638
+ estimatedMemoryBytes: p.estimateMemoryBytes()
5639
+ }));
5640
+ }
5641
+
5642
+ // src/partitioning/manager/write-log.ts
5643
+ function createWriteLogRecorder() {
5644
+ let current = null;
5645
+ return {
5646
+ open() {
5647
+ const log = { docIds: /* @__PURE__ */ new Set(), replaced: false };
5648
+ current = log;
5649
+ return {
5650
+ get partitionsReplaced() {
5651
+ return log.replaced;
5652
+ },
5653
+ get pendingDocuments() {
5654
+ return log.docIds.size;
5655
+ },
5656
+ takeWrittenDocIds() {
5657
+ const taken = log.docIds;
5658
+ log.docIds = /* @__PURE__ */ new Set();
5659
+ return taken;
5660
+ },
5661
+ close() {
5662
+ if (current === log) current = null;
5663
+ }
5664
+ };
5665
+ },
5666
+ documentWritten(docId) {
5667
+ if (current !== null) current.docIds.add(docId);
5668
+ },
5669
+ segmentWritten(segment) {
5670
+ if (current === null) return;
5671
+ for (const internalId of segment.docStore.allInternalIds()) {
5672
+ const docId = segment.docStore.getExternalId(internalId);
5673
+ if (docId !== void 0) current.docIds.add(docId);
5674
+ }
5675
+ },
5676
+ partitionsReplaced() {
5677
+ if (current !== null) current.replaced = true;
5678
+ }
5679
+ };
5680
+ }
5681
+
5681
5682
  // src/partitioning/manager/index.ts
5682
5683
  function createPartitionManager(indexName, config, language, router, initialPartitionCount, vectorIndexes) {
5683
5684
  requirePatternSearchFor(config.schema);
@@ -5689,6 +5690,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
5689
5690
  const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
5690
5691
  let partitions = [];
5691
5692
  const docPartitionMap = /* @__PURE__ */ new Map();
5693
+ const writeLog = createWriteLogRecorder();
5692
5694
  for (let i = 0; i < count; i++) {
5693
5695
  partitions.push(createPartitionIndex(i, trackPositions));
5694
5696
  }
@@ -5776,9 +5778,23 @@ function createPartitionManager(indexName, config, language, router, initialPart
5776
5778
  return [...partitions];
5777
5779
  },
5778
5780
  setPartitions(newPartitions) {
5781
+ writeLog.partitionsReplaced();
5779
5782
  partitions = [...newPartitions];
5780
5783
  rebuildDocPartitionMap();
5781
5784
  },
5785
+ openWriteLog() {
5786
+ return writeLog.open();
5787
+ },
5788
+ adoptRebuiltPartitions(rebuilt) {
5789
+ if (rebuilt.length !== partitions.length) {
5790
+ throw new NarsilError(
5791
+ ErrorCodes.PARTITION_CORRUPTED,
5792
+ `Index "${indexName}" holds ${partitions.length} partitions, but the rebuild produced ${rebuilt.length}`,
5793
+ { indexName, partitionCount: partitions.length, rebuiltCount: rebuilt.length }
5794
+ );
5795
+ }
5796
+ partitions = [...rebuilt];
5797
+ },
5782
5798
  addPartition() {
5783
5799
  const currentMaxPartitions = config.partitions?.maxPartitions;
5784
5800
  if (currentMaxPartitions !== void 0 && partitions.length >= currentMaxPartitions) {
@@ -5789,11 +5805,13 @@ function createPartitionManager(indexName, config, language, router, initialPart
5789
5805
  );
5790
5806
  }
5791
5807
  const partition = createPartitionIndex(partitions.length, trackPositions);
5808
+ writeLog.partitionsReplaced();
5792
5809
  partitions.push(partition);
5793
5810
  return partition;
5794
5811
  },
5795
5812
  removePartition(partitionId) {
5796
5813
  validatePartitionId(partitionId);
5814
+ writeLog.partitionsReplaced();
5797
5815
  partitions.splice(partitionId, 1);
5798
5816
  rebuildDocPartitionMap();
5799
5817
  },
@@ -5807,6 +5825,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
5807
5825
  if (newCount >= partitions.length) {
5808
5826
  return;
5809
5827
  }
5828
+ writeLog.partitionsReplaced();
5810
5829
  for (const [docId, pid] of docPartitionMap) {
5811
5830
  if (pid >= newCount) {
5812
5831
  docPartitionMap.delete(docId);
@@ -5829,6 +5848,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
5829
5848
  const insertOpts = resolveInsertOptions(options);
5830
5849
  partitions[pid].insert(docId, document, config.schema, language, insertOpts);
5831
5850
  docPartitionMap.set(docId, pid);
5851
+ writeLog.documentWritten(docId);
5832
5852
  },
5833
5853
  remove(docId) {
5834
5854
  const pid = locateDocument(docId);
@@ -5837,11 +5857,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
5837
5857
  }
5838
5858
  partitions[pid].remove(docId, config.schema, language, resolveInsertOptions());
5839
5859
  docPartitionMap.delete(docId);
5840
- },
5841
- rebuildTextIndex(partitionId) {
5842
- validatePartitionId(partitionId);
5843
- partitions[partitionId].rebuildTextIndex(config.schema, language, resolveInsertOptions());
5844
- preparePartialWords(partitionId);
5860
+ writeLog.documentWritten(docId);
5845
5861
  },
5846
5862
  beginBatchRemove() {
5847
5863
  for (let i = 0; i < partitions.length; i++) {
@@ -5861,6 +5877,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
5861
5877
  const updateOpts = resolveInsertOptions(options);
5862
5878
  partitions[pid].update(docId, document, config.schema, language, updateOpts);
5863
5879
  docPartitionMap.set(docId, pid);
5880
+ writeLog.documentWritten(docId);
5864
5881
  },
5865
5882
  get(docId, projection) {
5866
5883
  const pid = locateDocument(docId);
@@ -5911,6 +5928,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
5911
5928
  partition.endBatch();
5912
5929
  for (const docId of payload.docIds) {
5913
5930
  docPartitionMap.set(docId, partitionId);
5931
+ writeLog.documentWritten(docId);
5914
5932
  }
5915
5933
  },
5916
5934
  attachFrozenSegment(partitionId, segment) {
@@ -5925,6 +5943,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
5925
5943
  }
5926
5944
  asCompositePartition(partitionId).attachFrozenSegment(segment);
5927
5945
  registerSegmentDocuments(partitionId, segment);
5946
+ writeLog.segmentWritten(segment);
5928
5947
  },
5929
5948
  swapFrozenSegments(partitionId, dropSegmentIds, replacement) {
5930
5949
  validatePartitionId(partitionId);
@@ -5942,42 +5961,19 @@ function createPartitionManager(indexName, config, language, router, initialPart
5942
5961
  },
5943
5962
  deserializePartition(partitionId, data) {
5944
5963
  validatePartitionId(partitionId);
5964
+ writeLog.partitionsReplaced();
5945
5965
  partitions[partitionId].deserialize(data, config.schema);
5946
5966
  preparePartialWords(partitionId);
5947
5967
  rebuildDocPartitionMapForPartition(partitionId);
5948
5968
  },
5949
5969
  getAggregateStats() {
5950
- let totalDocuments = 0;
5951
- const docFrequencies = /* @__PURE__ */ Object.create(null);
5952
- const totalFieldLengths = /* @__PURE__ */ Object.create(null);
5953
- for (let i = 0; i < partitions.length; i++) {
5954
- const stats = partitions[i].stats;
5955
- totalDocuments += stats.totalDocuments;
5956
- for (const [term, freq] of Object.entries(stats.docFrequencies)) {
5957
- docFrequencies[term] = (docFrequencies[term] ?? 0) + freq;
5958
- }
5959
- for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
5960
- totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
5961
- }
5962
- }
5963
- return { totalDocuments, docFrequencies, totalFieldLengths };
5970
+ return aggregateStatsOf(partitions);
5964
5971
  },
5965
5972
  estimateMemoryBytes() {
5966
- let total = 0;
5967
- for (let i = 0; i < partitions.length; i++) {
5968
- total += partitions[i].estimateMemoryBytes();
5969
- }
5970
- for (const [, vecIndex] of vecIndexes) {
5971
- total += vecIndex.estimateMemoryBytes();
5972
- }
5973
- return total;
5973
+ return estimateBytesOf(partitions, vecIndexes);
5974
5974
  },
5975
5975
  getPartitionStats() {
5976
- return partitions.map((p, i) => ({
5977
- partitionId: i,
5978
- documentCount: p.count(),
5979
- estimatedMemoryBytes: p.estimateMemoryBytes()
5980
- }));
5976
+ return partitionStatsOf(partitions);
5981
5977
  },
5982
5978
  getVectorIndexes() {
5983
5979
  return vecIndexes;