@delali/narsil 0.3.1-next.87 → 0.3.1-next.89

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. package/dist/browser/{chunk-BBDAYUCO.mjs → chunk-3GZQKGVQ.mjs} +25 -9
  2. package/dist/browser/{chunk-3BXQPIXK.mjs → chunk-JACCR3BJ.mjs} +3 -3
  3. package/dist/browser/{chunk-TBMNPGFO.mjs → chunk-RAHUQOJL.mjs} +2074 -881
  4. package/dist/browser/{durability-wiring-7RUN7LNS.mjs → durability-wiring-VSUBLUDK.mjs} +3 -3
  5. package/dist/browser/{snapshot-A42DRERF.mjs → snapshot-5MXBEDXF.mjs} +3 -3
  6. package/dist/browser/{write-4U6Q26TJ.mjs → write-6SBYTS62.mjs} +2 -2
  7. package/dist/{chunk-PSUJCOSE.mjs → chunk-A3UB2MXL.mjs} +172 -122
  8. package/dist/{chunk-JFJBA3PD.mjs → chunk-CL4OM7GB.mjs} +3 -3
  9. package/dist/{chunk-AXDAS2VA.mjs → chunk-DCK4P5BI.mjs} +28 -30
  10. package/dist/{chunk-XBXKPWMR.mjs → chunk-EMXAYPTQ.mjs} +1735 -651
  11. package/dist/{chunk-IKST36MA.mjs → chunk-MWJIAREY.mjs} +4 -4
  12. package/dist/{chunk-T7DMUVTL.mjs → chunk-QAMUC2BJ.mjs} +27 -11
  13. package/dist/{chunk-AQU2QHSY.mjs → chunk-R6BMMQXH.mjs} +114 -45
  14. package/dist/{chunk-K4OWQ2M3.mjs → chunk-S4LQDSUL.mjs} +3 -3
  15. package/dist/{chunk-MKK4OZ5W.mjs → chunk-V7YUSKKZ.mjs} +3 -3
  16. package/dist/{chunk-WQSHDZ7Z.mjs → chunk-VY4DXZ5P.mjs} +1 -1
  17. package/dist/{chunk-SYRFSF6V.mjs → chunk-X4SEXL4U.mjs} +49 -11
  18. package/dist/{direct-executor-LD6JW57R.mjs → direct-executor-EGESD26S.mjs} +3 -3
  19. package/dist/distribution/transport/grpc.mjs +45 -7
  20. package/dist/distribution.mjs +14 -27
  21. package/dist/{durability-wiring-5S32KDDC.mjs → durability-wiring-ZLRQU6FG.mjs} +10 -10
  22. package/dist/index.browser.mjs +1 -1
  23. package/dist/index.mjs +5 -5
  24. package/dist/partial-words.mjs +1 -1
  25. package/dist/persistence/durability/checkpoint-worker.mjs +8 -8
  26. package/dist/react.mjs +4 -2
  27. package/dist/{snapshot-6BYSUPWL.mjs → snapshot-22CUEYYW.mjs} +10 -10
  28. package/dist/{thread-GS65E6RM.mjs → thread-ZP3UMIZT.mjs} +3 -3
  29. package/dist/workers/entry.mjs +3 -3
  30. package/dist/{write-HLEZTL2L.mjs → write-PXDKPG2R.mjs} +6 -6
  31. package/package.json +13 -8
@@ -1,5 +1,5 @@
1
- import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors } from './chunk-AQU2QHSY.mjs';
2
- import { getNestedValue } from './chunk-PSUJCOSE.mjs';
1
+ import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors } from './chunk-R6BMMQXH.mjs';
2
+ import { getNestedValue } from './chunk-A3UB2MXL.mjs';
3
3
  import { requireValidFieldTypes } from './chunk-CVQWQVFX.mjs';
4
4
  import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
5
5
  import { ErrorCodes, NarsilError } from './chunk-TPJU7NDJ.mjs';
@@ -40,12 +40,12 @@ function applyIndexEntry(entry, manager, vectorFieldPaths, vecIndexes) {
40
40
  }
41
41
  }
42
42
  }
43
- function applyDeleteEntry(entry, manager, vecIndexes) {
43
+ function applyDeleteEntry(entry, manager, vecIndexes, removedSpellings) {
44
44
  removeDocumentVectors(entry.documentId, vecIndexes);
45
45
  if (!manager.has(entry.documentId)) {
46
46
  return;
47
47
  }
48
- manager.remove(entry.documentId);
48
+ manager.remove(entry.documentId, removedSpellings);
49
49
  }
50
50
  function restoreVectorFields(document, vectorFieldPaths) {
51
51
  for (const fieldPath of vectorFieldPaths) {
@@ -1,9 +1,9 @@
1
- import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes } from './chunk-IKST36MA.mjs';
1
+ import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes } from './chunk-MWJIAREY.mjs';
2
2
  import { ReplicationMessageTypes } from './chunk-GCM3JFVA.mjs';
3
3
  import { WIRE_BATCH_BUDGET } from './chunk-P36MUFEJ.mjs';
4
- import { packChecksummedEnvelope, crc32, openChecksummedEnvelope, reconstructSchemaFromMetadata, HEADER_SIZE, prepareDocumentVectors } from './chunk-AQU2QHSY.mjs';
5
- import { isCompositePartition, generateId, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, resolvePartitionInsertOptions, resolveIndexAnalysis, createPartitionManager, createPartitionRouter, buildSegmentPayload, selectionKeeps, decodeMapWithoutFields, shareSegmentArrays, segmentArraysOf, mergeSegmentContents } from './chunk-XBXKPWMR.mjs';
6
- import { fnv1a } from './chunk-SYRFSF6V.mjs';
4
+ import { packChecksummedEnvelope, crc32, openChecksummedEnvelope, reconstructSchemaFromMetadata, HEADER_SIZE, prepareDocumentVectors } from './chunk-R6BMMQXH.mjs';
5
+ import { isCompositePartition, generateId, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, selectionKeeps, decodeMapWithoutFields, shareSegmentArrays, segmentArraysOf, mergeSegmentContents, resolveIndexAnalysis } from './chunk-EMXAYPTQ.mjs';
6
+ import { fnv1a } from './chunk-X4SEXL4U.mjs';
7
7
  import { getLanguage } from './chunk-PR4O7CAP.mjs';
8
8
  import { MAX_FILE_BYTES } from './chunk-RVM7AODE.mjs';
9
9
  import { extractVectorFieldsFromSchema, partialFieldsOf, messagePackUpperBoundBytes } from './chunk-CVQWQVFX.mjs';
@@ -1385,7 +1385,7 @@ function noSpellingsOf(analysis) {
1385
1385
  return { fields: [...analysis.fields], tokens: [], spellings: [], counts: [] };
1386
1386
  }
1387
1387
  function contentsOfGroup(group, request, tombstones) {
1388
- const { schema, partialWordAnalysis } = request;
1388
+ const { schema, partialWordAnalysis, textAnalysis } = request;
1389
1389
  if (group === null) {
1390
1390
  const payload = { ...emptyPayload(), partialSpellings: noSpellingsOf(partialWordAnalysis) };
1391
1391
  return {
@@ -1394,7 +1394,7 @@ function contentsOfGroup(group, request, tombstones) {
1394
1394
  arrays: segmentArraysOf(payload, { blob: new Uint8Array(0), offsets: new Uint32Array(1) })
1395
1395
  };
1396
1396
  }
1397
- const merged = mergeSegmentContents(group.selections, partialWordAnalysis);
1397
+ const merged = mergeSegmentContents(group.selections, partialWordAnalysis, textAnalysis);
1398
1398
  return {
1399
1399
  schema,
1400
1400
  tombstones,
@@ -1529,6 +1529,7 @@ async function compactPartitionSegments(input) {
1529
1529
  nextSegmentId: input.nextSegmentId,
1530
1530
  schema: input.schema,
1531
1531
  partialWordAnalysis: input.partialWordAnalysis,
1532
+ textAnalysis: input.textAnalysis,
1532
1533
  selections: frozen.map((segment) => ({ segment })),
1533
1534
  tombstones: retainedTombstones(frozen, contents),
1534
1535
  mergesIntoOneFile: true
@@ -1544,9 +1545,7 @@ async function compactPartitionSegments(input) {
1544
1545
  }
1545
1546
 
1546
1547
  // src/persistence/durability/segment/write.ts
1547
- function partialWordAnalysisOf(config) {
1548
- const fields = partialFieldsOf(config.schema);
1549
- if (fields.length === 0) return null;
1548
+ function textAnalysisOf(config) {
1550
1549
  let resolved = null;
1551
1550
  const resolve = () => {
1552
1551
  resolved ??= {
@@ -1556,7 +1555,6 @@ function partialWordAnalysisOf(config) {
1556
1555
  return resolved;
1557
1556
  };
1558
1557
  return {
1559
- fields,
1560
1558
  get language() {
1561
1559
  return resolve().language;
1562
1560
  },
@@ -1565,6 +1563,19 @@ function partialWordAnalysisOf(config) {
1565
1563
  }
1566
1564
  };
1567
1565
  }
1566
+ function partialWordAnalysisOf(config, text) {
1567
+ const fields = partialFieldsOf(config.schema);
1568
+ if (fields.length === 0) return null;
1569
+ return {
1570
+ fields,
1571
+ get language() {
1572
+ return text.language;
1573
+ },
1574
+ get options() {
1575
+ return text.options;
1576
+ }
1577
+ };
1578
+ }
1568
1579
  function resolveCompactionThreshold(configured) {
1569
1580
  if (!Number.isInteger(configured) || configured <= 0 || configured > MAX_SEGMENTS_PER_PARTITION) {
1570
1581
  return DEFAULT_COMPACTION_THRESHOLD;
@@ -1585,6 +1596,7 @@ async function writeWholePartition(context, partitionId, priorPartition, whole)
1585
1596
  nextSegmentId: priorPartition?.nextSegmentId ?? 0,
1586
1597
  schema: context.input.metadata.schema,
1587
1598
  partialWordAnalysis: context.partialWordAnalysis,
1599
+ textAnalysis: context.textAnalysis,
1588
1600
  selections: whole.selections,
1589
1601
  tombstones: []
1590
1602
  });
@@ -1608,6 +1620,7 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
1608
1620
  nextSegmentId,
1609
1621
  schema: context.input.metadata.schema,
1610
1622
  partialWordAnalysis: context.partialWordAnalysis,
1623
+ textAnalysis: context.textAnalysis,
1611
1624
  selections: built.selections,
1612
1625
  tombstones: built.tombstones
1613
1626
  });
@@ -1627,6 +1640,7 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
1627
1640
  partitionId,
1628
1641
  schema: context.input.metadata.schema,
1629
1642
  partialWordAnalysis: context.partialWordAnalysis,
1643
+ textAnalysis: context.textAnalysis,
1630
1644
  segments,
1631
1645
  nextSegmentId,
1632
1646
  compactionThreshold: context.compactionThreshold
@@ -1649,11 +1663,13 @@ function carryForwardUncheckpointedPartitions(prior, targets, partitions, checkp
1649
1663
  }
1650
1664
  async function writeCheckpointSegments(input) {
1651
1665
  const config = reconstructSchemaFromMetadata(input.metadata);
1666
+ const textAnalysis = textAnalysisOf(config);
1652
1667
  const context = {
1653
1668
  input,
1654
1669
  indexName: input.metadata.indexName,
1655
1670
  config,
1656
- partialWordAnalysis: partialWordAnalysisOf(config),
1671
+ partialWordAnalysis: partialWordAnalysisOf(config, textAnalysis),
1672
+ textAnalysis,
1657
1673
  vectorFieldPaths: new Set(extractVectorFieldsFromSchema(config.schema).keys()),
1658
1674
  compactionThreshold: resolveCompactionThreshold(input.compactionThreshold)
1659
1675
  };
@@ -1,6 +1,6 @@
1
- import { createDirectExecutor, generateId, createPartitionRouter, readHeapStatistics, createPartitionIndex, mergeFrozenSegments, createSharedFrozenSegment, isCompositePartition, freezeSegmentShared, resolvePartitionInsertOptions, encodeSegmentState, createFrozenSegment } from './chunk-XBXKPWMR.mjs';
2
- import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, queryBindingOf, executeQuery, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-AXDAS2VA.mjs';
3
- import { requirePatternSearchFor, requirePartialWordSearchFor, collectQueryTermStats, mergePartitionStats, oversampledFacetConfig, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, fnv1a } from './chunk-SYRFSF6V.mjs';
1
+ import { createDirectExecutor, generateId, createPartitionRouter, readHeapStatistics, createPartitionIndex, mergeFrozenSegments, createSharedFrozenSegment, isCompositePartition, freezeSegmentShared, resolvePartitionInsertOptions, encodeSegmentState, createFrozenSegment } from './chunk-EMXAYPTQ.mjs';
2
+ import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, queryBindingOf, executeQuery, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-DCK4P5BI.mjs';
3
+ import { requirePatternSearchFor, requirePartialWordSearchFor, collectQueryTermStats, mergePartitionStats, oversampledFacetConfig, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, fnv1a } from './chunk-X4SEXL4U.mjs';
4
4
  import { resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-J2BPXTD7.mjs';
5
5
  import { getLanguage } from './chunk-PR4O7CAP.mjs';
6
6
  import { splitWorkerBudget, resolveWorkerCount, NATIVE_CRC32_MIN_BYTES, resolveWorkerEntry, missingWorkerEntry, MAX_FILE_BYTES, resolveRequestThreadCount, CHECKSUM_YIELD_CHUNK_BYTES, workerResourceLimits, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH } from './chunk-RVM7AODE.mjs';
@@ -8,7 +8,7 @@ import { detectRuntime } from './chunk-AV4IJKD2.mjs';
8
8
  import { MAX_INDEX_CONFIGURATION_BYTES, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, FOREIGN_STATS_TTL_MS, MAX_IMPORT_REFUSALS_REPORTED, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE } from './chunk-Q5DC6WAB.mjs';
9
9
  import { requireSearchBackendSettings } from './chunk-QZN3MIJ3.mjs';
10
10
  import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-T6CNO72J.mjs';
11
- import { getNestedValue } from './chunk-PSUJCOSE.mjs';
11
+ import { getNestedValue } from './chunk-A3UB2MXL.mjs';
12
12
  import { flattenSchema, raiseEarlierReleaseGraphSetting, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, extractVectorFieldsFromSchema, schemaFieldsOf, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-CVQWQVFX.mjs';
13
13
  import { isSingleStringFieldType } from './chunk-BVOJIYRV.mjs';
14
14
  import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
@@ -1628,7 +1628,15 @@ function checkHeapAfterRecovery(core) {
1628
1628
  // src/core/partition/frozen/share.ts
1629
1629
  function shareFrozenSegment(segment, analysis) {
1630
1630
  if (segment.sharedSnapshot !== null) {
1631
- return { snapshot: segment.sharedSnapshot, tombstonedDocIds: segment.tombstonedDocIds() };
1631
+ const tombstonedDocIds = segment.tombstonedDocIds();
1632
+ if (tombstonedDocIds.length === 0 || !segment.removals.settled) {
1633
+ return { snapshot: segment.sharedSnapshot, tombstonedDocIds };
1634
+ }
1635
+ return {
1636
+ snapshot: segment.sharedSnapshot,
1637
+ tombstonedDocIds,
1638
+ survivingSurfaceForms: segment.surfaces.reader.serialize()
1639
+ };
1632
1640
  }
1633
1641
  const payload = encodeSegmentState(segment);
1634
1642
  const documents = payload.docIds.map((docId) => {
@@ -1646,6 +1654,35 @@ function shareFrozenSegment(segment, analysis) {
1646
1654
 
1647
1655
  // src/engine/worker-resync.ts
1648
1656
  var copyLoadSequence = 0;
1657
+ function emptySerializablePartition(manager, partitionId) {
1658
+ return {
1659
+ indexName: manager.indexName,
1660
+ partitionId,
1661
+ totalPartitions: manager.partitionCount,
1662
+ language: manager.language.name,
1663
+ schema: {},
1664
+ docCount: 0,
1665
+ avgDocLength: 0,
1666
+ documents: {},
1667
+ invertedIndex: {},
1668
+ fieldIndexes: { numeric: {}, boolean: {}, enum: {}, geopoint: {} },
1669
+ statistics: { totalDocuments: 0, totalFieldLengths: {}, averageFieldLengths: {}, docFrequencies: {} }
1670
+ };
1671
+ }
1672
+ function captureLiveTail(partition) {
1673
+ if (partition.count() === 0) return null;
1674
+ const payload = partition.encodeSegment();
1675
+ const documents = payload.docIds.map((docId) => {
1676
+ const document = partition.getRef(docId);
1677
+ if (document === void 0) {
1678
+ throw new NarsilError(ErrorCodes.PARTITION_CORRUPTED, `Document "${docId}" vanished while copying a partition`, {
1679
+ docId
1680
+ });
1681
+ }
1682
+ return document;
1683
+ });
1684
+ return { payload, documents };
1685
+ }
1649
1686
  function captureComposite(manager, partitionId) {
1650
1687
  const partition = manager.getPartition(partitionId);
1651
1688
  if (!isCompositePartition(partition)) return null;
@@ -1656,13 +1693,7 @@ function captureComposite(manager, partitionId) {
1656
1693
  if (shared === null) return null;
1657
1694
  frozen.push(shared);
1658
1695
  }
1659
- const live = partition.live.serialize(
1660
- manager.indexName,
1661
- manager.partitionCount,
1662
- manager.language.name,
1663
- manager.schema
1664
- );
1665
- return { partitionId, live, frozen };
1696
+ return { partitionId, live: captureLiveTail(partition.live), frozen };
1666
1697
  }
1667
1698
  function capturePartitions(manager) {
1668
1699
  const captured = [];
@@ -1670,30 +1701,45 @@ function capturePartitions(manager) {
1670
1701
  captured.push(
1671
1702
  captureComposite(manager, partitionId) ?? {
1672
1703
  partitionId,
1673
- live: manager.serializePartition(partitionId),
1704
+ live: captureLiveTail(manager.getPartition(partitionId)),
1674
1705
  frozen: []
1675
1706
  }
1676
1707
  );
1677
1708
  }
1678
1709
  return captured;
1679
1710
  }
1680
- async function sendPartition(indexName, executors, captured) {
1711
+ async function sendPartition(manager, executors, captured) {
1712
+ const { indexName } = manager;
1681
1713
  await Promise.all(
1682
1714
  executors.map(
1683
1715
  (workerExecutor) => workerExecutor.execute({
1684
1716
  type: "deserialize",
1685
1717
  indexName,
1686
1718
  partitionId: captured.partitionId,
1687
- data: captured.live,
1719
+ data: emptySerializablePartition(manager, captured.partitionId),
1688
1720
  requestId: `resync-${indexName}-${captured.partitionId}`
1689
1721
  })
1690
1722
  )
1691
1723
  );
1724
+ const live = captured.live;
1725
+ if (live !== null) {
1726
+ await Promise.all(
1727
+ executors.map(
1728
+ (workerExecutor) => workerExecutor.execute({
1729
+ type: "mergeSegments",
1730
+ indexName,
1731
+ segments: [{ partitionId: captured.partitionId, payload: live.payload, documents: live.documents }],
1732
+ requestId: `resync-live-${indexName}-${captured.partitionId}`
1733
+ })
1734
+ )
1735
+ );
1736
+ }
1692
1737
  if (captured.frozen.length === 0) return;
1693
1738
  const segments = captured.frozen.map((shared) => ({
1694
1739
  partitionId: captured.partitionId,
1695
1740
  snapshot: shared.snapshot,
1696
- tombstonedDocIds: shared.tombstonedDocIds
1741
+ tombstonedDocIds: shared.tombstonedDocIds,
1742
+ survivingSurfaceForms: shared.survivingSurfaceForms
1697
1743
  }));
1698
1744
  await Promise.all(
1699
1745
  executors.map(
@@ -1739,7 +1785,7 @@ async function loadIndexOntoWorkers(indexName, executors, config, manager, onDro
1739
1785
  )
1740
1786
  );
1741
1787
  for (const partition of captured) {
1742
- await sendPartition(indexName, executors, partition);
1788
+ await sendPartition(manager, executors, partition);
1743
1789
  }
1744
1790
  return copyLoad;
1745
1791
  }
@@ -2337,21 +2383,24 @@ async function compactPartitionSegments(state, indexName, manager, partitionId,
2337
2383
  const mainCanMerge = composite !== null && mainHoldsAll(composite, segmentIds);
2338
2384
  let snapshot = await compactOnWorker(state, indexName, partitionId, segmentIds);
2339
2385
  if (snapshot === null && mainCanMerge && composite !== null) {
2340
- snapshot = mergeFrozenSegments(composite.frozenSegmentsById(segmentIds), manager.partialWordAnalysis);
2386
+ snapshot = mergeFrozenSegments(
2387
+ composite.frozenSegmentsById(segmentIds),
2388
+ manager.partialWordAnalysis,
2389
+ manager.textAnalysis
2390
+ );
2341
2391
  }
2342
2392
  if (snapshot === null) {
2343
2393
  if (!mainCanMerge) forgetLedgerSegments(state, indexName, partitionId, segmentIds);
2344
2394
  return false;
2345
2395
  }
2346
- if (mainCanMerge && composite !== null) {
2347
- composite.swapFrozenSegments(segmentIds, createSharedFrozenSegment(snapshot));
2348
- }
2396
+ const removedSpellings = mainCanMerge && composite !== null ? composite.swapFrozenSegments(segmentIds, createSharedFrozenSegment(snapshot)) : void 0;
2349
2397
  await replicateToWorkers(state, {
2350
2398
  type: "swapSegments",
2351
2399
  indexName,
2352
2400
  partitionId,
2353
2401
  dropSegmentIds: [...segmentIds],
2354
2402
  snapshot,
2403
+ removedSpellings,
2355
2404
  requestId: createRequestId()
2356
2405
  });
2357
2406
  return true;
@@ -3871,7 +3920,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
3871
3920
  if (config?.lifecycle !== void 0 && durabilityTier === null) {
3872
3921
  throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
3873
3922
  }
3874
- const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-5S32KDDC.mjs');
3923
+ const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-ZLRQU6FG.mjs');
3875
3924
  const filesystemDurability = durabilityTier?.kind === "wal";
3876
3925
  const executor = createDirectExecutor({
3877
3926
  vectorWorkerCopies: vectorCopyPolicy,
@@ -5588,16 +5637,18 @@ async function prepareRemoval(ctx, indexName, docId) {
5588
5637
  if (ctx.durability && !rebalancing && !ctx.requireManager(indexName).has(docId)) {
5589
5638
  throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
5590
5639
  }
5591
- const restoreRef = ctx.durability && !rebalancing ? ctx.requireManager(indexName).getRef(docId) : void 0;
5640
+ const manager = ctx.requireManager(indexName);
5641
+ const restoreRef = ctx.durability && !rebalancing ? manager.getRef(docId) : void 0;
5592
5642
  return {
5593
5643
  docId,
5594
5644
  document: null,
5595
5645
  restoreDoc: restoreRef ? structuredClone(restoreRef) : void 0,
5646
+ removedSpellings: manager.removedSpellingsOf(docId),
5596
5647
  buffered: false
5597
5648
  };
5598
5649
  }
5599
5650
  function applyOfRemoval(ctx, indexName, prepared) {
5600
- const { docId } = prepared;
5651
+ const { docId, removedSpellings } = prepared;
5601
5652
  return async () => {
5602
5653
  if (ctx.isRebalancing(indexName)) {
5603
5654
  const bufferedState = ctx.bufferedDocState(indexName, docId);
@@ -5610,11 +5661,12 @@ function applyOfRemoval(ctx, indexName, prepared) {
5610
5661
  prepared.buffered = true;
5611
5662
  return;
5612
5663
  }
5613
- await ctx.executor.execute({ type: "remove", indexName, docId, requestId: docId });
5664
+ await ctx.executor.execute({ type: "remove", indexName, docId, requestId: docId, removedSpellings });
5614
5665
  removeDocumentVectors(docId, ctx.requireManager(indexName).getVectorIndexes());
5615
5666
  };
5616
5667
  }
5617
- async function settleRemoval(ctx, indexName, docId) {
5668
+ async function settleRemoval(ctx, indexName, prepared) {
5669
+ const { docId, removedSpellings } = prepared;
5618
5670
  try {
5619
5671
  await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId });
5620
5672
  } catch (err) {
@@ -5624,7 +5676,8 @@ async function settleRemoval(ctx, indexName, docId) {
5624
5676
  type: "remove",
5625
5677
  indexName,
5626
5678
  docId,
5627
- requestId: `replicate-remove-${docId}`
5679
+ requestId: `replicate-remove-${docId}`,
5680
+ removedSpellings
5628
5681
  });
5629
5682
  }
5630
5683
  async function removeDocument(ctx, indexName, docId, options) {
@@ -5642,7 +5695,7 @@ async function removeDocument(ctx, indexName, docId, options) {
5642
5695
  } else {
5643
5696
  await apply();
5644
5697
  }
5645
- if (!prepared.buffered) await settleRemoval(ctx, indexName, docId);
5698
+ if (!prepared.buffered) await settleRemoval(ctx, indexName, prepared);
5646
5699
  if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
5647
5700
  }
5648
5701
  async function removeDocumentBatch(ctx, indexName, docIds, options) {
@@ -5678,7 +5731,7 @@ async function removeDocumentBatch(ctx, indexName, docIds, options) {
5678
5731
  failed.push({ docId: removal.docId, error: asRemoveError(error) });
5679
5732
  continue;
5680
5733
  }
5681
- if (!removal.buffered) await settleRemoval(ctx, indexName, removal.docId);
5734
+ if (!removal.buffered) await settleRemoval(ctx, indexName, removal);
5682
5735
  succeeded.push(removal.docId);
5683
5736
  }
5684
5737
  if (chunkEnd < docIds.length) {
@@ -5726,7 +5779,8 @@ async function settleUpdate(ctx, indexName, prepared) {
5726
5779
  indexName,
5727
5780
  docId: prepared.docId,
5728
5781
  document: prepared.partitionDoc,
5729
- requestId: `replicate-update-${prepared.docId}`
5782
+ requestId: `replicate-update-${prepared.docId}`,
5783
+ removedSpellings: prepared.removedSpellings
5730
5784
  });
5731
5785
  const vecIndexes = ctx.requireManager(indexName).getVectorIndexes();
5732
5786
  for (const [fieldPath, vec] of prepared.extractedVectors) {
@@ -5793,12 +5847,13 @@ async function prepareUpdate(ctx, indexName, entry, docId, document) {
5793
5847
  oldDocument,
5794
5848
  rollbackDoc,
5795
5849
  extractedVectors,
5850
+ removedSpellings: manager.removedSpellingsOf(docId),
5796
5851
  buffered: false
5797
5852
  };
5798
5853
  }
5799
5854
  function applyOfUpdate(ctx, indexName, prepared) {
5800
5855
  const manager = ctx.requireManager(indexName);
5801
- const { docId, document, partitionDoc, rollbackDoc, extractedVectors } = prepared;
5856
+ const { docId, document, partitionDoc, rollbackDoc, extractedVectors, removedSpellings } = prepared;
5802
5857
  return async () => {
5803
5858
  if (ctx.isRebalancing(indexName)) {
5804
5859
  const bufferedState = ctx.bufferedDocState(indexName, docId);
@@ -5811,7 +5866,14 @@ function applyOfUpdate(ctx, indexName, prepared) {
5811
5866
  prepared.buffered = true;
5812
5867
  return;
5813
5868
  }
5814
- await ctx.executor.execute({ type: "update", indexName, docId, document: partitionDoc, requestId: docId });
5869
+ await ctx.executor.execute({
5870
+ type: "update",
5871
+ indexName,
5872
+ docId,
5873
+ document: partitionDoc,
5874
+ requestId: docId,
5875
+ removedSpellings
5876
+ });
5815
5877
  try {
5816
5878
  updateDocumentVectors(docId, extractedVectors, manager.getVectorIndexes(), manager.partitionIdOf(docId));
5817
5879
  } catch (err) {
@@ -5996,9 +6058,10 @@ async function replayEntry(manager, indexName, entry, ctx) {
5996
6058
  vectorFieldPaths
5997
6059
  );
5998
6060
  const overwrote = manager.has(entry.docId);
6061
+ const removedSpellings = overwrote ? manager.removedSpellingsOf(entry.docId) : void 0;
5999
6062
  if (overwrote) {
6000
6063
  removeDocumentVectors(entry.docId, vecIndexes);
6001
- manager.remove(entry.docId);
6064
+ manager.remove(entry.docId, removedSpellings);
6002
6065
  }
6003
6066
  manager.insert(entry.docId, partitionDoc);
6004
6067
  if (extractedVectors.size > 0) {
@@ -6012,17 +6075,20 @@ async function replayEntry(manager, indexName, entry, ctx) {
6012
6075
  } catch (err) {
6013
6076
  warnHookError("afterInsert", err);
6014
6077
  }
6015
- await ctx.orchestrator.replicateToWorkers({
6016
- type: overwrote ? "update" : "insert",
6078
+ const replicated = {
6017
6079
  indexName,
6018
6080
  docId: entry.docId,
6019
6081
  document: entry.document,
6020
6082
  requestId: `replicate-insert-${entry.docId}`
6021
- });
6083
+ };
6084
+ await ctx.orchestrator.replicateToWorkers(
6085
+ overwrote ? { type: "update", ...replicated, removedSpellings } : { type: "insert", ...replicated }
6086
+ );
6022
6087
  return;
6023
6088
  }
6024
6089
  if (entry.action === "remove") {
6025
- manager.remove(entry.docId);
6090
+ const removedSpellings = manager.removedSpellingsOf(entry.docId);
6091
+ manager.remove(entry.docId, removedSpellings);
6026
6092
  removeDocumentVectors(entry.docId, vecIndexes);
6027
6093
  try {
6028
6094
  await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId: entry.docId });
@@ -6033,15 +6099,17 @@ async function replayEntry(manager, indexName, entry, ctx) {
6033
6099
  type: "remove",
6034
6100
  indexName,
6035
6101
  docId: entry.docId,
6036
- requestId: `replicate-remove-${entry.docId}`
6102
+ requestId: `replicate-remove-${entry.docId}`,
6103
+ removedSpellings
6037
6104
  });
6038
6105
  return;
6039
6106
  }
6040
6107
  if (entry.action === "update" && entry.document) {
6041
6108
  const oldDocument = manager.get(entry.docId);
6042
6109
  const { updateDoc, updateVectors } = applyReplayUpdateVectors(entry, vectorFieldPaths);
6110
+ const removedSpellings = manager.removedSpellingsOf(entry.docId);
6043
6111
  if (manager.has(entry.docId)) {
6044
- manager.update(entry.docId, updateDoc);
6112
+ manager.update(entry.docId, updateDoc, void 0, removedSpellings);
6045
6113
  } else {
6046
6114
  manager.insert(entry.docId, updateDoc);
6047
6115
  }
@@ -6078,7 +6146,8 @@ async function replayEntry(manager, indexName, entry, ctx) {
6078
6146
  indexName,
6079
6147
  docId: entry.docId,
6080
6148
  document: entry.document,
6081
- requestId: `replicate-update-${entry.docId}`
6149
+ requestId: `replicate-update-${entry.docId}`,
6150
+ removedSpellings
6082
6151
  });
6083
6152
  }
6084
6153
  }
@@ -6727,7 +6796,7 @@ function portabilityOperations(core, config, engine) {
6727
6796
  };
6728
6797
  }
6729
6798
  async function restoreIndex(indexName, data) {
6730
- const { restoreFromSnapshot } = await import('./snapshot-6BYSUPWL.mjs');
6799
+ const { restoreFromSnapshot } = await import('./snapshot-22CUEYYW.mjs');
6731
6800
  await restoreFromSnapshot(indexName, data, {
6732
6801
  executor: core.executor,
6733
6802
  indexRegistry: core.indexRegistry,
@@ -6747,7 +6816,7 @@ function portabilityOperations(core, config, engine) {
6747
6816
  return {
6748
6817
  snapshot(indexName) {
6749
6818
  return withOpenIndex(indexName, async () => {
6750
- const { createSnapshot } = await import('./snapshot-6BYSUPWL.mjs');
6819
+ const { createSnapshot } = await import('./snapshot-22CUEYYW.mjs');
6751
6820
  return createSnapshot(snapshotDepsOf(indexName));
6752
6821
  });
6753
6822
  },
@@ -6759,8 +6828,8 @@ function portabilityOperations(core, config, engine) {
6759
6828
  exportDocuments(indexName) {
6760
6829
  return withOpenIndex(indexName, async () => {
6761
6830
  const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
6762
- import('./snapshot-6BYSUPWL.mjs'),
6763
- import('./write-HLEZTL2L.mjs')
6831
+ import('./snapshot-22CUEYYW.mjs'),
6832
+ import('./write-PXDKPG2R.mjs')
6764
6833
  ]);
6765
6834
  const header = indexConfigurationOf(core.requireIndex(indexName));
6766
6835
  return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
@@ -7244,4 +7313,4 @@ async function createNarsil(config) {
7244
7313
  // src/index.ts
7245
7314
  var VERSION = "0.1.0";
7246
7315
 
7247
- export { BYTES_TAG, DATE_TAG, DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, HEADER_SIZE, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, NUMBER_TAG, OBJECT_TAG, VERSION, crc322 as crc32, crc32Final, crc32Init, crc32Update, createByteReader, createEngineCore, createEngineIndex, createNarsil, createNarsilFromCore, decodeIndexConfiguration, encodeBase64, encodeIndexConfiguration, engineVersionParts, hasNrslMagic, insertDocumentVectors, openChecksummedEnvelope, packChecksummedEnvelope, prepareDocumentVectors, readHeader, reconstructSchemaFromMetadata, removeDocumentVectors, requireConfigurationWithinSizeLimit, requireKnownConfig, resolveRunnableConfig, runEngineListDocuments, runEnginePreflight, runEngineQuery, runEngineQueryStats, runEngineSuggest, startEngineCore, startsWithTagPrefix, validateEmbeddingConfig, validateRequiredFieldsInSchema, writeHeader };
7316
+ export { BYTES_TAG, DATE_TAG, DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, HEADER_SIZE, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, NUMBER_TAG, OBJECT_TAG, VERSION, crc322 as crc32, crc32Final, crc32Init, crc32Update, createByteReader, createEngineCore, createEngineIndex, createNarsil, createNarsilFromCore, decodeIndexConfiguration, emptySerializablePartition, encodeBase64, encodeIndexConfiguration, engineVersionParts, hasNrslMagic, insertDocumentVectors, openChecksummedEnvelope, packChecksummedEnvelope, prepareDocumentVectors, readHeader, reconstructSchemaFromMetadata, removeDocumentVectors, requireConfigurationWithinSizeLimit, requireKnownConfig, resolveRunnableConfig, runEngineListDocuments, runEnginePreflight, runEngineQuery, runEngineQueryStats, runEngineSuggest, startEngineCore, startsWithTagPrefix, validateEmbeddingConfig, validateRequiredFieldsInSchema, writeHeader };
@@ -1,8 +1,8 @@
1
- import { writeLiveVectors, decodeSegmentManifest, openManifestPayload, listedKeysOf, decodeManifestPayload, SINGLE_NODE_PRIMARY_TERM } from './chunk-JFJBA3PD.mjs';
1
+ import { writeLiveVectors, decodeSegmentManifest, openManifestPayload, listedKeysOf, decodeManifestPayload, SINGLE_NODE_PRIMARY_TERM } from './chunk-CL4OM7GB.mjs';
2
2
  import { vectorsPerFile } from './chunk-MXS5HWIH.mjs';
3
3
  import { vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION } from './chunk-YCJLVNWR.mjs';
4
- import { writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, commitCheckpointManifest, manifestKey, manifestFileRefs, partitionContentsOf } from './chunk-T7DMUVTL.mjs';
5
- import { HEADER_SIZE } from './chunk-AQU2QHSY.mjs';
4
+ import { writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, commitCheckpointManifest, manifestKey, manifestFileRefs, partitionContentsOf } from './chunk-QAMUC2BJ.mjs';
5
+ import { HEADER_SIZE } from './chunk-R6BMMQXH.mjs';
6
6
  import { MAX_FILE_BYTES } from './chunk-RVM7AODE.mjs';
7
7
  import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
8
8
 
@@ -1,6 +1,6 @@
1
- import { decodeSegmentManifest, loadPartitionSegments, loadVectorField } from './chunk-JFJBA3PD.mjs';
2
- import { manifestKey, segmentsPrefix } from './chunk-T7DMUVTL.mjs';
3
- import { readHeader, HEADER_SIZE, crc32, engineVersionParts, writeHeader } from './chunk-AQU2QHSY.mjs';
1
+ import { decodeSegmentManifest, loadPartitionSegments, loadVectorField } from './chunk-CL4OM7GB.mjs';
2
+ import { manifestKey, segmentsPrefix } from './chunk-QAMUC2BJ.mjs';
3
+ import { readHeader, HEADER_SIZE, crc32, engineVersionParts, writeHeader } from './chunk-R6BMMQXH.mjs';
4
4
  import { VALID_HNSW_METRICS } from './chunk-RVM7AODE.mjs';
5
5
  import { requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-CVQWQVFX.mjs';
6
6
  import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
@@ -1,4 +1,4 @@
1
- import { DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS } from './chunk-T7DMUVTL.mjs';
1
+ import { DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS } from './chunk-QAMUC2BJ.mjs';
2
2
  import { nodeThreadId } from './chunk-C7SLA6CM.mjs';
3
3
  import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
4
4
  import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';