@delali/narsil 0.3.1-next.109 → 0.3.1-next.112

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/dist/adapters/filesystem.mjs +1 -1
  2. package/dist/adapters/indexeddb.mjs +1 -1
  3. package/dist/browser/{chunk-RYWURYTW.mjs → chunk-2CBQ2PCC.mjs} +1 -1
  4. package/dist/browser/{chunk-EQ7BQ6VZ.mjs → chunk-LJS6PREA.mjs} +1 -0
  5. package/dist/browser/{chunk-3RXEGUGO.mjs → chunk-O5IKTA27.mjs} +41 -4
  6. package/dist/browser/{chunk-Z2K5AGZL.mjs → chunk-OVY4RPDS.mjs} +3 -3
  7. package/dist/browser/{chunk-S7KNCVPU.mjs → chunk-P7C2S5JX.mjs} +2 -2
  8. package/dist/browser/{chunk-SCBBEQWD.mjs → chunk-UR5MNN53.mjs} +189 -51
  9. package/dist/browser/{chunk-XHRHOH2P.mjs → chunk-YRYBTRWB.mjs} +139 -28
  10. package/dist/browser/{durability-wiring-JJGE7MED.mjs → durability-wiring-CPEMDDK7.mjs} +48 -20
  11. package/dist/browser/{module-PCKQADMQ.mjs → module-DYKR4INA.mjs} +5 -5
  12. package/dist/browser/{restore-queue-2STSWU2E.mjs → restore-queue-2H4QDQV4.mjs} +1 -1
  13. package/dist/browser/{runtime-ARKW5NZX.mjs → runtime-FL5K4JOH.mjs} +1 -1
  14. package/dist/browser/{snapshot-3AZDQKNA.mjs → snapshot-ISZMBDCN.mjs} +27 -14
  15. package/dist/browser/{write-XXLMPT3Y.mjs → write-IABOLE4N.mjs} +5 -5
  16. package/dist/{capabilities-D4MUmjfe.d.ts → capabilities-ByaX8Arf.d.ts} +2 -2
  17. package/dist/{chunk-CYXLPFBG.mjs → chunk-2IFU7SXG.mjs} +123 -35
  18. package/dist/{chunk-FN75BCXC.mjs → chunk-2RAQKITH.mjs} +1 -1
  19. package/dist/{chunk-VIXXQEX6.mjs → chunk-3COL24MT.mjs} +43 -6
  20. package/dist/{chunk-ICZW4J3I.mjs → chunk-3RRATITJ.mjs} +21 -21
  21. package/dist/{chunk-SAZB7ZHT.mjs → chunk-542TD4BL.mjs} +1 -1
  22. package/dist/{chunk-C57FPCTL.mjs → chunk-5WEBVYOA.mjs} +182 -44
  23. package/dist/{chunk-YP23IAV5.mjs → chunk-63HFWQEA.mjs} +4 -3
  24. package/dist/{chunk-FV5CP3E2.mjs → chunk-6ODIY6ZB.mjs} +1 -1
  25. package/dist/{chunk-RQ7LOBLD.mjs → chunk-6PN3U6XW.mjs} +1 -1
  26. package/dist/{chunk-SYHS56F7.mjs → chunk-7CI34JAP.mjs} +26 -13
  27. package/dist/{chunk-IBJWO5PF.mjs → chunk-7ZZQNOIY.mjs} +1 -1
  28. package/dist/{chunk-H5465KUZ.mjs → chunk-BVOUSX3U.mjs} +4 -3
  29. package/dist/{chunk-AWZVJCCW.mjs → chunk-DG62H7OD.mjs} +2 -2
  30. package/dist/{chunk-LWQTFKRS.mjs → chunk-FUUP45IS.mjs} +1 -1
  31. package/dist/{chunk-HL2GAS5N.mjs → chunk-GLZI636G.mjs} +7 -7
  32. package/dist/{chunk-C55OFKU6.mjs → chunk-HNAP2KTG.mjs} +2 -2
  33. package/dist/{chunk-A2HWRBQE.mjs → chunk-IBTKJIV7.mjs} +2 -2
  34. package/dist/{chunk-PA5WZWLC.mjs → chunk-IM76DLA7.mjs} +2 -2
  35. package/dist/{chunk-5LP7OU6B.mjs → chunk-IR4KQ4WB.mjs} +2 -2
  36. package/dist/{chunk-FABI2JA7.mjs → chunk-J6YVB2XM.mjs} +1 -1
  37. package/dist/{chunk-62GK7UUS.mjs → chunk-KCQG2PJT.mjs} +3 -3
  38. package/dist/{chunk-U6FT7LKG.mjs → chunk-LOBYK2FG.mjs} +3 -3
  39. package/dist/{chunk-XY6Z7T5W.mjs → chunk-N57XXGPG.mjs} +2 -2
  40. package/dist/{chunk-S2CKO6PP.mjs → chunk-OBWUWZRA.mjs} +1 -0
  41. package/dist/{chunk-CKRP7GCS.mjs → chunk-OCB7TMUQ.mjs} +1 -1
  42. package/dist/{chunk-NYWUBTMH.mjs → chunk-OFYKBKKV.mjs} +1 -1
  43. package/dist/{chunk-ZBZRBM2Z.mjs → chunk-PT7VRWS7.mjs} +3 -3
  44. package/dist/{chunk-DUWYU4D5.mjs → chunk-PWIWSUCX.mjs} +2 -2
  45. package/dist/{chunk-5W7ADHK7.mjs → chunk-QI6JW4PM.mjs} +1 -1
  46. package/dist/{chunk-SD2EYVU4.mjs → chunk-QIFOXLWF.mjs} +2 -2
  47. package/dist/{chunk-N75MAOM7.mjs → chunk-R2MRGYU5.mjs} +2 -2
  48. package/dist/{chunk-5DPTHLMX.mjs → chunk-V7G627TA.mjs} +1 -1
  49. package/dist/{chunk-4GD43FSW.mjs → chunk-V7ISFBFC.mjs} +10 -10
  50. package/dist/{chunk-MVWWAHTR.mjs → chunk-VK45J47R.mjs} +1 -1
  51. package/dist/{chunk-RSSLOFRP.mjs → chunk-VLRT47GU.mjs} +39 -8
  52. package/dist/{chunk-RPIE4QCL.mjs → chunk-YH5OIL7B.mjs} +2 -0
  53. package/dist/{chunk-JKOBHB4A.mjs → chunk-YIDSHBQS.mjs} +2 -2
  54. package/dist/{client-Ba9yP-Rr.d.ts → client-BdnzDFtO.d.ts} +2 -2
  55. package/dist/client.d.ts +6 -6
  56. package/dist/client.mjs +1 -0
  57. package/dist/{config-C-xXz8bC.d.ts → config-CSgvApt0.d.ts} +1 -1
  58. package/dist/{config-Dc_8-byz.d.ts → config-ujZESkJz.d.ts} +1 -1
  59. package/dist/{direct-executor-QH2M6AK2.mjs → direct-executor-ZSTKRA6Z.mjs} +11 -11
  60. package/dist/distribution/coordinator/etcd.mjs +94 -5
  61. package/dist/distribution/transport/grpc.mjs +24 -8
  62. package/dist/distribution/transport/in-memory.mjs +4 -4
  63. package/dist/distribution/transport/tcp.mjs +6 -6
  64. package/dist/distribution.d.ts +5 -5
  65. package/dist/distribution.mjs +1232 -377
  66. package/dist/{durability-wiring-4HWX7OVP.mjs → durability-wiring-EQTFL3B4.mjs} +65 -38
  67. package/dist/embeddings/openai.mjs +2 -2
  68. package/dist/{engine-CMZ5XSLy.d.ts → engine-C_HSy3iz.d.ts} +1 -1
  69. package/dist/{events-dnvgprIl.d.ts → events-CJsXF1F9.d.ts} +1 -0
  70. package/dist/index.browser.mjs +3 -3
  71. package/dist/index.d.ts +5 -5
  72. package/dist/index.mjs +16 -16
  73. package/dist/{module-IYILOTU3.mjs → module-WP4VVGV2.mjs} +9 -9
  74. package/dist/partial-words.mjs +5 -5
  75. package/dist/pattern.mjs +3 -3
  76. package/dist/persistence/durability/checkpoint-worker.d.ts +12 -0
  77. package/dist/persistence/durability/checkpoint-worker.mjs +25 -23
  78. package/dist/react.d.ts +6 -6
  79. package/dist/react.mjs +1 -0
  80. package/dist/{rerank-EHJVXEDP.mjs → rerank-3POMPQNX.mjs} +4 -4
  81. package/dist/rerankers/http.mjs +2 -2
  82. package/dist/{restore-queue-BONSQWCI.mjs → restore-queue-WZTMTJP2.mjs} +1 -1
  83. package/dist/{runtime-LEX2F7OW.mjs → runtime-C2435MPK.mjs} +5 -5
  84. package/dist/server.d.ts +8 -8
  85. package/dist/server.mjs +12 -12
  86. package/dist/{snapshot-DWDLN5YW.mjs → snapshot-EU6NNB2P.mjs} +25 -25
  87. package/dist/synonyms.mjs +3 -3
  88. package/dist/{tasks-B4B2cMK5.d.ts → tasks-CI3Fvkxw.d.ts} +1 -1
  89. package/dist/{thread-I32PUURR.mjs → thread-WGBCVJII.mjs} +11 -11
  90. package/dist/vector/search-worker.mjs +4 -4
  91. package/dist/workers/entry.mjs +5 -5
  92. package/dist/{write-AOFSHACR.mjs → write-CKRWS3P5.mjs} +18 -18
  93. package/package.json +7 -7
@@ -1,8 +1,8 @@
1
- import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes } from './chunk-3RXEGUGO.mjs';
2
- import { VERSION, rulesOfStoredSynonyms, flattenSchema, packChecksummedEnvelope, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, fnv1a, selectionKeeps, decodeMapWithoutFields, prepareDocumentVectors, messagePackUpperBoundBytes, segmentArraysOf, mergeSegmentContents, resolveIndexAnalysis } from './chunk-SCBBEQWD.mjs';
3
- import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-S7KNCVPU.mjs';
4
- import { compareCodePoints, MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-RYWURYTW.mjs';
5
- import { NarsilError, ErrorCodes } from './chunk-EQ7BQ6VZ.mjs';
1
+ import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes, uint64Bytes } from './chunk-O5IKTA27.mjs';
2
+ import { VERSION, rulesOfStoredSynonyms, flattenSchema, packChecksummedEnvelope, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, fnv1a, selectionKeeps, decodeMapWithoutFields, prepareDocumentVectors, messagePackUpperBoundBytes, segmentArraysOf, mergeSegmentContents, resolveIndexAnalysis } from './chunk-UR5MNN53.mjs';
3
+ import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-P7C2S5JX.mjs';
4
+ import { compareCodePoints, MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-2CBQ2PCC.mjs';
5
+ import { NarsilError, ErrorCodes } from './chunk-LJS6PREA.mjs';
6
6
  import { decode, encode } from '@msgpack/msgpack';
7
7
 
8
8
  // src/engine/durability-metadata.ts
@@ -733,7 +733,10 @@ function encodeSegmentManifest(manifest) {
733
733
  checkpoint: manifest.checkpoint.map((c) => ({
734
734
  partitionId: c.partitionId,
735
735
  lastSeqNo: c.lastSeqNo,
736
- primaryTerm: c.primaryTerm
736
+ primaryTerm: c.primaryTerm,
737
+ ...c.termHistory === void 0 ? {} : {
738
+ termHistory: c.termHistory.map((start) => ({ primaryTerm: start.primaryTerm, firstSeqNo: start.firstSeqNo }))
739
+ }
737
740
  })),
738
741
  partitions: manifest.partitions.map((p) => ({
739
742
  partitionId: p.partitionId,
@@ -922,7 +925,33 @@ function normalizeCheckpoint(raw) {
922
925
  invalidManifest("has an invalid partition checkpoint", { entry });
923
926
  }
924
927
  seen.add(entry.partitionId);
925
- return { partitionId: entry.partitionId, lastSeqNo: entry.lastSeqNo, primaryTerm: entry.primaryTerm };
928
+ const checkpoint = {
929
+ partitionId: entry.partitionId,
930
+ lastSeqNo: entry.lastSeqNo,
931
+ primaryTerm: entry.primaryTerm
932
+ };
933
+ if (entry.termHistory !== void 0) {
934
+ checkpoint.termHistory = normalizeTermHistory(entry.termHistory, entry.partitionId, entry.lastSeqNo);
935
+ }
936
+ return checkpoint;
937
+ });
938
+ }
939
+ function normalizeTermHistory(raw, partitionId, lastSeqNo) {
940
+ if (!Array.isArray(raw)) invalidManifest("has a term history that is not a list", { partitionId });
941
+ let previous = null;
942
+ return raw.map((start) => {
943
+ if (!isRecord(start) || !isSequenceNumber(start.primaryTerm) || !isSequenceNumber(start.firstSeqNo)) {
944
+ invalidManifest("has an invalid term history entry", { partitionId });
945
+ }
946
+ const read = { primaryTerm: start.primaryTerm, firstSeqNo: start.firstSeqNo };
947
+ if (read.firstSeqNo > lastSeqNo) {
948
+ invalidManifest(`has a term starting at seqNo ${read.firstSeqNo}, above lastSeqNo ${lastSeqNo}`, { partitionId });
949
+ }
950
+ if (previous !== null && (read.primaryTerm <= previous.primaryTerm || read.firstSeqNo <= previous.firstSeqNo)) {
951
+ invalidManifest("has a term history that fails to ascend strictly", { partitionId });
952
+ }
953
+ previous = read;
954
+ return read;
926
955
  });
927
956
  }
928
957
  function normalizePartitions(raw, indexName) {
@@ -997,30 +1026,40 @@ function frozenSegmentsOf(partition, shared) {
997
1026
  if (tail !== null) segments.push(tail);
998
1027
  return segments;
999
1028
  }
1000
- function partitionContentsOf(manager, partitionId) {
1029
+ function partitionContentsOf(manager, partitionId, captureRecords) {
1001
1030
  const partition = manager.getPartition(partitionId);
1002
- const selections = frozenSegmentsOf(partition, false).map((segment) => selectionOfMask(segment, keepMaskOf(segment)));
1003
- return { partitionId, docCount: partition.count(), selections };
1031
+ const segments = frozenSegmentsOf(partition, false);
1032
+ const selections = segments.map((segment) => selectionOfMask(segment, keepMaskOf(segment)));
1033
+ const contents = { partitionId, docCount: partition.count(), selections };
1034
+ if (captureRecords !== void 0) contents.writeRecords = captureRecords(segments);
1035
+ return contents;
1004
1036
  }
1005
- function captureWholePartition(manager, partitionId) {
1037
+ function captureWholePartition(manager, partitionId, captureRecords) {
1006
1038
  if (typeof SharedArrayBuffer !== "function") return null;
1007
1039
  const partition = manager.getPartition(partitionId);
1008
1040
  if (!isCompositePartition(partition) || partition.frozenSegmentCount() === 0) return null;
1041
+ const frozen = frozenSegmentsOf(partition, true);
1042
+ const records = captureRecords?.(frozen);
1009
1043
  const segments = [];
1010
- for (const segment of frozenSegmentsOf(partition, true)) {
1044
+ for (const segment of frozen) {
1011
1045
  if (segment.sharedSnapshot === null) return null;
1012
- segments.push({ snapshot: segment.sharedSnapshot, keep: keepMaskOf(segment) });
1046
+ const captured = { snapshot: segment.sharedSnapshot, keep: keepMaskOf(segment) };
1047
+ const held = records?.get(segment);
1048
+ if (held !== void 0) captured.writeRecords = held;
1049
+ segments.push(captured);
1013
1050
  }
1014
1051
  return { partitionId, docCount: partition.count(), segments };
1015
1052
  }
1016
1053
  function contentsOfCaptured(captured) {
1017
- return {
1018
- partitionId: captured.partitionId,
1019
- docCount: captured.docCount,
1020
- selections: captured.segments.map(
1021
- ({ snapshot, keep }) => selectionOfMask(createSharedFrozenSegment(snapshot), keep)
1022
- )
1023
- };
1054
+ const writeRecords = /* @__PURE__ */ new Map();
1055
+ const selections = captured.segments.map(({ snapshot, keep, writeRecords: held }) => {
1056
+ const segment = createSharedFrozenSegment(snapshot);
1057
+ if (held !== void 0) writeRecords.set(segment, held);
1058
+ return selectionOfMask(segment, keep);
1059
+ });
1060
+ const contents = { partitionId: captured.partitionId, docCount: captured.docCount, selections };
1061
+ if (writeRecords.size > 0) contents.writeRecords = writeRecords;
1062
+ return contents;
1024
1063
  }
1025
1064
 
1026
1065
  // src/persistence/durability/segment/store.ts
@@ -1383,6 +1422,15 @@ function optionalIndexUuid(value, payloadName) {
1383
1422
  }
1384
1423
  return value;
1385
1424
  }
1425
+ function optionalPreviousPrimaryTerm(value, payloadName) {
1426
+ if (value === void 0 || value === null) {
1427
+ return void 0;
1428
+ }
1429
+ if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
1430
+ throw new Error(`Invalid ${payloadName}: "previousPrimaryTerm" must be a non-negative integer or absent`);
1431
+ }
1432
+ return value;
1433
+ }
1386
1434
  function validateEntryPayload(decoded) {
1387
1435
  if (!isRecord2(decoded) || !isRecord2(decoded.entry)) {
1388
1436
  throw new Error('Invalid EntryPayload: missing or invalid "entry" field');
@@ -1413,6 +1461,7 @@ function validateEntryPayload(decoded) {
1413
1461
  throw new Error('Invalid EntryPayload: "entry.checksum" must be a number');
1414
1462
  }
1415
1463
  optionalIndexUuid(decoded.indexUuid, "EntryPayload");
1464
+ optionalPreviousPrimaryTerm(decoded.previousPrimaryTerm, "EntryPayload");
1416
1465
  return decoded;
1417
1466
  }
1418
1467
  function computeEntryChecksum(entry) {
@@ -1699,6 +1748,17 @@ async function* walEntriesInRange(directory, indexName, partitionId, fromSeqNoEx
1699
1748
  throw durableRecordMissing(indexName, partitionId, highestRead, wal.marker);
1700
1749
  }
1701
1750
  }
1751
+ function writeRecordsOf(documents, written) {
1752
+ const seqNos = new Float64Array(documents.length);
1753
+ const primaryTerms = new Float64Array(documents.length);
1754
+ for (let at = 0; at < documents.length; at++) {
1755
+ const entry = written.get(documents[at].docId);
1756
+ if (entry === void 0) continue;
1757
+ seqNos[at] = entry.seqNo;
1758
+ primaryTerms[at] = entry.primaryTerm;
1759
+ }
1760
+ return { seqNos, primaryTerms };
1761
+ }
1702
1762
  function everyVectorFieldIsTopLevel(vectorFieldPaths) {
1703
1763
  for (const fieldPath of vectorFieldPaths) {
1704
1764
  if (fieldPath.includes(".")) return false;
@@ -1716,15 +1776,18 @@ async function buildSegmentFromEntries(input) {
1716
1776
  const leavesVectorsUndecoded = everyVectorFieldIsTopLevel(input.vectorFieldPaths);
1717
1777
  const live = /* @__PURE__ */ new Map();
1718
1778
  const deleted = /* @__PURE__ */ new Set();
1779
+ const written = /* @__PURE__ */ new Map();
1719
1780
  for await (const entry of input.entries) {
1720
1781
  if (entry.operation === "DELETE") {
1721
1782
  deleted.add(entry.documentId);
1722
1783
  live.delete(entry.documentId);
1784
+ written.delete(entry.documentId);
1723
1785
  continue;
1724
1786
  }
1725
1787
  if (entry.document === null) continue;
1726
1788
  live.delete(entry.documentId);
1727
1789
  live.set(entry.documentId, documentWithoutVectors(entry.document, input.vectorFieldPaths, leavesVectorsUndecoded));
1790
+ if (input.keepsWriteRecords) written.set(entry.documentId, entry);
1728
1791
  }
1729
1792
  const tombstones = [];
1730
1793
  for (const docId of deleted) {
@@ -1747,7 +1810,9 @@ async function buildSegmentFromEntries(input) {
1747
1810
  payload,
1748
1811
  documents.map((entry) => entry.document)
1749
1812
  );
1750
- return { selections: [{ segment }], tombstones, docCount: live.size };
1813
+ const built = { selections: [{ segment }], tombstones, docCount: live.size };
1814
+ if (input.keepsWriteRecords) built.writeRecords = /* @__PURE__ */ new Map([[segment, writeRecordsOf(documents, written)]]);
1815
+ return built;
1751
1816
  }
1752
1817
  var FIXMAP = 128;
1753
1818
  var MAP16 = 222;
@@ -1950,7 +2015,8 @@ function encodeKeywordSegment(contents) {
1950
2015
  const { arrays } = contents;
1951
2016
  const postings = postingsInTokenOrder(arrays);
1952
2017
  const out = new MessagePackChunks();
1953
- out.map(TOP_LEVEL_ENTRIES);
2018
+ const { writeRecords } = contents;
2019
+ out.map(writeRecords === void 0 ? TOP_LEVEL_ENTRIES : TOP_LEVEL_ENTRIES + 2);
1954
2020
  out.key("v").value(KEYWORD_SEGMENT_PAYLOAD_VERSION);
1955
2021
  out.key("schema").value(contents.schema);
1956
2022
  out.key("tombstones").value(contents.tombstones);
@@ -1988,6 +2054,10 @@ function encodeKeywordSegment(contents) {
1988
2054
  writeSurfaceForms(out, arrays);
1989
2055
  out.key("partialWords");
1990
2056
  writePartialWords(out, arrays);
2057
+ if (writeRecords !== void 0) {
2058
+ out.key("seqNos").binary(uint64Bytes(writeRecords.seqNos));
2059
+ out.key("primaryTerms").binary(uint64Bytes(writeRecords.primaryTerms));
2060
+ }
1991
2061
  return out;
1992
2062
  }
1993
2063
 
@@ -2199,6 +2269,28 @@ function noSpellingsOf(analysis) {
2199
2269
  if (analysis === null || analysis.fields.length === 0) return null;
2200
2270
  return { fields: [...analysis.fields], tokens: [], spellings: [], counts: [] };
2201
2271
  }
2272
+ function writeRecordsOfGroup(group, records, count) {
2273
+ const seqNos = new Float64Array(count);
2274
+ const primaryTerms = new Float64Array(count);
2275
+ let at = 0;
2276
+ for (const selection of group?.selections ?? []) {
2277
+ const source = records.get(selection.segment);
2278
+ for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
2279
+ if (!selectionKeeps(selection, ordinal)) continue;
2280
+ if (source !== void 0) {
2281
+ seqNos[at] = source.seqNos[ordinal];
2282
+ primaryTerms[at] = source.primaryTerms[ordinal];
2283
+ }
2284
+ at += 1;
2285
+ }
2286
+ }
2287
+ return { seqNos, primaryTerms };
2288
+ }
2289
+ function withWriteRecords(contents, group, request) {
2290
+ if (request.writeRecords === void 0) return contents;
2291
+ const writeRecords = writeRecordsOfGroup(group, request.writeRecords, contents.arrays.documentCount);
2292
+ return { ...contents, writeRecords };
2293
+ }
2202
2294
  function contentsOfGroup(group, request, tombstones) {
2203
2295
  const { schema, partialWordAnalysis, textAnalysis } = request;
2204
2296
  if (group === null) {
@@ -2227,7 +2319,7 @@ function requireRoomForAnotherSegment(state) {
2227
2319
  async function writeGroup(state, group) {
2228
2320
  const { request } = state;
2229
2321
  const tombstones = state.tombstonesWritten ? [] : [...request.tombstones];
2230
- const contents = contentsOfGroup(group, request, tombstones);
2322
+ const contents = withWriteRecords(contentsOfGroup(group, request, tombstones), group, request);
2231
2323
  const encoded = encodeKeywordSegment(contents);
2232
2324
  if (HEADER_SIZE + encoded.byteLength > MAX_FILE_BYTES && group !== null && group.documentCount > 1) {
2233
2325
  const [first, second] = halvesOf(group);
@@ -2314,6 +2406,15 @@ function applyTimeOrder(frozen, contents) {
2314
2406
  }
2315
2407
  }
2316
2408
  }
2409
+ function writeRecordsOf2(frozen, contents) {
2410
+ if (contents.every((segment) => segment.writeRecords === void 0)) return void 0;
2411
+ const records = /* @__PURE__ */ new Map();
2412
+ for (let at = 0; at < frozen.length; at++) {
2413
+ const held = contents[at].writeRecords;
2414
+ if (held !== void 0) records.set(frozen[at], held);
2415
+ }
2416
+ return records;
2417
+ }
2317
2418
  function retainedTombstones(frozen, contents) {
2318
2419
  const retained = /* @__PURE__ */ new Set();
2319
2420
  for (const segment of contents) {
@@ -2337,6 +2438,7 @@ async function compactPartitionSegments(input) {
2337
2438
  for (const ref of windowRefs) contents.push(await readSegmentFile(input.store, ref));
2338
2439
  const frozen = contents.map(frozenOf);
2339
2440
  applyTimeOrder(frozen, contents);
2441
+ const writeRecords = writeRecordsOf2(frozen, contents);
2340
2442
  const written = await writeSegmentsOf({
2341
2443
  store: input.store,
2342
2444
  indexName: input.indexName,
@@ -2347,7 +2449,8 @@ async function compactPartitionSegments(input) {
2347
2449
  textAnalysis: input.textAnalysis,
2348
2450
  selections: frozen.map((segment) => ({ segment })),
2349
2451
  tombstones: retainedTombstones(frozen, contents),
2350
- mergesIntoOneFile: true
2452
+ mergesIntoOneFile: true,
2453
+ ...writeRecords === void 0 ? {} : { writeRecords }
2351
2454
  });
2352
2455
  return {
2353
2456
  segments: [
@@ -2413,7 +2516,8 @@ async function writeWholePartition(context, partitionId, priorPartition, whole)
2413
2516
  partialWordAnalysis: context.partialWordAnalysis,
2414
2517
  textAnalysis: context.textAnalysis,
2415
2518
  selections: whole.selections,
2416
- tombstones: []
2519
+ tombstones: [],
2520
+ ...whole.writeRecords === void 0 ? {} : { writeRecords: whole.writeRecords }
2417
2521
  });
2418
2522
  return { partitionId, nextSegmentId: written.nextSegmentId, segments: written.segments };
2419
2523
  }
@@ -2423,7 +2527,8 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
2423
2527
  config: context.config,
2424
2528
  language: getLanguage(context.config.language ?? "english"),
2425
2529
  vectorFieldPaths: context.vectorFieldPaths,
2426
- entries
2530
+ entries,
2531
+ keepsWriteRecords: context.input.keepsWriteRecords === true
2427
2532
  });
2428
2533
  let segments = priorPartition ? [...priorPartition.segments] : [];
2429
2534
  let nextSegmentId = priorPartition?.nextSegmentId ?? 0;
@@ -2437,7 +2542,8 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
2437
2542
  partialWordAnalysis: context.partialWordAnalysis,
2438
2543
  textAnalysis: context.textAnalysis,
2439
2544
  selections: built.selections,
2440
- tombstones: built.tombstones
2545
+ tombstones: built.tombstones,
2546
+ ...built.writeRecords === void 0 ? {} : { writeRecords: built.writeRecords }
2441
2547
  });
2442
2548
  segments.push(...written.segments);
2443
2549
  nextSegmentId = written.nextSegmentId;
@@ -2580,6 +2686,7 @@ async function loadPartitionSegments(store, partition, manager, options) {
2580
2686
  if (options.decodesEveryDocument) decodeEveryStoredDocument(contents.arrays.documentTable);
2581
2687
  const segment = createSharedFrozenSegment(shareSegmentArrays(contents.arrays, manager.partialWordAnalysis));
2582
2688
  removeWhatTheSegmentReplaces(manager, segment, contents);
2689
+ options.writeRecords?.loadSegment(manager.indexName, partition.partitionId, segment, contents);
2583
2690
  if (contents.arrays.documentCount > 0) manager.attachFrozenSegment(partition.partitionId, segment);
2584
2691
  await yieldToEventLoop();
2585
2692
  }
@@ -2626,7 +2733,11 @@ async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodes
2626
2733
  { indexName, partitionId: partition.partitionId, partitionCount: deps.manager.partitionCount }
2627
2734
  );
2628
2735
  }
2629
- await loadPartitionSegments(store, partition, deps.manager, { schema: manifest.schema, decodesEveryDocument });
2736
+ await loadPartitionSegments(store, partition, deps.manager, {
2737
+ schema: manifest.schema,
2738
+ decodesEveryDocument,
2739
+ writeRecords: deps.writeRecords ?? null
2740
+ });
2630
2741
  }
2631
2742
  for (const vector of manifest.vectors) {
2632
2743
  const vectorIndex = deps.vectorIndexes.get(vector.fieldPath);
@@ -1,11 +1,11 @@
1
1
  import { nodeThreadId, spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
2
- import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, adapterCheckpointStore, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, writeMetadataEnvelope, metadataFromConfig, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readSegmentManifest, snapshotBundleKey, reclaimUnlistedKeys, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, manifestFileRefs, DIRECTORY_LOCK_SETTLE_MS, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, isId, isUint32, isSequenceNumber, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, MAX_SEGMENTS_PER_PARTITION, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, buildEntry, readCurrentManifest, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-XHRHOH2P.mjs';
3
- import { applyDeleteEntry, applyIndexEntry } from './chunk-3RXEGUGO.mjs';
4
- import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-SCBBEQWD.mjs';
5
- import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-S7KNCVPU.mjs';
6
- import { compareCodePoints, MIN_M, MAX_FILE_BYTES, detectRuntime, resolveWorkerEntry } from './chunk-RYWURYTW.mjs';
2
+ import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, adapterCheckpointStore, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, writeMetadataEnvelope, metadataFromConfig, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readSegmentManifest, snapshotBundleKey, reclaimUnlistedKeys, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, manifestFileRefs, DIRECTORY_LOCK_SETTLE_MS, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, isId, isUint32, isSequenceNumber, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, MAX_SEGMENTS_PER_PARTITION, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, buildEntry, readCurrentManifest, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-YRYBTRWB.mjs';
3
+ import { applyDeleteEntry, applyIndexEntry } from './chunk-O5IKTA27.mjs';
4
+ import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-UR5MNN53.mjs';
5
+ import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-P7C2S5JX.mjs';
6
+ import { compareCodePoints, MIN_M, MAX_FILE_BYTES, detectRuntime, resolveWorkerEntry } from './chunk-2CBQ2PCC.mjs';
7
7
  import './chunk-JPTQZWJQ.mjs';
8
- import { NarsilError, ErrorCodes, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-EQ7BQ6VZ.mjs';
8
+ import { NarsilError, ErrorCodes, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-LJS6PREA.mjs';
9
9
  import { decode } from '@msgpack/msgpack';
10
10
 
11
11
  // src/persistence/durability/node-fs.browser.ts
@@ -1108,6 +1108,7 @@ async function truncateSegmentTail(directory, key, durableByteLength) {
1108
1108
  }
1109
1109
  }
1110
1110
  function applyEntry(entry, deps) {
1111
+ deps.writeRecords?.replayed(entry);
1111
1112
  if (entry.operation === "DELETE") {
1112
1113
  applyDeleteEntry(entry, deps.manager, deps.vectorIndexes);
1113
1114
  return;
@@ -1166,7 +1167,7 @@ function noWholePartitions() {
1166
1167
  return { contents: [], captured: [] };
1167
1168
  }
1168
1169
  function wholePartitionsWhereMostChanged(manager, rules) {
1169
- const { priorCheckpoint, priorPartitionIds, everyPartition, aWorkerCanSerialise } = rules;
1170
+ const { priorCheckpoint, priorPartitionIds, everyPartition, aWorkerCanSerialise, captureRecords } = rules;
1170
1171
  return (targets) => {
1171
1172
  const whole = { contents: [], captured: [] };
1172
1173
  const changed = targets.filter(
@@ -1180,17 +1181,18 @@ function wholePartitionsWhereMostChanged(manager, rules) {
1180
1181
  const logMatchesMemory = targets.length === 1 && priorPartitionIds.every((partitionId) => partitionId === 0);
1181
1182
  if (!everyPartition && !(mostOfEachChanged && logMatchesMemory)) return whole;
1182
1183
  for (const { partitionId } of changed) {
1183
- const captured = aWorkerCanSerialise ? captureWholePartition(manager, partitionId) : null;
1184
+ const capture2 = captureRecords?.(partitionId);
1185
+ const captured = aWorkerCanSerialise ? captureWholePartition(manager, partitionId, capture2) : null;
1184
1186
  if (captured !== null) {
1185
1187
  whole.captured.push(captured);
1186
1188
  } else if (everyPartition || !aWorkerCanSerialise) {
1187
- whole.contents.push(partitionContentsOf(manager, partitionId));
1189
+ whole.contents.push(partitionContentsOf(manager, partitionId, capture2));
1188
1190
  }
1189
1191
  }
1190
1192
  return whole;
1191
1193
  };
1192
1194
  }
1193
- async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = []) {
1195
+ async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf) {
1194
1196
  for (const vectorIndex of vectorIndexes.values()) await vectorIndex.completeGraph();
1195
1197
  return whileNoMutationApplies([...indexState.partitions.values()], () => {
1196
1198
  const documentCount = manager.countDocuments();
@@ -1205,11 +1207,13 @@ async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartit
1205
1207
  snapshotCheckpointFor(priorCheckpointThatLimitsTheRecords, partitionId) + LOG_RECORDS_PER_CHECKPOINT
1206
1208
  );
1207
1209
  recordsLeftInLog += appliedSeqNo - lastSeqNo;
1208
- targets.push({
1210
+ const target = {
1209
1211
  partitionId,
1210
1212
  lastSeqNo,
1211
1213
  primaryTerm: partition?.seqOwner.primaryTerm ?? SINGLE_NODE_PRIMARY_TERM
1212
- });
1214
+ };
1215
+ if (termHistoryOf !== void 0) target.termHistory = termHistoryOf(partitionId, lastSeqNo);
1216
+ targets.push(target);
1213
1217
  }
1214
1218
  const vectorPlans = /* @__PURE__ */ new Map();
1215
1219
  for (const [fieldPath, vectorIndex] of vectorIndexes) {
@@ -1415,13 +1419,15 @@ function workerCanWriteEverySegment(input) {
1415
1419
  }
1416
1420
  async function writeIndexCheckpointSegments(input) {
1417
1421
  const { directory, metadata, targets, compactionThreshold, prior, wholePartitions, capturedPartitions } = input;
1422
+ const { keepsWriteRecords } = input;
1418
1423
  const offloaded = workerCanWriteEverySegment(input) ? await runCheckpointOnWorker({
1419
1424
  root: directory.root,
1420
1425
  metadata,
1421
1426
  targets,
1422
1427
  compactionThreshold,
1423
1428
  prior,
1424
- capturedPartitions: [...capturedPartitions]
1429
+ capturedPartitions: [...capturedPartitions],
1430
+ keepsWriteRecords
1425
1431
  }) : null;
1426
1432
  if (offloaded !== null) {
1427
1433
  return offloaded;
@@ -1434,7 +1440,8 @@ async function writeIndexCheckpointSegments(input) {
1434
1440
  compactionThreshold,
1435
1441
  prior,
1436
1442
  wholePartitions,
1437
- capturedPartitions
1443
+ capturedPartitions,
1444
+ keepsWriteRecords
1438
1445
  });
1439
1446
  }
1440
1447
 
@@ -1539,6 +1546,8 @@ async function writeOneBoundedCheckpoint(input) {
1539
1546
  }
1540
1547
  const vectorIndexes = input.hooks.getVectorIndexes(input.indexName);
1541
1548
  const store = directoryCheckpointStore(input.directory);
1549
+ const writeRecords = input.hooks.writeRecords ?? null;
1550
+ const { indexName } = input;
1542
1551
  const priorManifest = await readCurrentManifest(store, input.indexName);
1543
1552
  const capture2 = await captureCheckpoint(
1544
1553
  input.indexState,
@@ -1548,10 +1557,14 @@ async function writeOneBoundedCheckpoint(input) {
1548
1557
  priorCheckpoint: priorManifest?.checkpoint ?? [],
1549
1558
  priorPartitionIds: (priorManifest?.partitions ?? []).map((partition) => partition.partitionId),
1550
1559
  everyPartition: input.fromMemory,
1551
- aWorkerCanSerialise: input.canOffload && checkpointWorkerIsUsable()
1560
+ aWorkerCanSerialise: input.canOffload && checkpointWorkerIsUsable(),
1561
+ ...writeRecords === null ? {} : {
1562
+ captureRecords: (partitionId) => (segments2) => writeRecords.captureSegments(indexName, partitionId, segments2)
1563
+ }
1552
1564
  }),
1553
1565
  input.fromMemory ? void 0 : priorManifest?.checkpoint ?? [],
1554
- priorManifest?.vectors ?? []
1566
+ priorManifest?.vectors ?? [],
1567
+ writeRecords === null ? void 0 : (partitionId, lastSeqNo) => writeRecords.termHistoryOf(indexName, partitionId, lastSeqNo)
1555
1568
  );
1556
1569
  const { targets, documentCount } = capture2;
1557
1570
  await makeEveryAppliedMutationDurable(input.indexState, input.failPartition);
@@ -1566,7 +1579,8 @@ async function writeOneBoundedCheckpoint(input) {
1566
1579
  canOffload: input.canOffload,
1567
1580
  prior: priorManifest,
1568
1581
  wholePartitions: capture2.wholePartitions.contents,
1569
- capturedPartitions: capture2.wholePartitions.captured
1582
+ capturedPartitions: capture2.wholePartitions.captured,
1583
+ keepsWriteRecords: writeRecords !== null
1570
1584
  })
1571
1585
  );
1572
1586
  await makeEveryAppliedMutationDurable(input.indexState, input.failPartition);
@@ -2980,7 +2994,8 @@ async function recoverCheckpoint(deps) {
2980
2994
  }
2981
2995
  const checkpoint = await loadSegmentedCheckpoint(deps.store, deps.indexName, decoded.manifest, {
2982
2996
  manager: deps.manager,
2983
- vectorIndexes: deps.vectorIndexes
2997
+ vectorIndexes: deps.vectorIndexes,
2998
+ writeRecords: deps.writeRecords ?? null
2984
2999
  });
2985
3000
  return { checkpoint: await deps.replay(checkpoint), upgradedDocumentCount: null };
2986
3001
  }
@@ -3020,15 +3035,23 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
3020
3035
  if (manager === void 0 || current === void 0) {
3021
3036
  return;
3022
3037
  }
3038
+ const writeRecords = hooks.writeRecords ?? null;
3039
+ writeRecords?.dropIndex(indexName);
3023
3040
  const replayDeps = {
3024
3041
  manager,
3025
3042
  vectorFieldPaths: hooks.getVectorFieldPaths(indexName),
3026
- vectorIndexes: hooks.getVectorIndexes(indexName)
3043
+ vectorIndexes: hooks.getVectorIndexes(indexName),
3044
+ writeRecords
3027
3045
  };
3028
3046
  const replay = async (checkpoint) => {
3029
3047
  const replayed = [];
3030
3048
  for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
3031
3049
  const fromSeqNo = snapshotCheckpointFor(checkpoint, partitionId);
3050
+ writeRecords?.loadTermHistory(
3051
+ indexName,
3052
+ partitionId,
3053
+ checkpoint.find((entry) => entry.partitionId === partitionId)
3054
+ );
3032
3055
  const { highestSeqNo, highestPrimaryTerm } = await replayWal(
3033
3056
  directory,
3034
3057
  indexName,
@@ -3055,7 +3078,8 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
3055
3078
  compactionThreshold: deps.compactionThreshold,
3056
3079
  keepsUnlistedFiles: deps.holdsCheckpointFiles(indexName),
3057
3080
  replay,
3058
- onLayoutUpgrade: hooks.onLayoutUpgrade
3081
+ onLayoutUpgrade: hooks.onLayoutUpgrade,
3082
+ writeRecords
3059
3083
  });
3060
3084
  for (const { partitionId, lastSeqNo, primaryTerm } of recovered.checkpoint) {
3061
3085
  deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
@@ -3281,6 +3305,9 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3281
3305
  commitDeferral = defersCommit ?? null;
3282
3306
  },
3283
3307
  isolatePartitionFailures: partitionFailures.isolate,
3308
+ keepWriteRecordsWith(keeper) {
3309
+ hooks.writeRecords = keeper;
3310
+ },
3284
3311
  heldCheckpoints: () => heldCheckpoints.report(indexes),
3285
3312
  async commitPartition(indexName, partitionId) {
3286
3313
  const partition = indexes.get(indexName)?.partitions.get(partitionId);
@@ -3324,6 +3351,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3324
3351
  }
3325
3352
  },
3326
3353
  async removeIndex(indexName) {
3354
+ hooks.writeRecords?.dropIndex(indexName);
3327
3355
  const indexState = indexes.get(indexName);
3328
3356
  if (indexState !== void 0) {
3329
3357
  await drainIndexStateForUnload(indexState, metadataWrites.get(indexName), closeWriterReportingFailure);
@@ -1,9 +1,9 @@
1
1
  import { spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
2
- import { VECTOR_WORKER_COPIES_ALLOWED, openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, osqBitsOf, noBuildHistory, createOsqQuantizer, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, ordinalFilterValues, handOverBuildTurns, threadsHoldCurrentLayout, filterForOptions, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, OSQ4_MIN_DIMENSION, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, WORKER_COPY_MIN_VECTORS, allLiveDocIds, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, BLOCK_MAX_BYTES, growBufferTo, unplacedChunkCount, STORE_DOC_ID_BYTES, recalibrateFromStore, signatureOf, nativeStoreScratchBytes, noteFallback, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, QUERY_STAGED, createVectorBlock, ensureBlockSlots, createHNSWIndex, adoptGraph, topLayerInParts, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-Z2K5AGZL.mjs';
3
- export { releaseLocksHeldBy } from './chunk-Z2K5AGZL.mjs';
4
- import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-S7KNCVPU.mjs';
5
- import { MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, compareCodePoints, UNPLACED_FLAG_CHUNK_ORDINALS, MAX_VECTOR_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, resolveWorkerCount, resolveWorkerEntry, detectRuntime, noteThreadSlotLive, noteThreadSlotGone, WORKER_YOUNG_GENERATION_MB, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS } from './chunk-RYWURYTW.mjs';
6
- import { NarsilError, ErrorCodes, isNarsilError } from './chunk-EQ7BQ6VZ.mjs';
2
+ import { VECTOR_WORKER_COPIES_ALLOWED, openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, osqBitsOf, noBuildHistory, createOsqQuantizer, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, ordinalFilterValues, handOverBuildTurns, threadsHoldCurrentLayout, filterForOptions, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, OSQ4_MIN_DIMENSION, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, WORKER_COPY_MIN_VECTORS, allLiveDocIds, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, BLOCK_MAX_BYTES, growBufferTo, unplacedChunkCount, STORE_DOC_ID_BYTES, recalibrateFromStore, signatureOf, nativeStoreScratchBytes, noteFallback, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, QUERY_STAGED, createVectorBlock, ensureBlockSlots, createHNSWIndex, adoptGraph, topLayerInParts, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-OVY4RPDS.mjs';
3
+ export { releaseLocksHeldBy } from './chunk-OVY4RPDS.mjs';
4
+ import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-P7C2S5JX.mjs';
5
+ import { MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, compareCodePoints, UNPLACED_FLAG_CHUNK_ORDINALS, MAX_VECTOR_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, resolveWorkerCount, resolveWorkerEntry, detectRuntime, noteThreadSlotLive, noteThreadSlotGone, WORKER_YOUNG_GENERATION_MB, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS } from './chunk-2CBQ2PCC.mjs';
6
+ import { NarsilError, ErrorCodes, isNarsilError } from './chunk-LJS6PREA.mjs';
7
7
 
8
8
  // src/core/heap.ts
9
9
  function createMaxHeap(compare) {
@@ -18,7 +18,7 @@ function restoreDepsOf(core, host) {
18
18
  function queueRestore(core, host, restoresInFlight, indexName, data) {
19
19
  const previous = restoresInFlight.get(indexName) ?? Promise.resolve();
20
20
  const restored = previous.then(async () => {
21
- const { restoreFromSnapshot } = await import('./snapshot-3AZDQKNA.mjs');
21
+ const { restoreFromSnapshot } = await import('./snapshot-ISZMBDCN.mjs');
22
22
  await restoreFromSnapshot(indexName, data, restoreDepsOf(core, host));
23
23
  });
24
24
  const settled = restored.then(
@@ -1,5 +1,5 @@
1
1
  import { MIN_RERANK_TIMEOUT_MS, MAX_RERANK_TIMEOUT_MS, MIN_RERANK_WINDOW, MAX_RERANK_WINDOW, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS, DEFAULT_RERANK_WINDOW } from './chunk-JPTQZWJQ.mjs';
2
- import { NarsilError, ErrorCodes, MIN_HYBRID_DEPTH, MAX_HYBRID_DEPTH, HYBRID_RESCORE_WINDOW, DEFAULT_LIMIT, DEFAULT_OFFSET, RESULT_WINDOW } from './chunk-EQ7BQ6VZ.mjs';
2
+ import { NarsilError, ErrorCodes, MIN_HYBRID_DEPTH, MAX_HYBRID_DEPTH, HYBRID_RESCORE_WINDOW, DEFAULT_LIMIT, DEFAULT_OFFSET, RESULT_WINDOW } from './chunk-LJS6PREA.mjs';
3
3
 
4
4
  // src/search/hybrid-depth.ts
5
5
  function fixedHybridDepthOf(hybrid) {
@@ -1,11 +1,11 @@
1
- import { vectorsPerFile } from './chunk-Z2K5AGZL.mjs';
2
- import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-XHRHOH2P.mjs';
3
- import './chunk-3RXEGUGO.mjs';
4
- import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, prepareStoredSynonyms, rejectSnapshot, readHeapStatistics, readResidentMemory, extractVectorFieldsFromSchema, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-SCBBEQWD.mjs';
5
- import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-S7KNCVPU.mjs';
6
- import { MAX_FILE_BYTES } from './chunk-RYWURYTW.mjs';
1
+ import { vectorsPerFile } from './chunk-OVY4RPDS.mjs';
2
+ import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-YRYBTRWB.mjs';
3
+ import './chunk-O5IKTA27.mjs';
4
+ import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, prepareStoredSynonyms, rejectSnapshot, readHeapStatistics, readResidentMemory, extractVectorFieldsFromSchema, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-UR5MNN53.mjs';
5
+ import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-P7C2S5JX.mjs';
6
+ import { MAX_FILE_BYTES } from './chunk-2CBQ2PCC.mjs';
7
7
  import './chunk-JPTQZWJQ.mjs';
8
- import { NarsilError, ErrorCodes } from './chunk-EQ7BQ6VZ.mjs';
8
+ import { NarsilError, ErrorCodes } from './chunk-LJS6PREA.mjs';
9
9
  import { decode } from '@msgpack/msgpack';
10
10
 
11
11
  // src/persistence/archive/memory-store.ts
@@ -79,12 +79,24 @@ function partitionVectorPlan(vectorIndex, docIds) {
79
79
  }
80
80
  function capture(input) {
81
81
  const partitionIds = input.partitionId === null ? Array.from({ length: input.manager.partitionCount }, (_, partitionId) => partitionId) : [input.partitionId];
82
- const targets = partitionIds.map((partitionId) => ({
83
- partitionId,
84
- lastSeqNo: input.appliedSeqNo(partitionId),
85
- primaryTerm: SINGLE_NODE_PRIMARY_TERM
86
- }));
87
- const contents = partitionIds.map((partitionId) => partitionContentsOf(input.manager, partitionId));
82
+ const { indexName } = input;
83
+ const writeRecords = input.writeRecords ?? null;
84
+ const targets = partitionIds.map((partitionId) => {
85
+ const target = {
86
+ partitionId,
87
+ lastSeqNo: input.appliedSeqNo(partitionId),
88
+ primaryTerm: SINGLE_NODE_PRIMARY_TERM
89
+ };
90
+ if (writeRecords !== null) target.termHistory = writeRecords.termHistoryOf(indexName, partitionId, target.lastSeqNo);
91
+ return target;
92
+ });
93
+ const contents = partitionIds.map(
94
+ (partitionId) => partitionContentsOf(
95
+ input.manager,
96
+ partitionId,
97
+ writeRecords === null ? void 0 : (segments) => writeRecords.captureSegments(indexName, partitionId, segments)
98
+ )
99
+ );
88
100
  const plans = /* @__PURE__ */ new Map();
89
101
  const partitionDocIds = input.partitionId === null ? null : [...input.manager.getPartition(input.partitionId).docIds()];
90
102
  for (const [fieldPath, vectorIndex] of input.vectorIndexes) {
@@ -107,7 +119,8 @@ async function buildArchiveFromMemory(input) {
107
119
  targets: captured.targets,
108
120
  compactionThreshold: DEFAULT_COMPACTION_THRESHOLD,
109
121
  prior: null,
110
- wholePartitions: captured.contents
122
+ wholePartitions: captured.contents,
123
+ keepsWriteRecords: input.writeRecords != null
111
124
  });
112
125
  const vectors = await writeLiveVectors({ store, indexName: input.indexName, plans: captured.plans, priorVectors: [] });
113
126
  const refs = vectors.refs;
@@ -1,9 +1,9 @@
1
- import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-3RXEGUGO.mjs';
2
- import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-SCBBEQWD.mjs';
3
- import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-S7KNCVPU.mjs';
4
- import './chunk-RYWURYTW.mjs';
1
+ import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-O5IKTA27.mjs';
2
+ import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-UR5MNN53.mjs';
3
+ import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-P7C2S5JX.mjs';
4
+ import './chunk-2CBQ2PCC.mjs';
5
5
  import './chunk-JPTQZWJQ.mjs';
6
- import { EXPORT_CHUNK_BYTES, NarsilError, ErrorCodes } from './chunk-EQ7BQ6VZ.mjs';
6
+ import { EXPORT_CHUNK_BYTES, NarsilError, ErrorCodes } from './chunk-LJS6PREA.mjs';
7
7
  import { decode } from '@msgpack/msgpack';
8
8
 
9
9
  // src/engine/export/json-line.ts