@delali/narsil 0.3.1-next.131 → 0.3.1-next.141

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/dist/adapters/filesystem.mjs +1 -1
  2. package/dist/adapters/indexeddb.mjs +1 -1
  3. package/dist/analysis-change-HQUT2Z3I.mjs +43 -0
  4. package/dist/analysis-change-entry-TZTG4AZB.mjs +1 -0
  5. package/dist/browser/analysis-change-JPV775LR.mjs +900 -0
  6. package/dist/browser/analysis-change-entry-7HR4LWL3.mjs +1 -0
  7. package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
  8. package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
  9. package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-5DB2VDGI.mjs} +1149 -1853
  10. package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
  11. package/dist/browser/chunk-CCOAOO73.mjs +86 -0
  12. package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-D33A5VMU.mjs} +16 -16
  13. package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-HKX2G6WH.mjs} +1520 -2159
  14. package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-KTOK275N.mjs} +606 -830
  15. package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-LVY2LI35.mjs} +2 -2
  16. package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
  17. package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
  18. package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
  19. package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
  20. package/dist/browser/chunk-R4WUDZQV.mjs +176 -0
  21. package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-TCLEOZ4P.mjs} +3 -3
  22. package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
  23. package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
  24. package/dist/browser/chunk-XNWPP5NS.mjs +21 -0
  25. package/dist/browser/chunk-ZEV26AQ6.mjs +270 -0
  26. package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
  27. package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-SNOWIT6Z.mjs} +370 -95
  28. package/dist/browser/import-TLB2A5LJ.mjs +9 -0
  29. package/dist/browser/module-BUFNZMEU.mjs +12 -0
  30. package/dist/browser/{module-4HIMOQM5.mjs → module-BVP4TKZU.mjs} +397 -109
  31. package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
  32. package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-ONYNYUDE.mjs} +8 -2
  33. package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
  34. package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-QXBKHJAG.mjs} +135 -183
  35. package/dist/browser/{write-YIC2MKRY.mjs → write-WHFBDWQS.mjs} +64 -90
  36. package/dist/{capabilities-B6WxrWxt.d.ts → capabilities-BCdUwK_L.d.ts} +10 -3
  37. package/dist/chunk-46NZOPHS.mjs +21 -0
  38. package/dist/{chunk-JT3YZGCE.mjs → chunk-5WYVUPGD.mjs} +3 -188
  39. package/dist/chunk-6SNBQESJ.mjs +214 -0
  40. package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
  41. package/dist/chunk-7AUASRE7.mjs +653 -0
  42. package/dist/{chunk-NMXRXFGZ.mjs → chunk-AST3SHFA.mjs} +27 -25
  43. package/dist/chunk-BOJZGUS6.mjs +177 -0
  44. package/dist/{chunk-T4OFG4IQ.mjs → chunk-BRUUBXCB.mjs} +92 -17
  45. package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
  46. package/dist/chunk-CZQ33DLW.mjs +78 -0
  47. package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
  48. package/dist/{chunk-EUCX5K74.mjs → chunk-DQNKVOJ2.mjs} +65 -65
  49. package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
  50. package/dist/chunk-FOCV4CBZ.mjs +87 -0
  51. package/dist/{chunk-HCGJKDZV.mjs → chunk-GLZHNSGB.mjs} +1 -1
  52. package/dist/chunk-GWOS5DYC.mjs +2637 -0
  53. package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
  54. package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
  55. package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
  56. package/dist/{chunk-TGHNXN7W.mjs → chunk-IO5F7ZBE.mjs} +261 -948
  57. package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
  58. package/dist/{chunk-7OIQ2SIO.mjs → chunk-KEIYVZUX.mjs} +5 -5
  59. package/dist/{chunk-PU6SKH7O.mjs → chunk-KGOXHLLZ.mjs} +7 -3
  60. package/dist/{chunk-WVRDRLIC.mjs → chunk-KZL72VTC.mjs} +70 -18
  61. package/dist/chunk-L76W3IZJ.mjs +193 -0
  62. package/dist/{chunk-2JWNJSNE.mjs → chunk-LA4WNDHL.mjs} +88 -801
  63. package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
  64. package/dist/{chunk-JJ6EUMXB.mjs → chunk-NJJCPFXN.mjs} +2 -2
  65. package/dist/chunk-NN33CZRE.mjs +86 -0
  66. package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
  67. package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
  68. package/dist/chunk-NRUXBYPH.mjs +121 -0
  69. package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
  70. package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
  71. package/dist/{chunk-F6AAXI7T.mjs → chunk-RHKRTYGO.mjs} +13 -4
  72. package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
  73. package/dist/{chunk-RCMNL7RI.mjs → chunk-SBMAWTRG.mjs} +41 -4
  74. package/dist/chunk-SRJBFG2T.mjs +881 -0
  75. package/dist/{chunk-X5CNUKD5.mjs → chunk-TNBIK2LQ.mjs} +1663 -1652
  76. package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
  77. package/dist/chunk-UUZCQQG2.mjs +209 -0
  78. package/dist/{chunk-6M6SDH3S.mjs → chunk-VEFN6X6Q.mjs} +8 -8
  79. package/dist/chunk-VJEGOELZ.mjs +177 -0
  80. package/dist/chunk-W2COJ5AH.mjs +148 -0
  81. package/dist/{chunk-RWESFSJI.mjs → chunk-WJGO5PGV.mjs} +317 -174
  82. package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
  83. package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
  84. package/dist/{chunk-Y4BVTQ53.mjs → chunk-XZ3SRNDJ.mjs} +25 -64
  85. package/dist/{chunk-K77FJXKS.mjs → chunk-YHUTF6EA.mjs} +474 -93
  86. package/dist/{chunk-OGLV4I3L.mjs → chunk-YLNSJQLX.mjs} +12 -7
  87. package/dist/{chunk-SOHMMNCH.mjs → chunk-Z4HFMYNX.mjs} +32 -27
  88. package/dist/{chunk-5W7AEVTH.mjs → chunk-ZMBFWQXN.mjs} +2 -2
  89. package/dist/{chunk-2MKX2SCO.mjs → chunk-ZPWS2SKJ.mjs} +297 -274
  90. package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
  91. package/dist/{client-gNAfaHcE.d.ts → client-B12YFALP.d.ts} +18 -3
  92. package/dist/client.d.ts +7 -7
  93. package/dist/client.mjs +11 -1
  94. package/dist/cold-module-4X7WJCFM.mjs +783 -0
  95. package/dist/{config-D1cDv3sE.d.ts → config-BhZxIeb1.d.ts} +10 -1
  96. package/dist/{config-CmYJO34X.d.ts → config-CIG5yDyg.d.ts} +2 -2
  97. package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-7XOQ7VQP.mjs} +14 -14
  98. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  99. package/dist/distribution/coordinator/etcd.mjs +1 -1
  100. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  101. package/dist/distribution/transport/grpc.mjs +1 -1
  102. package/dist/distribution.d.ts +24 -8
  103. package/dist/distribution.mjs +982 -211
  104. package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-324D2DDD.mjs} +362 -94
  105. package/dist/embeddings/openai.mjs +2 -2
  106. package/dist/{engine-m_Ddr1bz.d.ts → engine-CtzbHOmG.d.ts} +64 -7
  107. package/dist/import-OOBUW3RR.mjs +44 -0
  108. package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
  109. package/dist/index.browser.mjs +5 -5
  110. package/dist/index.d.ts +6 -6
  111. package/dist/index.mjs +22 -22
  112. package/dist/{long-query-thread-BSL25PD6.mjs → long-query-thread-4SW7RWDE.mjs} +15 -15
  113. package/dist/{module-2E4XYSGT.mjs → module-DXSJ5LT2.mjs} +378 -87
  114. package/dist/module-IYPLWHGC.mjs +48 -0
  115. package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
  116. package/dist/partial-words.mjs +5 -5
  117. package/dist/pattern.mjs +3 -3
  118. package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
  119. package/dist/persistence/durability/checkpoint-worker.mjs +28 -26
  120. package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
  121. package/dist/react.d.ts +9 -9
  122. package/dist/react.mjs +4 -0
  123. package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
  124. package/dist/rerankers/http.mjs +2 -2
  125. package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-FEYAC4SB.mjs} +8 -2
  126. package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
  127. package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
  128. package/dist/server.d.ts +12 -10
  129. package/dist/server.mjs +54 -21
  130. package/dist/{snapshot-DOQLGGEX.mjs → snapshot-CGM65VCD.mjs} +101 -197
  131. package/dist/synonyms.mjs +6 -4
  132. package/dist/{tasks-Co8HfRD7.d.ts → tasks-CjRJRGTb.d.ts} +26 -8
  133. package/dist/{thread-SKO7CN3R.mjs → thread-KGNAIKPD.mjs} +14 -14
  134. package/dist/vector/search-worker.d.ts +1 -1
  135. package/dist/vector/search-worker.mjs +6 -5
  136. package/dist/workers/entry.mjs +8 -8
  137. package/dist/{write-ZKR7SNRT.mjs → write-2DZPEQL2.mjs} +82 -107
  138. package/openapi.json +114 -6
  139. package/package.json +12 -8
@@ -1,8 +1,9 @@
1
- import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes, uint64Bytes } from './chunk-M5UAAXGU.mjs';
2
- import { VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, rulesOfStoredSynonyms, flattenSchema, packChecksummedEnvelope, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, fnv1a, selectionKeeps, decodeMapWithoutFields, prepareDocumentVectors, messagePackUpperBoundBytes, segmentArraysOf, mergeSegmentContents, resolveIndexAnalysis } from './chunk-2Y3CO4EQ.mjs';
3
- import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-D7WQF6DV.mjs';
4
- import { compareCodePoints, MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-AIIJDWMG.mjs';
5
- import { NarsilError, ErrorCodes } from './chunk-UDGB2MWT.mjs';
1
+ import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, encodeSegmentManifest, manifestKey, manifestReferencedKeys, segmentsPrefix, snapshotBundleKey, readListedFile, requireListedChecksum, vectorFileKey, DEFAULT_COMPACTION_THRESHOLD, writeSegmentsOf, readSegmentFile, graphPartKey, requireListedSize, SEGMENT_PAGE_BYTES, STABLE_HEAD_ATTEMPTS, RECORD_PAGE_BYTES, MERGED_SEGMENT_GROWTH_ALLOWANCE, MERGED_SEGMENT_TARGET_BYTES, COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT, SEGMENT_FILE_FRAMING_SLACK_BYTES } from './chunk-MVM33ZUL.mjs';
2
+ import { restoreVectorFields } from './chunk-TCLEOZ4P.mjs';
3
+ import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-LVY2LI35.mjs';
4
+ import { VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, rulesOfStoredSynonyms, flattenSchema, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, packChecksummedEnvelope, packEnvelopeWithChecksum, checksumOfChunks, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, decodeMapWithoutFields, prepareDocumentVectors, resolveIndexAnalysis } from './chunk-5DB2VDGI.mjs';
5
+ import { compareCodePoints, MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-D33A5VMU.mjs';
6
+ import { NarsilError, ErrorCodes } from './chunk-35B42UFM.mjs';
6
7
  import { decode, encode } from '@msgpack/msgpack';
7
8
 
8
9
  // src/engine/durability-metadata.ts
@@ -42,6 +43,12 @@ function metadataFromConfig(indexName, config, partitionCount, documentCount) {
42
43
  if (config.stopWordList !== void 0) {
43
44
  metadata.stopWordList = config.stopWordList;
44
45
  }
46
+ if (config.analysisGeneration !== void 0) {
47
+ metadata.analysisGeneration = config.analysisGeneration;
48
+ }
49
+ if (config.pendingAnalysis !== void 0) {
50
+ metadata.pendingAnalysis = config.pendingAnalysis;
51
+ }
45
52
  if (config.partitionLimits !== void 0) {
46
53
  metadata.partitionLimits = config.partitionLimits;
47
54
  }
@@ -51,6 +58,9 @@ function metadataFromConfig(indexName, config, partitionCount, documentCount) {
51
58
  if (config.trackPositions !== void 0) {
52
59
  metadata.trackPositions = config.trackPositions;
53
60
  }
61
+ if (config.textStorage !== void 0) {
62
+ metadata.textStorage = config.textStorage;
63
+ }
54
64
  if (config.strict !== void 0) {
55
65
  metadata.strict = config.strict;
56
66
  }
@@ -96,9 +106,12 @@ function metadataConfigOf(entry) {
96
106
  ...typeof entry.config.tokenizer === "string" ? { tokenizer: entry.config.tokenizer } : {},
97
107
  ...typeof entry.config.stopWords === "string" ? { stopWords: entry.config.stopWords } : {},
98
108
  ...entry.config.stopWords instanceof Set ? { stopWordList: [...entry.config.stopWords].sort(compareCodePoints) } : {},
109
+ ...entry.analysisGeneration > 0 ? { analysisGeneration: entry.analysisGeneration } : {},
110
+ ...entry.pendingAnalysis !== null ? { pendingAnalysis: entry.pendingAnalysis } : {},
99
111
  ...entry.config.partitions !== void 0 ? { partitionLimits: entry.config.partitions } : {},
100
112
  ...entry.config.defaultScoring !== void 0 ? { defaultScoring: entry.config.defaultScoring } : {},
101
113
  ...entry.config.trackPositions !== void 0 ? { trackPositions: entry.config.trackPositions } : {},
114
+ ...entry.config.textStorage !== void 0 ? { textStorage: entry.config.textStorage } : {},
102
115
  ...entry.config.strict !== void 0 ? { strict: entry.config.strict } : {},
103
116
  ...entry.config.required !== void 0 ? { required: entry.config.required } : {},
104
117
  ...entry.config.vectorPromotion !== void 0 ? { vectorPromotion: entry.config.vectorPromotion } : {},
@@ -106,30 +119,42 @@ function metadataConfigOf(entry) {
106
119
  ...entry.synonyms !== null ? { synonyms: rulesOfStoredSynonyms(entry.synonyms) } : {}
107
120
  };
108
121
  }
122
+ function durableMetadataConfigOf(entry) {
123
+ const terms = metadataConfigOf(entry);
124
+ const durable = entry.durableAnalysis;
125
+ if (durable === null) return terms;
126
+ const {
127
+ language,
128
+ tokenizer,
129
+ stopWords,
130
+ stopWordList,
131
+ analysisRevision: _analysisRevision,
132
+ arrayPositionGap: _arrayPositionGap,
133
+ analysisGeneration: _analysisGeneration,
134
+ ...rest
135
+ } = terms;
136
+ return {
137
+ ...rest,
138
+ language: durable.language,
139
+ ...durable.tokenizer !== void 0 ? { tokenizer: durable.tokenizer } : {},
140
+ ...durable.stopWords !== void 0 ? { stopWords: durable.stopWords } : {},
141
+ ...durable.stopWordList !== void 0 ? { stopWordList: durable.stopWordList } : {},
142
+ ...durable.analysisRevision !== null ? { analysisRevision: durable.analysisRevision } : {},
143
+ ...durable.arrayPositionGap !== null ? { arrayPositionGap: durable.arrayPositionGap } : {},
144
+ ...durable.generation > 0 ? { analysisGeneration: durable.generation } : {},
145
+ pendingAnalysis: {
146
+ generation: entry.analysisGeneration,
147
+ language,
148
+ ...tokenizer !== void 0 ? { tokenizer } : {},
149
+ ...stopWords !== void 0 ? { stopWords } : {},
150
+ ...stopWordList !== void 0 ? { stopWordList } : {}
151
+ }
152
+ };
153
+ }
109
154
  function indexMetadataOf(indexName, entry, partitionCount) {
110
155
  return metadataFromConfig(indexName, metadataConfigOf(entry), partitionCount);
111
156
  }
112
157
 
113
- // src/persistence/durability/constants.ts
114
- var DEFAULT_CHECKPOINT_INTERVAL_MS = 3e5;
115
- var DEFAULT_CHECKPOINT_MUTATION_THRESHOLD = 1e5;
116
- var DEFAULT_ASYNC_FLUSH_INTERVAL_MS = 1e3;
117
- var DIRECTORY_IDENTITY_CHECK_INTERVAL_MS = 1e3;
118
- var DIRECTORY_LOCK_ATTEMPTS = 3;
119
- var DIRECTORY_LOCK_SETTLE_MS = 50;
120
- var DEFAULT_SEGMENT_MAX_BYTES = 67108864;
121
- var DEFAULT_COMPACTION_THRESHOLD = 12;
122
- var COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT = 2e5;
123
- var SEGMENT_FILE_FRAMING_SLACK_BYTES = 65536;
124
- var SEGMENT_TOKEN_TABLE_SHARE_OF_FILE = 0.25;
125
- var WHOLE_PARTITION_MIN_CHANGED_SHARE = 0.5;
126
- var WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT = 1e5;
127
- var LOG_RECORDS_PER_CHECKPOINT = 1e5;
128
- var WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES = 2147483648;
129
- var CHECKPOINT_WORKER_TIMEOUT_MS = 6e5;
130
- var CHECKPOINT_WORKER_IDLE_MS = 5e3;
131
- var CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS = 50;
132
-
133
158
  // src/serialization/stored-vector-values.ts
134
159
  function dropStoredVectorValues(documents, schema) {
135
160
  const vectorFieldPaths = [];
@@ -337,6 +362,19 @@ function metadataToWire(meta) {
337
362
  if (meta.stopWordList !== void 0) {
338
363
  wire.stop_word_list = meta.stopWordList;
339
364
  }
365
+ if (meta.analysisGeneration !== void 0) {
366
+ wire.analysis_generation = meta.analysisGeneration;
367
+ }
368
+ if (meta.pendingAnalysis !== void 0) {
369
+ const pending = meta.pendingAnalysis;
370
+ wire.pending_analysis = {
371
+ generation: pending.generation,
372
+ language: pending.language,
373
+ ...pending.tokenizer !== void 0 ? { tokenizer: pending.tokenizer } : {},
374
+ ...pending.stopWords !== void 0 ? { stop_words: pending.stopWords } : {},
375
+ ...pending.stopWordList !== void 0 ? { stop_word_list: pending.stopWordList } : {}
376
+ };
377
+ }
340
378
  if (meta.partitionLimits !== void 0) {
341
379
  wire.partition_limits = {
342
380
  ...meta.partitionLimits.maxDocsPerPartition !== void 0 ? { max_docs_per_partition: meta.partitionLimits.maxDocsPerPartition } : {},
@@ -350,6 +388,9 @@ function metadataToWire(meta) {
350
388
  if (meta.trackPositions !== void 0) {
351
389
  wire.track_positions = meta.trackPositions;
352
390
  }
391
+ if (meta.textStorage !== void 0) {
392
+ wire.text_storage = meta.textStorage;
393
+ }
353
394
  if (meta.strict !== void 0) {
354
395
  wire.strict = meta.strict;
355
396
  }
@@ -432,6 +473,19 @@ function wireToMetadata(raw) {
432
473
  if (Array.isArray(raw.stop_word_list) && raw.stop_word_list.every((word) => typeof word === "string")) {
433
474
  meta.stopWordList = raw.stop_word_list;
434
475
  }
476
+ if (isPositiveInteger(raw.analysis_generation)) {
477
+ meta.analysisGeneration = raw.analysis_generation;
478
+ }
479
+ const pending = raw.pending_analysis;
480
+ if (pending && isPositiveInteger(pending.generation) && typeof pending.language === "string") {
481
+ meta.pendingAnalysis = {
482
+ generation: pending.generation,
483
+ language: pending.language,
484
+ ...typeof pending.tokenizer === "string" ? { tokenizer: pending.tokenizer } : {},
485
+ ...typeof pending.stop_words === "string" ? { stopWords: pending.stop_words } : {},
486
+ ...isStringList(pending.stop_word_list) ? { stopWordList: pending.stop_word_list } : {}
487
+ };
488
+ }
435
489
  if (raw.partition_limits && typeof raw.partition_limits === "object") {
436
490
  const limits = {
437
491
  ...isPositiveInteger(raw.partition_limits.max_docs_per_partition) ? { maxDocsPerPartition: raw.partition_limits.max_docs_per_partition } : {},
@@ -448,6 +502,14 @@ function wireToMetadata(raw) {
448
502
  if (typeof raw.track_positions === "boolean") {
449
503
  meta.trackPositions = raw.track_positions;
450
504
  }
505
+ if (raw.text_storage !== void 0) {
506
+ if (raw.text_storage !== "memory" && raw.text_storage !== "disk") {
507
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, "text_storage must be 'memory' or 'disk'", {
508
+ textStorage: raw.text_storage
509
+ });
510
+ }
511
+ meta.textStorage = raw.text_storage;
512
+ }
451
513
  if (typeof raw.strict === "boolean") {
452
514
  meta.strict = raw.strict;
453
515
  }
@@ -524,6 +586,9 @@ function wireToMetadata(raw) {
524
586
  function isNonNegativeInteger(value) {
525
587
  return typeof value === "number" && Number.isInteger(value) && value >= 0;
526
588
  }
589
+ function isStringList(value) {
590
+ return Array.isArray(value) && value.every((word) => typeof word === "string");
591
+ }
527
592
  function isPositiveInteger(value) {
528
593
  return typeof value === "number" && Number.isInteger(value) && value >= 1;
529
594
  }
@@ -595,7 +660,7 @@ async function packEnvelope(payloadBytes, options) {
595
660
  const checksum = flags.checksumPresent ? crc32(finalPayload) : 0;
596
661
  const [major, minor, patch] = engineVersionParts();
597
662
  const envelopeFormatVersion = options.envelopeFormatVersion ?? CURRENT_ENVELOPE_VERSION;
598
- const header2 = {
663
+ const header = {
599
664
  magic: "NRSL",
600
665
  envelopeFormatVersion,
601
666
  engineVersionMajor: major,
@@ -606,64 +671,64 @@ async function packEnvelope(payloadBytes, options) {
606
671
  checksum,
607
672
  reserved: new Uint8Array(14)
608
673
  };
609
- const headerBytes = writeHeader(header2);
674
+ const headerBytes = writeHeader(header);
610
675
  const envelope = new Uint8Array(HEADER_SIZE + finalPayload.length);
611
676
  envelope.set(headerBytes, 0);
612
677
  envelope.set(finalPayload, HEADER_SIZE);
613
678
  return envelope;
614
679
  }
615
680
  async function unpackEnvelope(data, maxAcceptedVersion = CURRENT_ENVELOPE_VERSION) {
616
- const header2 = readHeader(data);
617
- if (header2.envelopeFormatVersion > maxAcceptedVersion) {
681
+ const header = readHeader(data);
682
+ if (header.envelopeFormatVersion > maxAcceptedVersion) {
618
683
  throw new NarsilError(
619
684
  ErrorCodes.ENVELOPE_VERSION_MISMATCH,
620
- `This data holds Narsil envelope format v${header2.envelopeFormatVersion}, while this build of @delali/narsil reads up to v${maxAcceptedVersion}, so upgrade the package to read it.`
685
+ `This data holds Narsil envelope format v${header.envelopeFormatVersion}, while this build of @delali/narsil reads up to v${maxAcceptedVersion}, so upgrade the package to read it.`
621
686
  );
622
687
  }
623
- if (header2.flags.encryptionEnabled) {
688
+ if (header.flags.encryptionEnabled) {
624
689
  throw new NarsilError(
625
690
  ErrorCodes.ENVELOPE_VERSION_MISMATCH,
626
691
  "Encrypted envelopes are not supported in this version of Narsil"
627
692
  );
628
693
  }
629
- const compAlgo = header2.flags.compressionAlgorithm;
630
- if (header2.flags.compressionEnabled && compAlgo !== "gzip" && compAlgo !== "none") {
694
+ const compAlgo = header.flags.compressionAlgorithm;
695
+ if (header.flags.compressionEnabled && compAlgo !== "gzip" && compAlgo !== "none") {
631
696
  throw new NarsilError(
632
697
  ErrorCodes.ENVELOPE_VERSION_MISMATCH,
633
698
  `Compression algorithm "${compAlgo}" is not supported in this version of Narsil`
634
699
  );
635
700
  }
636
701
  const availablePayload = data.length - HEADER_SIZE;
637
- if (availablePayload < header2.payloadLength) {
702
+ if (availablePayload < header.payloadLength) {
638
703
  throw new NarsilError(
639
704
  ErrorCodes.PERSISTENCE_LOAD_FAILED,
640
- `Envelope data is truncated: header declares ${header2.payloadLength} bytes of payload but only ${availablePayload} bytes are available`
705
+ `Envelope data is truncated: header declares ${header.payloadLength} bytes of payload but only ${availablePayload} bytes are available`
641
706
  );
642
707
  }
643
- let payloadBytes = new Uint8Array(data.slice(HEADER_SIZE, HEADER_SIZE + header2.payloadLength));
644
- if (header2.flags.checksumPresent) {
708
+ let payloadBytes = new Uint8Array(data.slice(HEADER_SIZE, HEADER_SIZE + header.payloadLength));
709
+ if (header.flags.checksumPresent) {
645
710
  const computed = crc32(payloadBytes);
646
- if (computed !== header2.checksum) {
711
+ if (computed !== header.checksum) {
647
712
  throw new NarsilError(ErrorCodes.PERSISTENCE_CRC_MISMATCH, "CRC32 checksum mismatch: data may be corrupted", {
648
- expected: header2.checksum,
713
+ expected: header.checksum,
649
714
  computed
650
715
  });
651
716
  }
652
717
  }
653
- if (header2.flags.compressionEnabled) {
718
+ if (header.flags.compressionEnabled) {
654
719
  payloadBytes = await decompressGzip(payloadBytes);
655
720
  }
656
- return { header: header2, payloadBytes };
721
+ return { header, payloadBytes };
657
722
  }
658
723
  async function unpackEnvelopeBytes(data) {
659
724
  return unpackEnvelope(data, SNAPSHOT_ENVELOPE_VERSION);
660
725
  }
661
726
  async function unpackIndexSnapshotEnvelope(data) {
662
- const { header: header2, payloadBytes } = await unpackEnvelope(data, INDEX_SNAPSHOT_ENVELOPE_VERSION);
663
- if (header2.envelopeFormatVersion !== INDEX_SNAPSHOT_ENVELOPE_VERSION) {
727
+ const { header, payloadBytes } = await unpackEnvelope(data, INDEX_SNAPSHOT_ENVELOPE_VERSION);
728
+ if (header.envelopeFormatVersion !== INDEX_SNAPSHOT_ENVELOPE_VERSION) {
664
729
  throw new NarsilError(
665
730
  ErrorCodes.ENVELOPE_VERSION_MISMATCH,
666
- `Expected an index snapshot envelope of format v${INDEX_SNAPSHOT_ENVELOPE_VERSION}, received format v${header2.envelopeFormatVersion}`
731
+ `Expected an index snapshot envelope of format v${INDEX_SNAPSHOT_ENVELOPE_VERSION}, received format v${header.envelopeFormatVersion}`
667
732
  );
668
733
  }
669
734
  return payloadBytes;
@@ -673,126 +738,9 @@ async function writeMetadataEnvelope(meta, options = {}) {
673
738
  return packEnvelope(payloadBytes, options);
674
739
  }
675
740
  async function readMetadataEnvelope(data) {
676
- const { header: header2, payloadBytes } = await unpackEnvelope(data);
741
+ const { header, payloadBytes } = await unpackEnvelope(data);
677
742
  const metadata = deserializeMetadata(payloadBytes);
678
- return { header: header2, metadata };
679
- }
680
-
681
- // src/persistence/durability/segment/layout.ts
682
- var SEGMENT_ID_WIDTH = 16;
683
- function manifestKey(indexName) {
684
- return `${indexName}/manifest`;
685
- }
686
- function snapshotBundleKey(indexName) {
687
- return `${indexName}/snapshot`;
688
- }
689
- function segmentPrefix(indexName, partitionId) {
690
- return `${indexName}/segments/${partitionId}/`;
691
- }
692
- function segmentKey(indexName, partitionId, segmentId) {
693
- return `${segmentPrefix(indexName, partitionId)}s${formatSegmentId(segmentId)}`;
694
- }
695
- function segmentsPrefix(indexName) {
696
- return `${indexName}/segments/`;
697
- }
698
- function vectorFileKey(indexName, fieldPath, fileId) {
699
- return `${segmentsPrefix(indexName)}vec-${encodeFieldPath(fieldPath)}-f${formatSegmentId(fileId)}`;
700
- }
701
- function requireGeneration(generation) {
702
- if (Number.isSafeInteger(generation) && generation > 0) return;
703
- throw new NarsilError(
704
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
705
- `Vector graph generation ${generation} must be a positive integer`,
706
- { generation }
707
- );
708
- }
709
- function graphPartKey(indexName, fieldPath, generation, part) {
710
- requireGeneration(generation);
711
- return `${segmentsPrefix(indexName)}vec-${encodeFieldPath(fieldPath)}-graph-g${generation}-p${formatSegmentId(part)}`;
712
- }
713
- function formatSegmentId(segmentId) {
714
- if (!Number.isInteger(segmentId) || segmentId < 0) {
715
- throw new NarsilError(
716
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
717
- `Segment id ${segmentId} must be a non-negative integer`,
718
- { segmentId }
719
- );
720
- }
721
- return segmentId.toString().padStart(SEGMENT_ID_WIDTH, "0");
722
- }
723
- var FIELD_PATH_PATTERN = /^[A-Za-z0-9_.]+$/;
724
- function encodeFieldPath(fieldPath) {
725
- if (!FIELD_PATH_PATTERN.test(fieldPath)) {
726
- throw new NarsilError(
727
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
728
- `Vector field path "${fieldPath}" contains characters that cannot be encoded in a segment key`,
729
- { fieldPath }
730
- );
731
- }
732
- const fingerprint = fnv1a(fieldPath).toString(36);
733
- return `${fieldPath.replace(/\./g, "_")}-${fingerprint}`;
734
- }
735
- var SEGMENT_MANIFEST_VERSION = 7;
736
- var EARLIER_SEGMENT_MANIFEST_VERSION = 6;
737
- var MANIFEST_ENVELOPE_VERSION = 2;
738
- var MAX_SEGMENTS_PER_PARTITION = 65536;
739
- var MAX_SEGMENT_ID = 281474976710655;
740
- function fileRefOnTheWire(ref) {
741
- return { key: ref.key, bytes: ref.bytes, checksum: ref.checksum };
742
- }
743
- function encodeSegmentManifest(manifest) {
744
- const payload = encode({
745
- version: SEGMENT_MANIFEST_VERSION,
746
- schema: manifest.schema,
747
- language: manifest.language,
748
- checkpoint: manifest.checkpoint.map((c) => ({
749
- partitionId: c.partitionId,
750
- lastSeqNo: c.lastSeqNo,
751
- primaryTerm: c.primaryTerm,
752
- ...c.termHistory === void 0 ? {} : {
753
- termHistory: c.termHistory.map((start) => ({ primaryTerm: start.primaryTerm, firstSeqNo: start.firstSeqNo }))
754
- }
755
- })),
756
- partitions: manifest.partitions.map((p) => ({
757
- partitionId: p.partitionId,
758
- nextSegmentId: p.nextSegmentId,
759
- segments: p.segments.map((s) => ({
760
- id: s.id,
761
- key: s.key,
762
- docCount: s.docCount,
763
- tombstoneCount: s.tombstoneCount,
764
- bytes: s.bytes,
765
- checksum: s.checksum
766
- }))
767
- })),
768
- vectors: manifest.vectors.map((v) => ({
769
- fieldPath: v.fieldPath,
770
- nextFileId: v.nextFileId,
771
- files: v.files.map((f) => ({ id: f.id, ...fileRefOnTheWire(f), count: f.count, dead: f.dead })),
772
- graphGeneration: v.graphGeneration,
773
- graphParts: v.graphParts.map(fileRefOnTheWire)
774
- }))
775
- });
776
- return packChecksummedEnvelope([payload], MANIFEST_ENVELOPE_VERSION, "segment manifest");
777
- }
778
- function manifestReferencedKeys(manifest) {
779
- const keys = /* @__PURE__ */ new Set();
780
- for (const partition of manifest.partitions) {
781
- for (const segment of partition.segments) {
782
- keys.add(segment.key);
783
- }
784
- }
785
- for (const vector of manifest.vectors) {
786
- for (const file of vector.files) keys.add(file.key);
787
- for (const part of vector.graphParts) keys.add(part.key);
788
- }
789
- return keys;
790
- }
791
- function manifestFileRefs(manifest) {
792
- const refs = [];
793
- for (const partition of manifest.partitions) refs.push(...partition.segments);
794
- for (const vector of manifest.vectors) refs.push(...vector.files, ...vector.graphParts);
795
- return refs;
743
+ return { header, metadata };
796
744
  }
797
745
 
798
746
  // src/persistence/durability/segment/manifest-fields.ts
@@ -823,7 +771,20 @@ function readFileRef(raw, expectedKey, what) {
823
771
  invalidManifest(`gives the ${what} "${expectedKey}" an invalid length`, { key: expectedKey, bytes });
824
772
  }
825
773
  if (!isUint32(raw.checksum)) invalidManifest(`gives the ${what} "${expectedKey}" an invalid checksum`);
826
- return { key: expectedKey, bytes, checksum: raw.checksum };
774
+ if (raw.headOffset === void 0) return { key: expectedKey, bytes, checksum: raw.checksum };
775
+ const { headOffset, headLength, headChecksum } = raw;
776
+ if (!isSequenceNumber(headOffset) || !isUint32(headLength) || !isUint32(headChecksum)) {
777
+ invalidManifest(`gives the ${what} "${expectedKey}" an invalid head`, { key: expectedKey });
778
+ }
779
+ if (headOffset + headLength > bytes - HEADER_SIZE) {
780
+ invalidManifest(`ends the head of the ${what} "${expectedKey}" outside its payload`, {
781
+ key: expectedKey,
782
+ headOffset,
783
+ headLength,
784
+ bytes
785
+ });
786
+ }
787
+ return { key: expectedKey, bytes, checksum: raw.checksum, headOffset, headLength, headChecksum };
827
788
  }
828
789
 
829
790
  // src/persistence/durability/segment/manifest-vectors-decode.ts
@@ -920,12 +881,16 @@ function decodeManifestPayload(raw, indexName) {
920
881
  if (!isRecord(schema)) invalidManifest("is missing its schema");
921
882
  requireValidFieldTypes(schema);
922
883
  if (typeof raw.language !== "string") invalidManifest("is missing its language");
884
+ if (raw.analysisGeneration !== void 0 && !isSequenceNumber(raw.analysisGeneration)) {
885
+ invalidManifest("has an invalid analysis generation");
886
+ }
923
887
  return {
924
888
  kind: "current",
925
889
  manifest: {
926
890
  version: SEGMENT_MANIFEST_VERSION,
927
891
  schema,
928
892
  language: raw.language,
893
+ ...raw.analysisGeneration ? { analysisGeneration: raw.analysisGeneration } : {},
929
894
  checkpoint: normalizeCheckpoint(raw.checkpoint),
930
895
  partitions: normalizePartitions(raw.partitions, indexName),
931
896
  vectors: normalizeVectorFields(raw.vectors, indexName)
@@ -1077,83 +1042,292 @@ function contentsOfCaptured(captured) {
1077
1042
  return contents;
1078
1043
  }
1079
1044
 
1080
- // src/persistence/durability/segment/store.ts
1081
- function directoryCheckpointStore(directory) {
1082
- const store = {
1083
- read: (key) => directory.read(key),
1084
- write: (key, chunks) => directory.atomicWrite(key, chunks),
1085
- remove: (key) => directory.remove(key),
1086
- list: (prefix) => directory.list(prefix),
1087
- pathOf: (key) => directory.pathOf(key)
1088
- };
1089
- const byteLength = directory.byteLength;
1090
- if (byteLength !== void 0) store.byteLength = (key) => byteLength.call(directory, key);
1091
- return store;
1092
- }
1093
- function joined(chunks) {
1094
- if (chunks.length === 1) return chunks[0];
1095
- let total = 0;
1096
- for (const chunk of chunks) total += chunk.byteLength;
1097
- const bytes = new Uint8Array(total);
1045
+ // src/serialization/crc32-combine.ts
1046
+ var IEEE_POLYNOMIAL = 3988292384;
1047
+ var CRC_BITS = 32;
1048
+ var zeroByteOperators = [];
1049
+ function timesVector(operator, vector) {
1050
+ let sum = 0;
1051
+ let rest = vector >>> 0;
1052
+ for (let row = 0; rest !== 0; row++, rest >>>= 1) {
1053
+ if (rest & 1) sum ^= operator[row];
1054
+ }
1055
+ return sum >>> 0;
1056
+ }
1057
+ function squared(operator) {
1058
+ const result = new Uint32Array(CRC_BITS);
1059
+ for (let row = 0; row < CRC_BITS; row++) result[row] = timesVector(operator, operator[row]);
1060
+ return result;
1061
+ }
1062
+ function firstOperator() {
1063
+ let oneZeroBit = new Uint32Array(CRC_BITS);
1064
+ oneZeroBit[0] = IEEE_POLYNOMIAL;
1065
+ for (let row = 1; row < CRC_BITS; row++) oneZeroBit[row] = 1 << row - 1;
1066
+ for (let doubling = 0; doubling < 3; doubling++) oneZeroBit = squared(oneZeroBit);
1067
+ return oneZeroBit;
1068
+ }
1069
+ function operatorForZeroBytesPower(power) {
1070
+ if (zeroByteOperators.length === 0) zeroByteOperators.push(firstOperator());
1071
+ while (zeroByteOperators.length <= power) {
1072
+ zeroByteOperators.push(squared(zeroByteOperators[zeroByteOperators.length - 1]));
1073
+ }
1074
+ return zeroByteOperators[power];
1075
+ }
1076
+ function crc32Combine(first, second, secondByteLength) {
1077
+ let crc = first >>> 0;
1078
+ let rest = secondByteLength;
1079
+ for (let power = 0; rest > 0; power++) {
1080
+ if (rest % 2 === 1) crc = timesVector(operatorForZeroBytesPower(power), crc);
1081
+ rest = Math.floor(rest / 2);
1082
+ }
1083
+ return (crc ^ second) >>> 0;
1084
+ }
1085
+ var BIN8 = 196;
1086
+ var BIN16 = 197;
1087
+ var BIN32 = 198;
1088
+ var ARRAY16 = 220;
1089
+ var ARRAY32 = 221;
1090
+ var FIXMAP = 128;
1091
+ var FIXARRAY = 144;
1092
+ var FIXED_LIST_MAX = 15;
1093
+ var FIXSTR = 160;
1094
+ var FIXSTR_MAX_LENGTH = 31;
1095
+ var ASCII_MAX = 127;
1096
+ var UTF8_MAX_BYTES_PER_UNIT = 3;
1097
+ var STRING_HEADER_MAX_BYTES = 5;
1098
+ var BIN8_MAX_BYTES = 255;
1099
+ var UINT16_MAX = 65535;
1100
+ function binHeader(byteLength) {
1101
+ if (byteLength <= BIN8_MAX_BYTES) return Uint8Array.of(BIN8, byteLength);
1102
+ if (byteLength <= UINT16_MAX) return Uint8Array.of(BIN16, byteLength >> 8, byteLength & 255);
1103
+ const header = new Uint8Array(5);
1104
+ header[0] = BIN32;
1105
+ new DataView(header.buffer).setUint32(1, byteLength, false);
1106
+ return header;
1107
+ }
1108
+ function arrayHeader(count) {
1109
+ if (count <= FIXED_LIST_MAX) return Uint8Array.of(FIXARRAY | count);
1110
+ if (count <= UINT16_MAX) return Uint8Array.of(ARRAY16, count >> 8, count & 255);
1111
+ const header = new Uint8Array(5);
1112
+ header[0] = ARRAY32;
1113
+ new DataView(header.buffer).setUint32(1, count, false);
1114
+ return header;
1115
+ }
1116
+ function createPayloadBuilder() {
1117
+ const chunks = [];
1098
1118
  let offset = 0;
1099
- for (const chunk of chunks) {
1100
- bytes.set(chunk, offset);
1119
+ function push(chunk) {
1120
+ chunks.push(chunk);
1101
1121
  offset += chunk.byteLength;
1102
1122
  }
1123
+ return {
1124
+ chunks,
1125
+ get offset() {
1126
+ return offset;
1127
+ },
1128
+ push,
1129
+ entry(key, value) {
1130
+ push(encode(key));
1131
+ push(encode(value));
1132
+ },
1133
+ rawEntry(key, encoded) {
1134
+ push(encode(key));
1135
+ const at = offset;
1136
+ push(encoded);
1137
+ return at;
1138
+ },
1139
+ bytesEntry(key, bytes) {
1140
+ push(encode(key));
1141
+ push(binHeader(bytes.byteLength));
1142
+ const at = offset;
1143
+ push(bytes);
1144
+ return at;
1145
+ }
1146
+ };
1147
+ }
1148
+ function pageChecksums(bytes, pageBytes) {
1149
+ const pages = Math.ceil(bytes.byteLength / pageBytes);
1150
+ const checksums = new Uint8Array(pages * 4);
1151
+ const view = new DataView(checksums.buffer);
1152
+ for (let page = 0; page < pages; page++) {
1153
+ const from = page * pageBytes;
1154
+ view.setUint32(page * 4, crc32(bytes.subarray(from, Math.min(from + pageBytes, bytes.byteLength))), true);
1155
+ }
1156
+ return checksums;
1157
+ }
1158
+ function sectionOf(name, offset, bytes, pageBytes) {
1159
+ return { name, offset, length: bytes.byteLength, pageBytes, checksums: pageChecksums(bytes, pageBytes) };
1160
+ }
1161
+ function uint32Bytes(values) {
1162
+ const bytes = new Uint8Array(values.length * 4);
1163
+ const view = new DataView(bytes.buffer);
1164
+ for (let i = 0; i < values.length; i++) view.setUint32(i * 4, values[i], true);
1103
1165
  return bytes;
1104
1166
  }
1105
- function adapterFailure(code, verb, key, error) {
1106
- if (error instanceof NarsilError) return error;
1107
- const cause = error instanceof Error ? error.message : String(error);
1108
- return new NarsilError(code, `The persistence adapter failed to ${verb} "${key}": ${cause}`, { key, cause });
1167
+ function writtenAsShortAscii(docId, list, at) {
1168
+ if (docId.length > FIXSTR_MAX_LENGTH) return false;
1169
+ for (let unit = 0; unit < docId.length; unit++) {
1170
+ const code = docId.charCodeAt(unit);
1171
+ if (code > ASCII_MAX) return false;
1172
+ list[at + 1 + unit] = code;
1173
+ }
1174
+ list[at] = FIXSTR | docId.length;
1175
+ return true;
1109
1176
  }
1110
- async function guarded(run, code, verb, key) {
1111
- try {
1112
- return await run();
1113
- } catch (error) {
1114
- throw adapterFailure(code, verb, key, error);
1177
+ function encodedIdList(docIds) {
1178
+ const header = arrayHeader(docIds.length);
1179
+ let capacity = header.byteLength;
1180
+ for (const docId of docIds) capacity += docId.length * UTF8_MAX_BYTES_PER_UNIT + STRING_HEADER_MAX_BYTES;
1181
+ const list = new Uint8Array(capacity);
1182
+ list.set(header);
1183
+ const offsets = new Uint32Array(docIds.length + 1);
1184
+ let cursor = header.byteLength;
1185
+ for (let i = 0; i < docIds.length; i++) {
1186
+ offsets[i] = cursor;
1187
+ if (writtenAsShortAscii(docIds[i], list, cursor)) {
1188
+ cursor += docIds[i].length + 1;
1189
+ continue;
1190
+ }
1191
+ const encoded = encode(docIds[i]);
1192
+ list.set(encoded, cursor);
1193
+ cursor += encoded.byteLength;
1115
1194
  }
1195
+ offsets[docIds.length] = cursor;
1196
+ return { list: list.subarray(0, cursor), offsets };
1116
1197
  }
1117
- function adapterCheckpointStore(adapter) {
1118
- return {
1119
- read: (key) => guarded(() => adapter.load(key), ErrorCodes.PERSISTENCE_LOAD_FAILED, "load", key),
1120
- write: (key, chunks) => guarded(() => adapter.save(key, joined(chunks)), ErrorCodes.PERSISTENCE_SAVE_FAILED, "save", key),
1121
- remove: (key) => guarded(() => adapter.delete(key), ErrorCodes.PERSISTENCE_DELETE_FAILED, "delete", key),
1122
- list: (prefix) => guarded(() => adapter.list(prefix), ErrorCodes.PERSISTENCE_LOAD_FAILED, "list", prefix)
1123
- };
1198
+ function positionsInIdOrder(docIds) {
1199
+ const positions = Array.from(docIds.keys());
1200
+ positions.sort((left, right) => compareCodePoints(docIds[left], docIds[right]));
1201
+ return Uint32Array.from(positions);
1124
1202
  }
1125
- async function readListedFile(store, key, expected) {
1126
- const length = store.byteLength === void 0 ? void 0 : await store.byteLength(key);
1127
- if (length === null) missingFile(expected.what, key);
1128
- if (length !== void 0) requireListedLength(expected.what, key, length, expected.bytes);
1129
- const bytes = await store.read(key);
1130
- if (bytes === null) missingFile(expected.what, key);
1131
- requireListedLength(expected.what, key, bytes.byteLength, expected.bytes);
1203
+ function magnitudeBytes(magnitudes) {
1204
+ const bytes = new Uint8Array(magnitudes.length * 8);
1205
+ const view = new DataView(bytes.buffer);
1206
+ for (let i = 0; i < magnitudes.length; i++) view.setFloat64(i * 8, magnitudes[i], true);
1132
1207
  return bytes;
1133
1208
  }
1134
- function missingFile(what, key) {
1135
- throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The ${what} "${key}" that the manifest lists is missing`, {
1136
- key
1137
- });
1138
- }
1139
- function requireListedLength(what, key, found, listed) {
1140
- if (found === listed) return;
1141
- throw new NarsilError(
1142
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
1143
- `The ${what} "${key}" holds ${found} bytes where the manifest lists ${listed}`,
1144
- { key, bytes: found, listed }
1209
+ function checksumAcrossPages(checksums, byteLength, pageBytes) {
1210
+ const view = new DataView(checksums.buffer, checksums.byteOffset, checksums.byteLength);
1211
+ let whole = 0;
1212
+ for (let page = 0; page * 4 < checksums.byteLength; page++) {
1213
+ const pageLength = Math.min(pageBytes, byteLength - page * pageBytes);
1214
+ whole = crc32Combine(whole, view.getUint32(page * 4, true), pageLength);
1215
+ }
1216
+ return whole;
1217
+ }
1218
+ function encodedHeadAndVectorsPlace(sections, headOffsetAfterHeader, vectors, checksums, vectorsKeyAndHeaderBytes) {
1219
+ let headLength = 0;
1220
+ let head = new Uint8Array(0);
1221
+ let vectorsSection = {
1222
+ name: "vectors",
1223
+ offset: 0,
1224
+ length: vectors.byteLength,
1225
+ pageBytes: SEGMENT_PAGE_BYTES,
1226
+ checksums
1227
+ };
1228
+ for (let attempt = 0; attempt < STABLE_HEAD_ATTEMPTS; attempt++) {
1229
+ vectorsSection = {
1230
+ ...vectorsSection,
1231
+ offset: headOffsetAfterHeader(headLength) + headLength + vectorsKeyAndHeaderBytes
1232
+ };
1233
+ head = encode({ sections: [...sections, vectorsSection] });
1234
+ if (head.byteLength === headLength) break;
1235
+ headLength = head.byteLength;
1236
+ }
1237
+ return { head, vectorsSection };
1238
+ }
1239
+ function pagedVectorFileChunks(payload, heldMagnitudes) {
1240
+ const builder = createPayloadBuilder();
1241
+ const { codes } = payload;
1242
+ builder.push(Uint8Array.of(FIXMAP | 10));
1243
+ builder.entry("v", payload.v);
1244
+ builder.entry("fieldName", payload.fieldName);
1245
+ builder.entry("dimension", payload.dimension);
1246
+ const { list, offsets } = encodedIdList(payload.docIds);
1247
+ const docIdsAt = builder.rawEntry("docIds", list);
1248
+ const sections = [sectionOf("docIds", docIdsAt, list, SEGMENT_PAGE_BYTES)];
1249
+ builder.push(encode("codes"));
1250
+ if (codes === null) {
1251
+ builder.push(encode(null));
1252
+ } else {
1253
+ builder.push(Uint8Array.of(FIXMAP | 3));
1254
+ builder.entry("bits", codes.bits);
1255
+ builder.entry("centroid", codes.centroid);
1256
+ const recordsAt = builder.bytesEntry("records", codes.records);
1257
+ sections.push(sectionOf("records", recordsAt, codes.records, RECORD_PAGE_BYTES));
1258
+ }
1259
+ const magnitudes = magnitudeBytes(heldMagnitudes);
1260
+ sections.push(sectionOf("magnitudes", builder.bytesEntry("magnitudes", magnitudes), magnitudes, SEGMENT_PAGE_BYTES));
1261
+ const idOffsets = uint32Bytes(offsets);
1262
+ sections.push(sectionOf("idOffsets", builder.bytesEntry("idOffsets", idOffsets), idOffsets, SEGMENT_PAGE_BYTES));
1263
+ const idOrder = uint32Bytes(positionsInIdOrder(payload.docIds));
1264
+ sections.push(sectionOf("idOrder", builder.bytesEntry("idOrder", idOrder), idOrder, SEGMENT_PAGE_BYTES));
1265
+ const headKey = encode("head");
1266
+ const vectorsKey = encode("vectors");
1267
+ const vectorsHeader = binHeader(payload.vectors.byteLength);
1268
+ const beforeHead = builder.offset + headKey.byteLength;
1269
+ const vectorChecksums = pageChecksums(payload.vectors, SEGMENT_PAGE_BYTES);
1270
+ const { head } = encodedHeadAndVectorsPlace(
1271
+ sections,
1272
+ (headLength) => beforeHead + binHeader(headLength).byteLength,
1273
+ payload.vectors,
1274
+ vectorChecksums,
1275
+ vectorsKey.byteLength + vectorsHeader.byteLength
1145
1276
  );
1277
+ const headOffset = builder.bytesEntry("head", head);
1278
+ builder.push(vectorsKey);
1279
+ builder.push(vectorsHeader);
1280
+ const beforeVectors = [...builder.chunks];
1281
+ builder.push(payload.vectors);
1282
+ return {
1283
+ chunks: builder.chunks,
1284
+ headOffset,
1285
+ headLength: head.byteLength,
1286
+ headChecksum: crc32(head),
1287
+ beforeVectors,
1288
+ vectors: {
1289
+ byteLength: payload.vectors.byteLength,
1290
+ checksum: checksumAcrossPages(vectorChecksums, payload.vectors.byteLength, SEGMENT_PAGE_BYTES)
1291
+ }
1292
+ };
1146
1293
  }
1147
- function requireListedChecksum(what, key, found, listed) {
1148
- if (found === listed) return;
1149
- throw new NarsilError(
1150
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
1151
- `The ${what} "${key}" carries checksum ${found} where the manifest lists ${listed}`,
1152
- { key, checksum: found, listed }
1153
- );
1294
+ function nodeOffsetsOf(graph) {
1295
+ const offsets = new Uint32Array(graph.levels.length + 1);
1296
+ const view = new DataView(graph.neighbours.buffer, graph.neighbours.byteOffset, graph.neighbours.byteLength);
1297
+ let cursor = 0;
1298
+ for (let at = 0; at < graph.levels.length; at++) {
1299
+ offsets[at] = cursor;
1300
+ for (let layer = 0; layer < graph.levels[at]; layer++) cursor += (view.getUint32(cursor, true) + 1) * 4;
1301
+ }
1302
+ offsets[graph.levels.length] = cursor;
1303
+ return uint32Bytes(offsets);
1304
+ }
1305
+ function pagedGraphPartChunks(part) {
1306
+ const builder = createPayloadBuilder();
1307
+ builder.push(Uint8Array.of(FIXMAP | 4));
1308
+ builder.entry("v", part.v);
1309
+ builder.entry("fieldName", part.fieldName);
1310
+ builder.push(encode("graphs"));
1311
+ builder.push(arrayHeader(part.graphs.length));
1312
+ const layouts = [];
1313
+ for (const graph of part.graphs) {
1314
+ const { levels, neighbours, ...header } = graph;
1315
+ const headerEntries = Object.entries(header).filter(([, value]) => value !== void 0);
1316
+ builder.push(Uint8Array.of(FIXMAP | headerEntries.length + 2));
1317
+ for (const [key, value] of headerEntries) builder.entry(key, value);
1318
+ const levelsAt = builder.bytesEntry("levels", levels);
1319
+ const neighboursAt = builder.bytesEntry("neighbours", neighbours);
1320
+ layouts.push({
1321
+ ...header,
1322
+ levels: sectionOf("levels", levelsAt, levels, SEGMENT_PAGE_BYTES),
1323
+ nodeOffsets: nodeOffsetsOf(graph),
1324
+ neighbours: sectionOf("neighbours", neighboursAt, neighbours, RECORD_PAGE_BYTES)
1325
+ });
1326
+ }
1327
+ const head = encode({ graphs: layouts });
1328
+ const headOffset = builder.bytesEntry("head", head);
1329
+ return { chunks: builder.chunks, headOffset, headLength: head.byteLength, headChecksum: crc32(head) };
1154
1330
  }
1155
-
1156
- // src/persistence/durability/segment/vector-load.ts
1157
1331
  var VECTOR_FILE_ENVELOPE_VERSION = 2;
1158
1332
  var GRAPH_PART_ENVELOPE_VERSION = 2;
1159
1333
  function liveVectorsOf(ref) {
@@ -1163,13 +1337,13 @@ function liveVectorsOf(ref) {
1163
1337
  }
1164
1338
  async function openListedFile(store, ref, what, allowsCompression) {
1165
1339
  const bytes = await readListedFile(store, ref.key, { bytes: ref.bytes, what });
1166
- const { header: header2, payload } = await openChecksummedEnvelope(bytes, {
1340
+ const { header, payload } = await openChecksummedEnvelope(bytes, {
1167
1341
  envelopeFormatVersion: VECTOR_FILE_ENVELOPE_VERSION,
1168
1342
  what: `${what} "${ref.key}"`,
1169
1343
  allowsCompression
1170
1344
  });
1171
- requireListedChecksum(what, ref.key, header2.checksum, ref.checksum);
1172
- return { payload, payloadLength: header2.payloadLength };
1345
+ requireListedChecksum(what, ref.key, header.checksum, ref.checksum);
1346
+ return { payload, payloadLength: header.payloadLength };
1173
1347
  }
1174
1348
  async function readGraphParts(store, ref) {
1175
1349
  if (ref.graphParts.length === 0) return null;
@@ -1207,7 +1381,46 @@ async function readVectorFile(store, file, fieldPath) {
1207
1381
  }
1208
1382
  return { payload, payloadLength };
1209
1383
  }
1210
- async function loadVectorField(store, ref, vectorIndex) {
1384
+ function holdsEveryHead(ref) {
1385
+ if (ref.graphParts.length === 0 || ref.files.length === 0) return false;
1386
+ return [...ref.files, ...ref.graphParts].every((file) => file.headOffset !== void 0);
1387
+ }
1388
+ async function loadFieldOnDemand(store, ref, vectorIndex, resolvePartition) {
1389
+ const { pathOf } = store;
1390
+ if (pathOf === void 0 || vectorIndex.storage !== "disk" || !holdsEveryHead(ref)) return false;
1391
+ const { loadFilesByParts, openColdVectorField, osqBitsOf, waitBeforeColdLoad } = await import('./cold-module-MJMKNUYJ.mjs');
1392
+ await Promise.all([
1393
+ ...ref.files.map((file) => requireListedSize(store, file.key, { bytes: file.bytes, what: "vector file" })),
1394
+ ...ref.graphParts.map((part) => requireListedSize(store, part.key, { bytes: part.bytes, what: "graph part" }))
1395
+ ]);
1396
+ const files = await Promise.all(
1397
+ ref.files.map(async (file) => ({ ref: file, path: await pathOf.call(store, file.key) }))
1398
+ );
1399
+ const graphParts = await Promise.all(
1400
+ ref.graphParts.map(async (part) => ({ ref: part, path: await pathOf.call(store, part.key) }))
1401
+ );
1402
+ const field = openColdVectorField({
1403
+ fieldPath: ref.fieldPath,
1404
+ dimension: vectorIndex.dimension,
1405
+ metric: vectorIndex.metric,
1406
+ codeBits: osqBitsOf(vectorIndex.quantization),
1407
+ files,
1408
+ graphParts,
1409
+ resolvePartition
1410
+ });
1411
+ if (field === null) return false;
1412
+ vectorIndex.openColdField(field, async (beginRestore, stillWanted, firstSearchAnswered) => {
1413
+ await waitBeforeColdLoad(firstSearchAnswered);
1414
+ if (!stillWanted()) return;
1415
+ const graph = await readGraphParts(store, ref);
1416
+ if (!stillWanted()) return;
1417
+ const restore = beginRestore({ liveVectors: liveVectorsOf(ref), holdsGraph: true, graph });
1418
+ await loadFilesByParts(files, restore, ref.fieldPath, vectorIndex.dimension, stillWanted);
1419
+ });
1420
+ return true;
1421
+ }
1422
+ async function loadVectorField(store, ref, vectorIndex, resolvePartition = null) {
1423
+ if (resolvePartition !== null && await loadFieldOnDemand(store, ref, vectorIndex, resolvePartition)) return;
1211
1424
  const restore = vectorIndex.restoreCheckpoint({
1212
1425
  liveVectors: liveVectorsOf(ref),
1213
1426
  holdsGraph: ref.graphParts.length > 0,
@@ -1230,24 +1443,24 @@ async function loadVectorField(store, ref, vectorIndex) {
1230
1443
  }
1231
1444
  var EMPTY_BIN = new Uint8Array(0);
1232
1445
  var EMPTY_BIN_BYTES = 2;
1233
- var BIN8 = 196;
1234
- var BIN16 = 197;
1235
- var BIN32 = 198;
1236
- var BIN8_MAX_BYTES = 255;
1446
+ var BIN82 = 196;
1447
+ var BIN162 = 197;
1448
+ var BIN322 = 198;
1449
+ var BIN8_MAX_BYTES2 = 255;
1237
1450
  var BIN16_MAX_BYTES = 65535;
1238
- function binHeader(byteLength) {
1239
- if (byteLength <= BIN8_MAX_BYTES) return Uint8Array.of(BIN8, byteLength);
1240
- if (byteLength <= BIN16_MAX_BYTES) return Uint8Array.of(BIN16, byteLength >> 8, byteLength & 255);
1241
- const header2 = new Uint8Array(5);
1242
- header2[0] = BIN32;
1243
- new DataView(header2.buffer).setUint32(1, byteLength, false);
1244
- return header2;
1451
+ function binHeader2(byteLength) {
1452
+ if (byteLength <= BIN8_MAX_BYTES2) return Uint8Array.of(BIN82, byteLength);
1453
+ if (byteLength <= BIN16_MAX_BYTES) return Uint8Array.of(BIN162, byteLength >> 8, byteLength & 255);
1454
+ const header = new Uint8Array(5);
1455
+ header[0] = BIN322;
1456
+ new DataView(header.buffer).setUint32(1, byteLength, false);
1457
+ return header;
1245
1458
  }
1246
1459
  function chunksEndingInBytes(withTheBytes, withEmptyBytesInTheirPlace, bytes) {
1247
1460
  const head = encode(withEmptyBytesInTheirPlace);
1248
- const endsWithTheEmptyBin = head[head.length - EMPTY_BIN_BYTES] === BIN8 && head[head.length - 1] === 0;
1461
+ const endsWithTheEmptyBin = head[head.length - EMPTY_BIN_BYTES] === BIN82 && head[head.length - 1] === 0;
1249
1462
  if (!endsWithTheEmptyBin) return [encode(withTheBytes)];
1250
- return [head.subarray(0, head.length - EMPTY_BIN_BYTES), binHeader(bytes.byteLength), bytes];
1463
+ return [head.subarray(0, head.length - EMPTY_BIN_BYTES), binHeader2(bytes.byteLength), bytes];
1251
1464
  }
1252
1465
  function vectorPartChunks(part) {
1253
1466
  return chunksEndingInBytes(part, { ...part, vectors: EMPTY_BIN }, part.vectors);
@@ -1263,20 +1476,27 @@ function keptFileRef(input, prior, kept) {
1263
1476
  { key: kept.key }
1264
1477
  );
1265
1478
  }
1266
- return { ...kept, bytes: listed.bytes, checksum: listed.checksum };
1479
+ return { ...kept, bytes: listed.bytes, checksum: listed.checksum, ...headFieldsOf(listed) };
1480
+ }
1481
+ function headFieldsOf(ref) {
1482
+ return ref.headOffset === void 0 ? {} : { headOffset: ref.headOffset, headLength: ref.headLength, headChecksum: ref.headChecksum };
1483
+ }
1484
+ function headRefOf(paged) {
1485
+ return paged === null ? {} : { headOffset: paged.headOffset, headLength: paged.headLength, headChecksum: paged.headChecksum };
1267
1486
  }
1268
1487
  async function writeGraphParts(input, fieldPath, plan, generation) {
1269
1488
  const parts = plan.graphParts ?? [];
1270
1489
  const refs = [];
1271
1490
  for (let part = 0; part < parts.length; part++) {
1272
1491
  const key = graphPartKey(input.indexName, fieldPath, generation, part);
1492
+ const paged = plan.storesHeads ? pagedGraphPartChunks(parts[part]) : null;
1273
1493
  const packed = await packChecksummedEnvelope(
1274
- [encode(parts[part])],
1494
+ paged === null ? [encode(parts[part])] : paged.chunks,
1275
1495
  GRAPH_PART_ENVELOPE_VERSION,
1276
1496
  `graph part "${key}"`
1277
1497
  );
1278
1498
  await input.store.write(key, packed.chunks);
1279
- refs.push({ key, bytes: packed.byteLength, checksum: packed.checksum });
1499
+ refs.push({ key, bytes: packed.byteLength, checksum: packed.checksum, ...headRefOf(paged) });
1280
1500
  }
1281
1501
  return refs;
1282
1502
  }
@@ -1290,17 +1510,28 @@ async function writeField(input, fieldPath, plan, outcome) {
1290
1510
  const files = plan.kept.map((kept) => keptFileRef(input, prior, kept));
1291
1511
  const written = [];
1292
1512
  for (let index = 0; index < plan.newFiles; index++) {
1293
- const { payload, ordinals, dead } = plan.readNewFile(index);
1513
+ const { payload, ordinals, dead, magnitudes } = plan.readNewFile(index);
1294
1514
  const id = nextFileId;
1295
1515
  nextFileId += 1;
1296
1516
  const key = vectorFileKey(input.indexName, fieldPath, id);
1297
- const packed = await packChecksummedEnvelope(
1298
- vectorPartChunks(payload),
1517
+ const paged = plan.storesHeads && magnitudes !== null ? pagedVectorFileChunks(payload, magnitudes) : null;
1518
+ const what = `vector file "${key}"`;
1519
+ const packed = paged === null ? await packChecksummedEnvelope(vectorPartChunks(payload), VECTOR_FILE_ENVELOPE_VERSION, what) : packEnvelopeWithChecksum(
1520
+ paged.chunks,
1299
1521
  VECTOR_FILE_ENVELOPE_VERSION,
1300
- `vector file "${key}"`
1522
+ what,
1523
+ crc32Combine(await checksumOfChunks(paged.beforeVectors), paged.vectors.checksum, paged.vectors.byteLength)
1301
1524
  );
1302
1525
  await input.store.write(key, packed.chunks);
1303
- files.push({ id, key, count: payload.docIds.length, dead, bytes: packed.byteLength, checksum: packed.checksum });
1526
+ files.push({
1527
+ id,
1528
+ key,
1529
+ count: payload.docIds.length,
1530
+ dead,
1531
+ bytes: packed.byteLength,
1532
+ checksum: packed.checksum,
1533
+ ...headRefOf(paged)
1534
+ });
1304
1535
  written.push({ id, key, ordinals });
1305
1536
  outcome.layouts.push({
1306
1537
  fieldPath,
@@ -1503,10 +1734,10 @@ var RECORD_LENGTH_SIZE = 4;
1503
1734
  var FRAME_CRC_SIZE = 4;
1504
1735
  var MAX_WAL_RECORD_BYTES = 268435456;
1505
1736
  function writeSegmentHeader() {
1506
- const header2 = new Uint8Array(SEGMENT_HEADER_SIZE);
1507
- header2.set(WAL_MAGIC, 0);
1508
- header2[4] = WAL_FORMAT_VERSION & 255;
1509
- return header2;
1737
+ const header = new Uint8Array(SEGMENT_HEADER_SIZE);
1738
+ header.set(WAL_MAGIC, 0);
1739
+ header[4] = WAL_FORMAT_VERSION & 255;
1740
+ return header;
1510
1741
  }
1511
1742
  function frameRecord(entry) {
1512
1743
  const payload = encode(entry);
@@ -1547,9 +1778,9 @@ function corrupt(message, details) {
1547
1778
  throw new NarsilError(ErrorCodes.PERSISTENCE_WAL_CORRUPT, message, details);
1548
1779
  }
1549
1780
  function readDurableRegion(data, durableByteLength, maxRecordBytes = MAX_WAL_RECORD_BYTES) {
1550
- const header2 = checkSegmentHeader(data);
1551
- if (!header2.ok) {
1552
- corrupt(`WAL segment header invalid in durable region: ${header2.reason}`, { reason: header2.reason });
1781
+ const header = checkSegmentHeader(data);
1782
+ if (!header.ok) {
1783
+ corrupt(`WAL segment header invalid in durable region: ${header.reason}`, { reason: header.reason });
1553
1784
  }
1554
1785
  if (durableByteLength < SEGMENT_HEADER_SIZE || durableByteLength > data.length) {
1555
1786
  corrupt("Commit marker durable byte length is outside the segment", {
@@ -1648,6 +1879,9 @@ function readTailBeyondFrontier(data, durableByteLength, lastDurableSeqNo, maxRe
1648
1879
  function walPrefix(indexName, partitionId) {
1649
1880
  return `${indexName}/wal/${partitionId}/`;
1650
1881
  }
1882
+ function nextSegmentKey(indexName, partitionId) {
1883
+ return `${walPrefix(indexName, partitionId)}next`;
1884
+ }
1651
1885
  function segmentStartSeqNo(key, prefix) {
1652
1886
  if (!key.startsWith(prefix)) {
1653
1887
  return null;
@@ -1692,14 +1926,14 @@ async function* readSegmentsInOrder(directory, wal) {
1692
1926
  continue;
1693
1927
  }
1694
1928
  if (startSeqNo < marker.state.activeSegmentSeqNo) {
1695
- const header2 = checkSegmentHeader(bytes);
1696
- if (!header2.ok) {
1929
+ const header = checkSegmentHeader(bytes);
1930
+ if (!header.ok) {
1697
1931
  throw new NarsilError(
1698
1932
  ErrorCodes.PERSISTENCE_WAL_CORRUPT,
1699
- `Sealed WAL segment header invalid: ${header2.reason}`,
1933
+ `Sealed WAL segment header invalid: ${header.reason}`,
1700
1934
  {
1701
1935
  key,
1702
- reason: header2.reason
1936
+ reason: header.reason
1703
1937
  }
1704
1938
  );
1705
1939
  }
@@ -1829,569 +2063,23 @@ async function buildSegmentFromEntries(input) {
1829
2063
  if (input.keepsWriteRecords) built.writeRecords = /* @__PURE__ */ new Map([[segment, writeRecordsOf(documents, written)]]);
1830
2064
  return built;
1831
2065
  }
1832
- var FIXMAP = 128;
1833
- var MAP16 = 222;
1834
- var MAP32 = 223;
1835
- var FIXARRAY = 144;
1836
- var ARRAY16 = 220;
1837
- var ARRAY32 = 221;
1838
- var BIN82 = 196;
1839
- var BIN162 = 197;
1840
- var BIN322 = 198;
1841
- var FIX_COUNT_LIMIT = 16;
1842
- var UINT8_LIMIT = 256;
1843
- var UINT16_LIMIT = 65536;
1844
- function header(fixed, sixteen, thirtyTwo, count) {
1845
- if (count < FIX_COUNT_LIMIT) return Uint8Array.of(fixed | count);
1846
- if (count < UINT16_LIMIT) return Uint8Array.of(sixteen, count >> 8, count & 255);
1847
- const bytes = new Uint8Array(5);
1848
- bytes[0] = thirtyTwo;
1849
- new DataView(bytes.buffer).setUint32(1, count, false);
1850
- return bytes;
1851
- }
1852
- function binHeader2(byteLength) {
1853
- if (byteLength < UINT8_LIMIT) return Uint8Array.of(BIN82, byteLength);
1854
- if (byteLength < UINT16_LIMIT) return Uint8Array.of(BIN162, byteLength >> 8, byteLength & 255);
1855
- const bytes = new Uint8Array(5);
1856
- bytes[0] = BIN322;
1857
- new DataView(bytes.buffer).setUint32(1, byteLength, false);
1858
- return bytes;
1859
- }
1860
- var MessagePackChunks = class {
1861
- chunks = [];
1862
- byteLength = 0;
1863
- push(chunk) {
1864
- if (chunk.byteLength === 0) return;
1865
- this.chunks.push(chunk);
1866
- this.byteLength += chunk.byteLength;
1867
- }
1868
- map(entries) {
1869
- this.push(header(FIXMAP, MAP16, MAP32, entries));
1870
- return this;
1871
- }
1872
- array(items) {
1873
- this.push(header(FIXARRAY, ARRAY16, ARRAY32, items));
1874
- return this;
1875
- }
1876
- value(value) {
1877
- this.push(encode(value));
1878
- return this;
1879
- }
1880
- key(name) {
1881
- return this.value(name);
1882
- }
1883
- binary(bytes) {
1884
- this.push(binHeader2(bytes.byteLength));
1885
- this.push(bytes);
1886
- return this;
1887
- }
1888
- binaryOrNil(bytes) {
1889
- return bytes === null ? this.value(null) : this.binary(bytes);
1890
- }
1891
- };
1892
-
1893
- // src/serialization/keyword-segment/token-order.ts
1894
- function slotsAreInTokenOrder(payloadSlots) {
1895
- for (let at = 0; at < payloadSlots.length; at++) {
1896
- if (payloadSlots[at] !== at) return false;
1897
- }
1898
- return true;
1899
- }
1900
- function reorderedPositions(arrays, postingOffsets) {
1901
- const { positionOffsets, positionValues, tokenTable } = arrays;
1902
- if (positionOffsets === null || positionValues === null) return { positionOffsets: null, positionValues: null };
1903
- const postings = postingOffsets[postingOffsets.length - 1];
1904
- const offsets = new Uint32Array(postings + 1);
1905
- const values = new Uint32Array(positionValues.length);
1906
- let cursor = 0;
1907
- for (let token = 0; token < tokenTable.payloadSlots.length; token++) {
1908
- const slot = tokenTable.payloadSlots[token];
1909
- for (let from = arrays.postingOffsets[slot]; from < arrays.postingOffsets[slot + 1]; from++) {
1910
- const to = postingOffsets[token] + (from - arrays.postingOffsets[slot]);
1911
- offsets[to] = cursor;
1912
- const start = positionOffsets[from];
1913
- const end = positionOffsets[from + 1];
1914
- values.set(positionValues.subarray(start, end), cursor);
1915
- cursor += end - start;
1916
- }
1917
- }
1918
- offsets[postings] = cursor;
1919
- return { positionOffsets: offsets, positionValues: values };
1920
- }
1921
- function postingsInTokenOrder(arrays) {
1922
- const { tokenTable } = arrays;
1923
- if (slotsAreInTokenOrder(tokenTable.payloadSlots)) return arrays;
1924
- const tokens = tokenTable.payloadSlots.length;
1925
- const postingOffsets = new Uint32Array(tokens + 1);
1926
- for (let token = 0; token < tokens; token++) {
1927
- const slot = tokenTable.payloadSlots[token];
1928
- postingOffsets[token + 1] = postingOffsets[token] + arrays.postingOffsets[slot + 1] - arrays.postingOffsets[slot];
1929
- }
1930
- const postings = postingOffsets[tokens];
1931
- const postingDocIds = new Uint32Array(postings);
1932
- const postingFrequencies = new Uint16Array(postings);
1933
- const postingFieldIndices = new Uint16Array(postings);
1934
- for (let token = 0; token < tokens; token++) {
1935
- const slot = tokenTable.payloadSlots[token];
1936
- const start = arrays.postingOffsets[slot];
1937
- const end = arrays.postingOffsets[slot + 1];
1938
- postingDocIds.set(arrays.postingDocIds.subarray(start, end), postingOffsets[token]);
1939
- postingFrequencies.set(arrays.postingFrequencies.subarray(start, end), postingOffsets[token]);
1940
- postingFieldIndices.set(arrays.postingFieldIndices.subarray(start, end), postingOffsets[token]);
1941
- }
1942
- return {
1943
- postingOffsets,
1944
- postingDocIds,
1945
- postingFrequencies,
1946
- postingFieldIndices,
1947
- ...reorderedPositions(arrays, postingOffsets)
1948
- };
1949
- }
1950
-
1951
- // src/serialization/keyword-segment/encode.ts
1952
- var TOP_LEVEL_ENTRIES = 28;
1953
- function writeFieldLengths(out, arrays) {
1954
- out.array(arrays.fieldLengthNames.length);
1955
- for (let f = 0; f < arrays.fieldLengthNames.length; f++) {
1956
- out.map(2).key("field").value(arrays.fieldLengthNames[f]);
1957
- out.key("lengths").binary(littleEndianBytes(arrays.fieldLengthColumns[f]));
1958
- }
1959
- }
1960
- function writeValueColumns(out, columns) {
1961
- out.array(columns.length);
1962
- for (const column of columns) {
1963
- out.map(3).key("field").value(column.fieldPath);
1964
- out.key("documents").binary(littleEndianBytes(column.docIds));
1965
- out.key("values").binary(littleEndianBytes(column.values));
1966
- }
1967
- }
1968
- function writeBooleanColumns(out, columns) {
1969
- out.array(columns.length);
1970
- for (const column of columns) {
1971
- out.map(3).key("field").value(column.fieldPath);
1972
- out.key("trueDocuments").binary(littleEndianBytes(column.trueDocs));
1973
- out.key("falseDocuments").binary(littleEndianBytes(column.falseDocs));
1974
- }
1975
- }
1976
- function writeEnumColumns(out, columns) {
1977
- out.array(columns.length);
1978
- for (const column of columns) {
1979
- out.map(4).key("field").value(column.fieldPath);
1980
- out.key("values").value(column.values);
1981
- out.key("offsets").binary(littleEndianBytes(column.offsets));
1982
- out.key("documents").binary(littleEndianBytes(column.docIds));
1983
- }
1984
- }
1985
- function writeGeopointColumns(out, columns) {
1986
- out.array(columns.length);
1987
- for (const column of columns) {
1988
- out.map(4).key("field").value(column.fieldPath);
1989
- out.key("documents").binary(littleEndianBytes(column.docIds));
1990
- out.key("latitudes").binary(littleEndianBytes(column.latitudes));
1991
- out.key("longitudes").binary(littleEndianBytes(column.longitudes));
1992
- }
1993
- }
1994
- function writePatternColumns(out, columns) {
1995
- out.array(columns.length);
1996
- for (const column of columns) {
1997
- out.map(5).key("field").value(column.fieldPath);
1998
- out.key("documents").binary(littleEndianBytes(column.docIds));
1999
- out.key("runs").binary(littleEndianBytes(column.runs));
2000
- out.key("offsets").binary(littleEndianBytes(column.offsets));
2001
- out.key("postings").binary(littleEndianBytes(column.postings));
2002
- }
2003
- }
2004
- function writeSurfaceForms(out, arrays) {
2005
- const table = arrays.surfaceTable;
2006
- if (table === null || table.counts.length === 0) {
2007
- out.value(null);
2008
- return;
2009
- }
2010
- out.map(3).key("text").binary(table.blob);
2011
- out.key("offsets").binary(littleEndianBytes(table.offsets));
2012
- out.key("counts").binary(littleEndianBytes(table.counts));
2013
- }
2014
- function writePartialWords(out, arrays) {
2015
- const table = arrays.partialWords;
2016
- if (table === null) {
2017
- out.value(null);
2018
- return;
2019
- }
2020
- out.map(8).key("fields").value(table.fields);
2021
- out.key("text").binary(table.spellings);
2022
- out.key("offsets").binary(littleEndianBytes(table.spellingOffsets));
2023
- out.key("tokens").binary(littleEndianBytes(table.spellingTokens));
2024
- out.key("counts").binary(littleEndianBytes(table.spellingCounts));
2025
- out.key("runs").binary(littleEndianBytes(table.runs));
2026
- out.key("runOffsets").binary(littleEndianBytes(table.runOffsets));
2027
- out.key("runEntries").binary(littleEndianBytes(table.runEntries));
2028
- }
2029
- function encodeKeywordSegment(contents) {
2030
- const { arrays } = contents;
2031
- const postings = postingsInTokenOrder(arrays);
2032
- const out = new MessagePackChunks();
2033
- const { writeRecords } = contents;
2034
- out.map(writeRecords === void 0 ? TOP_LEVEL_ENTRIES : TOP_LEVEL_ENTRIES + 2);
2035
- out.key("v").value(KEYWORD_SEGMENT_PAYLOAD_VERSION);
2036
- out.key("schema").value(contents.schema);
2037
- out.key("tombstones").value(contents.tombstones);
2038
- out.key("documentCount").value(arrays.documentCount);
2039
- out.key("ids").binary(arrays.idTable.blob);
2040
- out.key("idOffsets").binary(littleEndianBytes(arrays.idTable.offsets));
2041
- out.key("idOrder").binary(littleEndianBytes(arrays.idTable.sortedOrdinals));
2042
- out.key("documents").binary(arrays.documentTable.blob);
2043
- out.key("documentOffsets").binary(littleEndianBytes(arrays.documentTable.offsets));
2044
- out.key("fieldNames").value(arrays.fieldNames);
2045
- out.key("tokens").binary(arrays.tokenTable.blob);
2046
- out.key("tokenOffsets").binary(littleEndianBytes(arrays.tokenTable.offsets));
2047
- out.key("documentFrequencies").binary(littleEndianBytes(arrays.tokenTable.documentFrequencies));
2048
- out.key("postingOffsets").binary(littleEndianBytes(postings.postingOffsets));
2049
- out.key("postingDocuments").binary(littleEndianBytes(postings.postingDocIds));
2050
- out.key("postingFrequencies").binary(littleEndianBytes(postings.postingFrequencies));
2051
- out.key("postingFields").binary(littleEndianBytes(postings.postingFieldIndices));
2052
- out.key("positionOffsets").binaryOrNil(postings.positionOffsets === null ? null : littleEndianBytes(postings.positionOffsets));
2053
- out.key("positions").binaryOrNil(postings.positionValues === null ? null : littleEndianBytes(postings.positionValues));
2054
- out.key("fieldLengths");
2055
- writeFieldLengths(out, arrays);
2056
- out.key("numeric");
2057
- writeValueColumns(out, arrays.numeric);
2058
- out.key("sizes");
2059
- writeValueColumns(out, arrays.sizes);
2060
- out.key("boolean");
2061
- writeBooleanColumns(out, arrays.boolean);
2062
- out.key("enum");
2063
- writeEnumColumns(out, arrays.enums);
2064
- out.key("geopoint");
2065
- writeGeopointColumns(out, arrays.geo);
2066
- out.key("pattern");
2067
- writePatternColumns(out, arrays.pattern);
2068
- out.key("surfaceForms");
2069
- writeSurfaceForms(out, arrays);
2070
- out.key("partialWords");
2071
- writePartialWords(out, arrays);
2072
- if (writeRecords !== void 0) {
2073
- out.key("seqNos").binary(uint64Bytes(writeRecords.seqNos));
2074
- out.key("primaryTerms").binary(uint64Bytes(writeRecords.primaryTerms));
2075
- }
2076
- return out;
2077
- }
2078
-
2079
- // src/persistence/durability/segment/segment-file.ts
2080
- function sharesMemoryWithWorkers() {
2081
- return typeof SharedArrayBuffer === "function";
2082
- }
2083
- async function readSegmentFile(store, ref) {
2084
- const bytes = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "segment" });
2085
- const { header: header2, payload } = await openChecksummedEnvelope(bytes, {
2086
- envelopeFormatVersion: KEYWORD_SEGMENT_ENVELOPE_VERSION,
2087
- what: `segment "${ref.key}"`,
2088
- allowsCompression: true
2089
- });
2090
- requireListedChecksum("segment", ref.key, header2.checksum, ref.checksum);
2091
- return decodeKeywordSegment(payload, sharesMemoryWithWorkers());
2092
- }
2093
- var ORDINAL_BYTES = 4;
2094
- var POSTING_BYTES = 8;
2095
- var VALUE_ENTRY_BYTES = 12;
2096
- var GEOPOINT_ENTRY_BYTES = 20;
2097
- var TOKEN_ENTRY_BYTES = 12;
2098
- var STRING_HEADER_BYTES = 5;
2099
- var encoder = new TextEncoder();
2100
- function documentBytes(segment, ordinal) {
2101
- const table = segment.arrays.documentTable;
2102
- if (table !== null) return table.offsets[ordinal + 1] - table.offsets[ordinal];
2103
- return messagePackUpperBoundBytes(segment.documentSource.docAt(ordinal));
2104
- }
2105
- function addPerEntry(costs, docIds, bytes) {
2106
- for (let at = 0; at < docIds.length; at++) costs[docIds[at]] += bytes;
2107
- }
2108
- function chargePostings(segment, costs, chargesTokens) {
2109
- const arrays = segment.arrays;
2110
- const positions = arrays.positionOffsets;
2111
- for (let p = 0; p < arrays.postingDocIds.length; p++) {
2112
- let bytes = POSTING_BYTES;
2113
- if (positions !== null) bytes += ORDINAL_BYTES * (1 + positions[p + 1] - positions[p]);
2114
- costs[arrays.postingDocIds[p]] += bytes;
2115
- }
2116
- if (!chargesTokens) return;
2117
- const table = arrays.tokenTable;
2118
- for (let at = 0; at < table.size; at++) {
2119
- const slot = table.payloadSlot(at);
2120
- const bytes = table.byteLengthAt(at) + TOKEN_ENTRY_BYTES;
2121
- for (let p = arrays.postingOffsets[slot]; p < arrays.postingOffsets[slot + 1]; p++) {
2122
- costs[arrays.postingDocIds[p]] += bytes;
2123
- }
2124
- }
2125
- }
2126
- function chargeColumns(segment, costs) {
2127
- const arrays = segment.arrays;
2128
- for (const column of [...arrays.numeric, ...arrays.sizes]) addPerEntry(costs, column.docIds, VALUE_ENTRY_BYTES);
2129
- for (const column of arrays.boolean) {
2130
- addPerEntry(costs, column.trueDocs, ORDINAL_BYTES);
2131
- addPerEntry(costs, column.falseDocs, ORDINAL_BYTES);
2132
- }
2133
- for (const column of arrays.enums) addPerEntry(costs, column.docIds, ORDINAL_BYTES);
2134
- for (const column of arrays.geo) addPerEntry(costs, column.docIds, GEOPOINT_ENTRY_BYTES);
2135
- for (const column of arrays.pattern) {
2136
- addPerEntry(costs, column.docIds, ORDINAL_BYTES);
2137
- addPerEntry(costs, column.postings, ORDINAL_BYTES);
2138
- }
2139
- }
2140
- function stringListBytes(values) {
2141
- let bytes = STRING_HEADER_BYTES;
2142
- for (const value of values) bytes += STRING_HEADER_BYTES + encoder.encode(value).byteLength;
2143
- return bytes;
2144
- }
2145
- function fixedBytesOf(segment, chargesTokens) {
2146
- const arrays = segment.arrays;
2147
- let bytes = stringListBytes(arrays.fieldNames);
2148
- if (!chargesTokens) bytes += arrays.tokenTable.bytes;
2149
- for (const column of arrays.enums) bytes += stringListBytes(column.values) + column.offsets.byteLength;
2150
- for (const column of arrays.pattern) bytes += column.runs.byteLength + column.offsets.byteLength;
2151
- const surfaces = segment.sharedSnapshot?.surfaceTable ?? null;
2152
- if (surfaces !== null) bytes += surfaces.blob.byteLength + surfaces.offsets.byteLength + surfaces.counts.byteLength;
2153
- else bytes += messagePackUpperBoundBytes(arrays.readSurfaceForms());
2154
- return bytes + segment.partialWords.bytes();
2155
- }
2156
- function keptDocumentsOf(selection, chargesTokens) {
2157
- const { segment } = selection;
2158
- const count = segment.arrays.documentCount;
2159
- const costs = new Float64Array(count);
2160
- const perDocument = ORDINAL_BYTES * (3 + segment.arrays.fieldLengthColumns.length);
2161
- for (let ordinal = 0; ordinal < count; ordinal++) {
2162
- costs[ordinal] = perDocument + encoder.encode(segment.arrays.idTable.idAt(ordinal)).byteLength;
2163
- }
2164
- chargePostings(segment, costs, chargesTokens);
2165
- chargeColumns(segment, costs);
2166
- const ordinals = [];
2167
- for (let ordinal = 0; ordinal < count; ordinal++) {
2168
- if (!selectionKeeps(selection, ordinal)) continue;
2169
- costs[ordinal] += documentBytes(segment, ordinal);
2170
- ordinals.push(ordinal);
2171
- }
2172
- return { segment, ordinals, costs };
2173
- }
2174
- function selectionOf(segment, ordinals) {
2175
- const marked = new Uint8Array(segment.arrays.documentCount);
2176
- for (const ordinal of ordinals) marked[ordinal] = 1;
2177
- return { segment, keep: (ordinal) => marked[ordinal] === 1 };
2178
- }
2179
- function tooLargeForOneFile(cost, budget) {
2180
- throw new NarsilError(
2181
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
2182
- `A document needs up to ${Math.ceil(cost)} bytes of a segment, more than the ${Math.max(0, Math.floor(budget))} that one file leaves it`,
2183
- { bytes: Math.ceil(cost), budget: Math.floor(budget), maximum: MAX_FILE_BYTES }
2184
- );
2185
- }
2186
- function splitIntoSegmentFiles(selections, schema, tombstones) {
2187
- let tokenBytes = 0;
2188
- for (const { segment } of selections) tokenBytes += segment.arrays.tokenTable.bytes;
2189
- const chargesTokens = tokenBytes > MAX_FILE_BYTES * SEGMENT_TOKEN_TABLE_SHARE_OF_FILE;
2190
- let fixed = HEADER_SIZE + SEGMENT_FILE_FRAMING_SLACK_BYTES + encode(schema).byteLength + encode(tombstones).byteLength;
2191
- for (const { segment } of selections) fixed += fixedBytesOf(segment, chargesTokens);
2192
- const budget = MAX_FILE_BYTES - fixed;
2193
- const groups = [];
2194
- let current = [];
2195
- let currentBytes = 0;
2196
- let currentCount = 0;
2197
- const flush = () => {
2198
- if (currentCount === 0) return;
2199
- groups.push({
2200
- selections: current.map((entry) => selectionOf(entry.segment, entry.ordinals)),
2201
- documentCount: currentCount
2202
- });
2203
- current = [];
2204
- currentBytes = 0;
2205
- currentCount = 0;
2206
- };
2207
- for (const selection of selections) {
2208
- const kept = keptDocumentsOf(selection, chargesTokens);
2209
- let entry = null;
2210
- for (const ordinal of kept.ordinals) {
2211
- const cost = kept.costs[ordinal];
2212
- if (cost > budget) tooLargeForOneFile(cost, budget);
2213
- if (currentBytes + cost > budget) {
2214
- flush();
2215
- entry = null;
2216
- }
2217
- if (entry === null) {
2218
- entry = { segment: kept.segment, ordinals: [] };
2219
- current.push(entry);
2220
- }
2221
- entry.ordinals.push(ordinal);
2222
- currentBytes += cost;
2223
- currentCount += 1;
2224
- }
2225
- }
2226
- flush();
2227
- return groups;
2228
- }
2229
- function halvesOf(group) {
2230
- const flat = [];
2231
- for (const selection of group.selections) {
2232
- for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
2233
- if (!selectionKeeps(selection, ordinal)) continue;
2234
- flat.push({ segment: selection.segment, ordinal });
2235
- }
2236
- }
2237
- const middle = Math.ceil(flat.length / 2);
2238
- const groupOf = (entries) => {
2239
- const bySegment = /* @__PURE__ */ new Map();
2240
- for (const { segment, ordinal } of entries) {
2241
- let ordinals = bySegment.get(segment);
2242
- if (ordinals === void 0) {
2243
- ordinals = [];
2244
- bySegment.set(segment, ordinals);
2245
- }
2246
- ordinals.push(ordinal);
2247
- }
2248
- return {
2249
- selections: [...bySegment].map(([segment, ordinals]) => selectionOf(segment, ordinals)),
2250
- documentCount: entries.length
2251
- };
2252
- };
2253
- return [groupOf(flat.slice(0, middle)), groupOf(flat.slice(middle))];
2254
- }
2255
-
2256
- // src/persistence/durability/segment/segment-write.ts
2257
- function emptyPayload() {
2258
- return {
2259
- documentCount: 0,
2260
- docIds: [],
2261
- fieldNames: [],
2262
- tokens: [],
2263
- postingOffsets: new Uint32Array(1),
2264
- postingDocIds: new Uint32Array(0),
2265
- postingFrequencies: new Uint16Array(0),
2266
- postingFieldIndices: new Uint16Array(0),
2267
- positionOffsets: null,
2268
- positionValues: null,
2269
- fieldLengthNames: [],
2270
- fieldLengthColumns: [],
2271
- totalFieldLengths: {},
2272
- docFrequencies: {},
2273
- surfaceForms: null,
2274
- partialSpellings: null,
2275
- numeric: [],
2276
- boolean: [],
2277
- enums: [],
2278
- geo: [],
2279
- pattern: [],
2280
- sizes: []
2281
- };
2282
- }
2283
- function noSpellingsOf(analysis) {
2284
- if (analysis === null || analysis.fields.length === 0) return null;
2285
- return { fields: [...analysis.fields], tokens: [], spellings: [], counts: [] };
2286
- }
2287
- function writeRecordsOfGroup(group, records, count) {
2288
- const seqNos = new Float64Array(count);
2289
- const primaryTerms = new Float64Array(count);
2290
- let at = 0;
2291
- for (const selection of group?.selections ?? []) {
2292
- const source = records.get(selection.segment);
2293
- for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
2294
- if (!selectionKeeps(selection, ordinal)) continue;
2295
- if (source !== void 0) {
2296
- seqNos[at] = source.seqNos[ordinal];
2297
- primaryTerms[at] = source.primaryTerms[ordinal];
2298
- }
2299
- at += 1;
2300
- }
2301
- }
2302
- return { seqNos, primaryTerms };
2303
- }
2304
- function withWriteRecords(contents, group, request) {
2305
- if (request.writeRecords === void 0) return contents;
2306
- const writeRecords = writeRecordsOfGroup(group, request.writeRecords, contents.arrays.documentCount);
2307
- return { ...contents, writeRecords };
2308
- }
2309
- function contentsOfGroup(group, request, tombstones) {
2310
- const { schema, partialWordAnalysis, textAnalysis } = request;
2311
- if (group === null) {
2312
- const payload = { ...emptyPayload(), partialSpellings: noSpellingsOf(partialWordAnalysis) };
2313
- return {
2314
- schema,
2315
- tombstones,
2316
- arrays: segmentArraysOf(payload, { blob: new Uint8Array(0), offsets: new Uint32Array(1) })
2317
- };
2318
- }
2319
- const merged = mergeSegmentContents(group.selections, partialWordAnalysis, textAnalysis);
2320
- return {
2321
- schema,
2322
- tombstones,
2323
- arrays: segmentArraysOf(merged.payload, merged.documents, merged.reusedPartialWords)
2324
- };
2325
- }
2326
- function requireRoomForAnotherSegment(state) {
2327
- if (state.segments.length < MAX_SEGMENTS_PER_PARTITION) return;
2328
- throw new NarsilError(
2329
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
2330
- `Partition ${state.request.partitionId} would hold more than ${MAX_SEGMENTS_PER_PARTITION} segments`,
2331
- { partitionId: state.request.partitionId, maximum: MAX_SEGMENTS_PER_PARTITION }
2332
- );
2333
- }
2334
- async function writeGroup(state, group) {
2335
- const { request } = state;
2336
- const tombstones = state.tombstonesWritten ? [] : [...request.tombstones];
2337
- const contents = withWriteRecords(contentsOfGroup(group, request, tombstones), group, request);
2338
- const encoded = encodeKeywordSegment(contents);
2339
- if (HEADER_SIZE + encoded.byteLength > MAX_FILE_BYTES && group !== null && group.documentCount > 1) {
2340
- const [first, second] = halvesOf(group);
2341
- await writeGroup(state, first);
2342
- await writeGroup(state, second);
2343
- return;
2344
- }
2345
- requireRoomForAnotherSegment(state);
2346
- const id = state.nextSegmentId;
2347
- const key = segmentKey(request.indexName, request.partitionId, id);
2348
- const packed = await packChecksummedEnvelope(encoded.chunks, KEYWORD_SEGMENT_ENVELOPE_VERSION, `segment "${key}"`);
2349
- await request.store.write(key, packed.chunks);
2350
- state.segments.push({
2351
- id,
2352
- key,
2353
- bytes: packed.byteLength,
2354
- checksum: packed.checksum,
2355
- docCount: contents.arrays.documentCount,
2356
- tombstoneCount: tombstones.length
2357
- });
2358
- state.nextSegmentId = id + 1;
2359
- state.tombstonesWritten = true;
2360
- }
2361
- function keptDocumentCount(selections) {
2362
- let count = 0;
2363
- for (const selection of selections) {
2364
- for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
2365
- if (selectionKeeps(selection, ordinal)) count += 1;
2366
- }
2367
- }
2368
- return count;
2369
- }
2370
- async function writeSegmentsOf(request) {
2371
- const state = { request, segments: [], nextSegmentId: request.nextSegmentId, tombstonesWritten: false };
2372
- const groups = request.mergesIntoOneFile === true ? [{ selections: [...request.selections], documentCount: keptDocumentCount(request.selections) }] : splitIntoSegmentFiles(request.selections, request.schema, request.tombstones);
2373
- for (const group of groups) {
2374
- if (group.documentCount > 0) await writeGroup(state, group);
2375
- }
2376
- if (!state.tombstonesWritten && request.tombstones.length > 0) await writeGroup(state, null);
2377
- return { segments: state.segments, nextSegmentId: state.nextSegmentId };
2378
- }
2379
2066
 
2380
2067
  // src/persistence/durability/segment/compaction.ts
2381
- function windowFits(segments, start, windowLength) {
2068
+ function windowFits(segments, start, windowLength, paged) {
2382
2069
  let documents = 0;
2383
2070
  let bytes = SEGMENT_FILE_FRAMING_SLACK_BYTES;
2384
2071
  for (let at = start; at < start + windowLength; at++) {
2385
2072
  documents += segments[at].docCount;
2386
2073
  bytes += segments[at].bytes;
2387
2074
  }
2075
+ if (paged) return bytes * MERGED_SEGMENT_GROWTH_ALLOWANCE <= MERGED_SEGMENT_TARGET_BYTES;
2388
2076
  return documents <= COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT && bytes <= MAX_FILE_BYTES;
2389
2077
  }
2390
- function smallestFittingWindow(segments, windowLength) {
2078
+ function smallestFittingWindow(segments, windowLength, paged) {
2391
2079
  let best = null;
2392
2080
  let bestBytes = Number.POSITIVE_INFINITY;
2393
2081
  for (let start = 0; start + windowLength <= segments.length; start++) {
2394
- if (!windowFits(segments, start, windowLength)) continue;
2082
+ if (!windowFits(segments, start, windowLength, paged)) continue;
2395
2083
  let bytes = 0;
2396
2084
  for (let at = start; at < start + windowLength; at++) bytes += segments[at].bytes;
2397
2085
  if (bytes < bestBytes) {
@@ -2401,9 +2089,9 @@ function smallestFittingWindow(segments, windowLength) {
2401
2089
  }
2402
2090
  return best;
2403
2091
  }
2404
- function chooseWindow(segments, wantedLength) {
2092
+ function chooseWindow(segments, wantedLength, paged) {
2405
2093
  for (let windowLength = Math.min(wantedLength, segments.length); windowLength >= 2; windowLength--) {
2406
- const window = smallestFittingWindow(segments, windowLength);
2094
+ const window = smallestFittingWindow(segments, windowLength, paged);
2407
2095
  if (window !== null) return window;
2408
2096
  }
2409
2097
  return null;
@@ -2444,11 +2132,39 @@ async function compactPartitionSegments(input) {
2444
2132
  if (segments.length <= compactionThreshold) {
2445
2133
  return { segments, nextSegmentId: input.nextSegmentId };
2446
2134
  }
2447
- const window = chooseWindow(segments, segments.length - compactionThreshold + 1);
2135
+ const paged = input.paged === true;
2136
+ const window = chooseWindow(segments, segments.length - compactionThreshold + 1, paged);
2448
2137
  if (window === null) {
2449
2138
  return { segments, nextSegmentId: input.nextSegmentId };
2450
2139
  }
2451
2140
  const windowRefs = segments.slice(window.start, window.start + window.windowLength);
2141
+ const replacing = (written2) => ({
2142
+ segments: [
2143
+ ...segments.slice(0, window.start),
2144
+ ...written2.segments,
2145
+ ...segments.slice(window.start + window.windowLength)
2146
+ ],
2147
+ nextSegmentId: written2.nextSegmentId
2148
+ });
2149
+ if (paged) {
2150
+ const { directory } = input.store;
2151
+ if (directory === void 0) return { segments, nextSegmentId: input.nextSegmentId };
2152
+ const { mergePagedWindow } = await import('./module-BUFNZMEU.mjs');
2153
+ return replacing(
2154
+ await mergePagedWindow({
2155
+ store: input.store,
2156
+ directory,
2157
+ indexName: input.indexName,
2158
+ partitionId: input.partitionId,
2159
+ schema: input.schema,
2160
+ partialWordAnalysis: input.partialWordAnalysis,
2161
+ textAnalysis: input.textAnalysis,
2162
+ nextSegmentId: input.nextSegmentId,
2163
+ refs: windowRefs,
2164
+ dropsEveryTombstone: window.start === 0
2165
+ })
2166
+ );
2167
+ }
2452
2168
  const contents = [];
2453
2169
  for (const ref of windowRefs) contents.push(await readSegmentFile(input.store, ref));
2454
2170
  const frozen = contents.map(frozenOf);
@@ -2467,14 +2183,7 @@ async function compactPartitionSegments(input) {
2467
2183
  mergesIntoOneFile: true,
2468
2184
  ...writeRecords === void 0 ? {} : { writeRecords }
2469
2185
  });
2470
- return {
2471
- segments: [
2472
- ...segments.slice(0, window.start),
2473
- ...written.segments,
2474
- ...segments.slice(window.start + window.windowLength)
2475
- ],
2476
- nextSegmentId: written.nextSegmentId
2477
- };
2186
+ return replacing(written);
2478
2187
  }
2479
2188
 
2480
2189
  // src/persistence/durability/segment/write.ts
@@ -2521,6 +2230,21 @@ function wholeContentsOf(input, partitionId) {
2521
2230
  const captured = input.capturedPartitions?.find((candidate) => candidate.partitionId === partitionId);
2522
2231
  return captured === void 0 ? void 0 : contentsOfCaptured(captured);
2523
2232
  }
2233
+ function pagedSettingsOf(context) {
2234
+ if (context.input.metadata.textStorage !== "disk") return {};
2235
+ return { paged: true, keepsWriteRecords: context.input.keepsWriteRecords === true };
2236
+ }
2237
+ async function tombstonesOfDelta(context, older, selections, deleted) {
2238
+ const { directory } = context.input.store;
2239
+ if (context.input.metadata.textStorage !== "disk" || directory === void 0) return deleted;
2240
+ const { pagedTombstonesFor } = await import('./module-BUFNZMEU.mjs');
2241
+ return pagedTombstonesFor(
2242
+ directory,
2243
+ older,
2244
+ selections.map((selection) => selection.segment),
2245
+ deleted
2246
+ );
2247
+ }
2524
2248
  async function writeWholePartition(context, partitionId, priorPartition, whole) {
2525
2249
  const written = await writeSegmentsOf({
2526
2250
  store: context.input.store,
@@ -2532,7 +2256,8 @@ async function writeWholePartition(context, partitionId, priorPartition, whole)
2532
2256
  textAnalysis: context.textAnalysis,
2533
2257
  selections: whole.selections,
2534
2258
  tombstones: [],
2535
- ...whole.writeRecords === void 0 ? {} : { writeRecords: whole.writeRecords }
2259
+ ...whole.writeRecords === void 0 ? {} : { writeRecords: whole.writeRecords },
2260
+ ...pagedSettingsOf(context)
2536
2261
  });
2537
2262
  return { partitionId, nextSegmentId: written.nextSegmentId, segments: written.segments };
2538
2263
  }
@@ -2548,6 +2273,7 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
2548
2273
  let segments = priorPartition ? [...priorPartition.segments] : [];
2549
2274
  let nextSegmentId = priorPartition?.nextSegmentId ?? 0;
2550
2275
  if (built !== null) {
2276
+ const tombstones = await tombstonesOfDelta(context, segments, built.selections, built.tombstones);
2551
2277
  const written = await writeSegmentsOf({
2552
2278
  store: context.input.store,
2553
2279
  indexName: context.indexName,
@@ -2557,8 +2283,9 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
2557
2283
  partialWordAnalysis: context.partialWordAnalysis,
2558
2284
  textAnalysis: context.textAnalysis,
2559
2285
  selections: built.selections,
2560
- tombstones: built.tombstones,
2561
- ...built.writeRecords === void 0 ? {} : { writeRecords: built.writeRecords }
2286
+ tombstones,
2287
+ ...built.writeRecords === void 0 ? {} : { writeRecords: built.writeRecords },
2288
+ ...pagedSettingsOf(context)
2562
2289
  });
2563
2290
  segments.push(...written.segments);
2564
2291
  nextSegmentId = written.nextSegmentId;
@@ -2579,7 +2306,8 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
2579
2306
  textAnalysis: context.textAnalysis,
2580
2307
  segments,
2581
2308
  nextSegmentId,
2582
- compactionThreshold: context.compactionThreshold
2309
+ compactionThreshold: context.compactionThreshold,
2310
+ paged: context.input.metadata.textStorage === "disk"
2583
2311
  });
2584
2312
  segments = compacted.segments;
2585
2313
  nextSegmentId = compacted.nextSegmentId;
@@ -2642,8 +2370,10 @@ async function writeCheckpointSegments(input) {
2642
2370
  async function commitCheckpointManifest(store, metadata, written, vectors) {
2643
2371
  const indexName = metadata.indexName;
2644
2372
  const manifest = {
2373
+ version: SEGMENT_MANIFEST_VERSION,
2645
2374
  schema: metadata.schema,
2646
2375
  language: metadata.language,
2376
+ ...metadata.analysisGeneration ? { analysisGeneration: metadata.analysisGeneration } : {},
2647
2377
  checkpoint: written.checkpoint,
2648
2378
  partitions: written.partitions,
2649
2379
  vectors
@@ -2694,8 +2424,41 @@ function removeWhatTheSegmentReplaces(manager, segment, contents) {
2694
2424
  function yieldToEventLoop() {
2695
2425
  return new Promise((resolve) => setTimeout(resolve, 0));
2696
2426
  }
2427
+ function removeFromPartition(manager, partitionId, docIds) {
2428
+ const { composite, docPartitionMap, insertOptions } = manager.diskSegmentAccess(partitionId);
2429
+ composite.beginBatch();
2430
+ try {
2431
+ for (const docId of docIds) {
2432
+ if (!composite.has(docId)) continue;
2433
+ composite.remove(docId, manager.config.schema, manager.language, insertOptions);
2434
+ docPartitionMap.delete(docId);
2435
+ }
2436
+ } finally {
2437
+ composite.endBatch();
2438
+ }
2439
+ }
2440
+ async function attachPagedSegment(directory, ref, partitionId, manager, options) {
2441
+ const { checkPagedSegmentFully, createPagedFrozenSegment, openPagedSnapshot, pagedListingOf } = await import('./module-BUFNZMEU.mjs');
2442
+ if (options.decodesEveryDocument) await checkPagedSegmentFully(directory, ref);
2443
+ const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
2444
+ requireAgreedSchema(options.schema, snapshot.schema, ref.key);
2445
+ const segment = createPagedFrozenSegment(snapshot);
2446
+ removeFromPartition(manager, partitionId, snapshot.tombstones);
2447
+ options.writeRecords?.loadPagedSegment(manager.indexName, partitionId, segment);
2448
+ if (snapshot.documentCount > 0) manager.attachFrozenSegment(partitionId, segment);
2449
+ else segment.paged?.close();
2450
+ }
2697
2451
  async function loadPartitionSegments(store, partition, manager, options) {
2452
+ const { directory } = store;
2698
2453
  for (const ref of partition.segments) {
2454
+ if (manager.config.textStorage === "disk" && directory !== void 0) {
2455
+ const { isPagedSegmentFile } = await import('./module-BUFNZMEU.mjs');
2456
+ if (await isPagedSegmentFile(directory, ref.key)) {
2457
+ await attachPagedSegment(directory, ref, partition.partitionId, manager, options);
2458
+ await yieldToEventLoop();
2459
+ continue;
2460
+ }
2461
+ }
2699
2462
  const contents = await readSegmentFile(store, ref);
2700
2463
  requireAgreedSchema(options.schema, contents.schema, ref.key);
2701
2464
  if (options.decodesEveryDocument) decodeEveryStoredDocument(contents.arrays.documentTable);
@@ -2740,6 +2503,14 @@ async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodes
2740
2503
  if (partition.partitionId > highestPartitionId) highestPartitionId = partition.partitionId;
2741
2504
  }
2742
2505
  growPartitionsTo(deps.manager, highestPartitionId);
2506
+ const { directory } = store;
2507
+ const opensVectorsOnDemand = manifest.vectors.some(
2508
+ (field) => deps.vectorIndexes.get(field.fieldPath)?.storage === "disk" && field.files.some((file) => file.headOffset !== void 0)
2509
+ );
2510
+ if (!decodesEveryDocument && directory !== void 0 && opensVectorsOnDemand) {
2511
+ const { startReadingOpeningSectionsAhead } = await import('./module-BUFNZMEU.mjs');
2512
+ await startReadingOpeningSectionsAhead(directory, manifest);
2513
+ }
2743
2514
  for (const partition of manifest.partitions) {
2744
2515
  if (partition.partitionId >= deps.manager.partitionCount) {
2745
2516
  throw new NarsilError(
@@ -2757,7 +2528,12 @@ async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodes
2757
2528
  for (const vector of manifest.vectors) {
2758
2529
  const vectorIndex = deps.vectorIndexes.get(vector.fieldPath);
2759
2530
  if (vectorIndex === void 0) continue;
2760
- await loadVectorField(store, vector, vectorIndex);
2531
+ await loadVectorField(
2532
+ store,
2533
+ vector,
2534
+ vectorIndex,
2535
+ decodesEveryDocument ? null : (docId) => deps.manager.partitionIdOf(docId)
2536
+ );
2761
2537
  }
2762
2538
  return manifest.checkpoint;
2763
2539
  }
@@ -2866,4 +2642,4 @@ function deserializePayloadV2(data) {
2866
2642
  };
2867
2643
  }
2868
2644
 
2869
- export { CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_IDLE_MS, CHECKPOINT_WORKER_TIMEOUT_MS, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_SEGMENT_MAX_BYTES, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, LOG_RECORDS_PER_CHECKPOINT, MAX_SEGMENTS_PER_PARTITION, SEGMENT_HEADER_SIZE, SINGLE_NODE_PRIMARY_TERM, VECTOR_FILE_ENVELOPE_VERSION, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, adapterCheckpointStore, buildEntry, captureWholePartition, checkpointLastSeqNo, checkpointPrimaryTerm, commitCheckpointManifest, createSeqOwner, decodeManifestPayload, decodeSegmentManifest, deserializePayloadV1, deserializePayloadV2, directoryCheckpointStore, documentsOfPayloadV2, durableRecordMissing, encodeMarkerSlot, encodeSegmentManifest, frameRecord, growPartitionsTo, indexMetadataOf, invalidManifest, isId, isRecord, isSequenceNumber, isUint32, listedKeysOf, loadSegmentedCheckpoint, manifestFileRefs, manifestKey, manifestReferencedKeys, markerSlotOffset, metadataConfigOf, metadataFromConfig, nextMarkerSlot, openManifestPayload, partitionContentsOf, readCommitMarker, readCurrentManifest, readGraphParts, readListedFile, readMetadataEnvelope, readSegmentFile, readSegmentManifest, readVectorFile, readWalSegments, reclaimUnlistedKeys, removeCheckpointGarbage, snapshotBundleKey, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, writeCheckpointSegments, writeLiveVectors, writeMetadataEnvelope, writeSegmentHeader };
2645
+ export { SEGMENT_HEADER_SIZE, SINGLE_NODE_PRIMARY_TERM, VECTOR_FILE_ENVELOPE_VERSION, buildEntry, captureWholePartition, checkpointLastSeqNo, checkpointPrimaryTerm, commitCheckpointManifest, createSeqOwner, decodeManifestPayload, decodeSegmentManifest, deserializePayloadV1, deserializePayloadV2, documentsOfPayloadV2, durableMetadataConfigOf, durableRecordMissing, encodeMarkerSlot, frameRecord, growPartitionsTo, indexMetadataOf, invalidManifest, isId, isRecord, isSequenceNumber, isUint32, listedKeysOf, loadSegmentedCheckpoint, markerSlotOffset, metadataConfigOf, metadataFromConfig, nextMarkerSlot, nextSegmentKey, openManifestPayload, partitionContentsOf, readCommitMarker, readCurrentManifest, readGraphParts, readMetadataEnvelope, readSegmentManifest, readVectorFile, readWalSegments, reclaimUnlistedKeys, removeCheckpointGarbage, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, writeCheckpointSegments, writeLiveVectors, writeMetadataEnvelope, writeSegmentHeader };