@delali/narsil 0.3.1-next.131 → 0.3.1-next.141

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/dist/adapters/filesystem.mjs +1 -1
  2. package/dist/adapters/indexeddb.mjs +1 -1
  3. package/dist/analysis-change-HQUT2Z3I.mjs +43 -0
  4. package/dist/analysis-change-entry-TZTG4AZB.mjs +1 -0
  5. package/dist/browser/analysis-change-JPV775LR.mjs +900 -0
  6. package/dist/browser/analysis-change-entry-7HR4LWL3.mjs +1 -0
  7. package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
  8. package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
  9. package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-5DB2VDGI.mjs} +1149 -1853
  10. package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
  11. package/dist/browser/chunk-CCOAOO73.mjs +86 -0
  12. package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-D33A5VMU.mjs} +16 -16
  13. package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-HKX2G6WH.mjs} +1520 -2159
  14. package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-KTOK275N.mjs} +606 -830
  15. package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-LVY2LI35.mjs} +2 -2
  16. package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
  17. package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
  18. package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
  19. package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
  20. package/dist/browser/chunk-R4WUDZQV.mjs +176 -0
  21. package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-TCLEOZ4P.mjs} +3 -3
  22. package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
  23. package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
  24. package/dist/browser/chunk-XNWPP5NS.mjs +21 -0
  25. package/dist/browser/chunk-ZEV26AQ6.mjs +270 -0
  26. package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
  27. package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-SNOWIT6Z.mjs} +370 -95
  28. package/dist/browser/import-TLB2A5LJ.mjs +9 -0
  29. package/dist/browser/module-BUFNZMEU.mjs +12 -0
  30. package/dist/browser/{module-4HIMOQM5.mjs → module-BVP4TKZU.mjs} +397 -109
  31. package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
  32. package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-ONYNYUDE.mjs} +8 -2
  33. package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
  34. package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-QXBKHJAG.mjs} +135 -183
  35. package/dist/browser/{write-YIC2MKRY.mjs → write-WHFBDWQS.mjs} +64 -90
  36. package/dist/{capabilities-B6WxrWxt.d.ts → capabilities-BCdUwK_L.d.ts} +10 -3
  37. package/dist/chunk-46NZOPHS.mjs +21 -0
  38. package/dist/{chunk-JT3YZGCE.mjs → chunk-5WYVUPGD.mjs} +3 -188
  39. package/dist/chunk-6SNBQESJ.mjs +214 -0
  40. package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
  41. package/dist/chunk-7AUASRE7.mjs +653 -0
  42. package/dist/{chunk-NMXRXFGZ.mjs → chunk-AST3SHFA.mjs} +27 -25
  43. package/dist/chunk-BOJZGUS6.mjs +177 -0
  44. package/dist/{chunk-T4OFG4IQ.mjs → chunk-BRUUBXCB.mjs} +92 -17
  45. package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
  46. package/dist/chunk-CZQ33DLW.mjs +78 -0
  47. package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
  48. package/dist/{chunk-EUCX5K74.mjs → chunk-DQNKVOJ2.mjs} +65 -65
  49. package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
  50. package/dist/chunk-FOCV4CBZ.mjs +87 -0
  51. package/dist/{chunk-HCGJKDZV.mjs → chunk-GLZHNSGB.mjs} +1 -1
  52. package/dist/chunk-GWOS5DYC.mjs +2637 -0
  53. package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
  54. package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
  55. package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
  56. package/dist/{chunk-TGHNXN7W.mjs → chunk-IO5F7ZBE.mjs} +261 -948
  57. package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
  58. package/dist/{chunk-7OIQ2SIO.mjs → chunk-KEIYVZUX.mjs} +5 -5
  59. package/dist/{chunk-PU6SKH7O.mjs → chunk-KGOXHLLZ.mjs} +7 -3
  60. package/dist/{chunk-WVRDRLIC.mjs → chunk-KZL72VTC.mjs} +70 -18
  61. package/dist/chunk-L76W3IZJ.mjs +193 -0
  62. package/dist/{chunk-2JWNJSNE.mjs → chunk-LA4WNDHL.mjs} +88 -801
  63. package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
  64. package/dist/{chunk-JJ6EUMXB.mjs → chunk-NJJCPFXN.mjs} +2 -2
  65. package/dist/chunk-NN33CZRE.mjs +86 -0
  66. package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
  67. package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
  68. package/dist/chunk-NRUXBYPH.mjs +121 -0
  69. package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
  70. package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
  71. package/dist/{chunk-F6AAXI7T.mjs → chunk-RHKRTYGO.mjs} +13 -4
  72. package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
  73. package/dist/{chunk-RCMNL7RI.mjs → chunk-SBMAWTRG.mjs} +41 -4
  74. package/dist/chunk-SRJBFG2T.mjs +881 -0
  75. package/dist/{chunk-X5CNUKD5.mjs → chunk-TNBIK2LQ.mjs} +1663 -1652
  76. package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
  77. package/dist/chunk-UUZCQQG2.mjs +209 -0
  78. package/dist/{chunk-6M6SDH3S.mjs → chunk-VEFN6X6Q.mjs} +8 -8
  79. package/dist/chunk-VJEGOELZ.mjs +177 -0
  80. package/dist/chunk-W2COJ5AH.mjs +148 -0
  81. package/dist/{chunk-RWESFSJI.mjs → chunk-WJGO5PGV.mjs} +317 -174
  82. package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
  83. package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
  84. package/dist/{chunk-Y4BVTQ53.mjs → chunk-XZ3SRNDJ.mjs} +25 -64
  85. package/dist/{chunk-K77FJXKS.mjs → chunk-YHUTF6EA.mjs} +474 -93
  86. package/dist/{chunk-OGLV4I3L.mjs → chunk-YLNSJQLX.mjs} +12 -7
  87. package/dist/{chunk-SOHMMNCH.mjs → chunk-Z4HFMYNX.mjs} +32 -27
  88. package/dist/{chunk-5W7AEVTH.mjs → chunk-ZMBFWQXN.mjs} +2 -2
  89. package/dist/{chunk-2MKX2SCO.mjs → chunk-ZPWS2SKJ.mjs} +297 -274
  90. package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
  91. package/dist/{client-gNAfaHcE.d.ts → client-B12YFALP.d.ts} +18 -3
  92. package/dist/client.d.ts +7 -7
  93. package/dist/client.mjs +11 -1
  94. package/dist/cold-module-4X7WJCFM.mjs +783 -0
  95. package/dist/{config-D1cDv3sE.d.ts → config-BhZxIeb1.d.ts} +10 -1
  96. package/dist/{config-CmYJO34X.d.ts → config-CIG5yDyg.d.ts} +2 -2
  97. package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-7XOQ7VQP.mjs} +14 -14
  98. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  99. package/dist/distribution/coordinator/etcd.mjs +1 -1
  100. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  101. package/dist/distribution/transport/grpc.mjs +1 -1
  102. package/dist/distribution.d.ts +24 -8
  103. package/dist/distribution.mjs +982 -211
  104. package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-324D2DDD.mjs} +362 -94
  105. package/dist/embeddings/openai.mjs +2 -2
  106. package/dist/{engine-m_Ddr1bz.d.ts → engine-CtzbHOmG.d.ts} +64 -7
  107. package/dist/import-OOBUW3RR.mjs +44 -0
  108. package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
  109. package/dist/index.browser.mjs +5 -5
  110. package/dist/index.d.ts +6 -6
  111. package/dist/index.mjs +22 -22
  112. package/dist/{long-query-thread-BSL25PD6.mjs → long-query-thread-4SW7RWDE.mjs} +15 -15
  113. package/dist/{module-2E4XYSGT.mjs → module-DXSJ5LT2.mjs} +378 -87
  114. package/dist/module-IYPLWHGC.mjs +48 -0
  115. package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
  116. package/dist/partial-words.mjs +5 -5
  117. package/dist/pattern.mjs +3 -3
  118. package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
  119. package/dist/persistence/durability/checkpoint-worker.mjs +28 -26
  120. package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
  121. package/dist/react.d.ts +9 -9
  122. package/dist/react.mjs +4 -0
  123. package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
  124. package/dist/rerankers/http.mjs +2 -2
  125. package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-FEYAC4SB.mjs} +8 -2
  126. package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
  127. package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
  128. package/dist/server.d.ts +12 -10
  129. package/dist/server.mjs +54 -21
  130. package/dist/{snapshot-DOQLGGEX.mjs → snapshot-CGM65VCD.mjs} +101 -197
  131. package/dist/synonyms.mjs +6 -4
  132. package/dist/{tasks-Co8HfRD7.d.ts → tasks-CjRJRGTb.d.ts} +26 -8
  133. package/dist/{thread-SKO7CN3R.mjs → thread-KGNAIKPD.mjs} +14 -14
  134. package/dist/vector/search-worker.d.ts +1 -1
  135. package/dist/vector/search-worker.mjs +6 -5
  136. package/dist/workers/entry.mjs +8 -8
  137. package/dist/{write-ZKR7SNRT.mjs → write-2DZPEQL2.mjs} +82 -107
  138. package/openapi.json +114 -6
  139. package/package.json +12 -8
@@ -1,13 +1,16 @@
1
- import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-Y2U5VVWX.mjs';
2
- import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, readSegmentFile, manifestKey, segmentsPrefix, readListedFile, requireListedChecksum, vectorFileKey, graphPartKey } from './chunk-2JWNJSNE.mjs';
3
- import { VERSION, rulesOfStoredSynonyms, openChecksummedEnvelope, readHeader, HEADER_SIZE, crc32, engineVersionParts, writeHeader, packChecksummedEnvelope } from './chunk-TGHNXN7W.mjs';
4
- import { decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays } from './chunk-RWESFSJI.mjs';
5
- import { DEFAULT_BM25_B, DEFAULT_BM25_K1 } from './chunk-2MKX2SCO.mjs';
6
- import { flattenSchema, requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-PU6SKH7O.mjs';
1
+ import { loadVectorField, VECTOR_FILE_ENVELOPE_VERSION, GRAPH_PART_ENVELOPE_VERSION } from './chunk-NRUXBYPH.mjs';
2
+ import { SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES } from './chunk-W2COJ5AH.mjs';
3
+ import { VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-ICW45TTU.mjs';
4
+ import { readSegmentFile, STABLE_HEAD_ATTEMPTS } from './chunk-7AUASRE7.mjs';
5
+ import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, manifestKey, segmentsPrefix, vectorFileKey, graphPartKey } from './chunk-UUZCQQG2.mjs';
6
+ import { VERSION, rulesOfStoredSynonyms, openChecksummedEnvelope, readHeader, HEADER_SIZE, crc32, engineVersionParts, writeHeader, packChecksummedEnvelope, packEnvelopeWithChecksum, checksumOfChunks } from './chunk-IO5F7ZBE.mjs';
7
+ import { decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays } from './chunk-WJGO5PGV.mjs';
8
+ import { DEFAULT_BM25_B, DEFAULT_BM25_K1 } from './chunk-ZPWS2SKJ.mjs';
9
+ import { flattenSchema, requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-KGOXHLLZ.mjs';
7
10
  import { MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-U476IXKQ.mjs';
8
11
  import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
9
12
  import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
10
- import { NarsilError, ErrorCodes } from './chunk-CZ7ISLXA.mjs';
13
+ import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
11
14
  import { decode, encode } from '@msgpack/msgpack';
12
15
 
13
16
  // src/engine/durability-metadata.ts
@@ -47,6 +50,12 @@ function metadataFromConfig(indexName, config, partitionCount, documentCount) {
47
50
  if (config.stopWordList !== void 0) {
48
51
  metadata.stopWordList = config.stopWordList;
49
52
  }
53
+ if (config.analysisGeneration !== void 0) {
54
+ metadata.analysisGeneration = config.analysisGeneration;
55
+ }
56
+ if (config.pendingAnalysis !== void 0) {
57
+ metadata.pendingAnalysis = config.pendingAnalysis;
58
+ }
50
59
  if (config.partitionLimits !== void 0) {
51
60
  metadata.partitionLimits = config.partitionLimits;
52
61
  }
@@ -56,6 +65,9 @@ function metadataFromConfig(indexName, config, partitionCount, documentCount) {
56
65
  if (config.trackPositions !== void 0) {
57
66
  metadata.trackPositions = config.trackPositions;
58
67
  }
68
+ if (config.textStorage !== void 0) {
69
+ metadata.textStorage = config.textStorage;
70
+ }
59
71
  if (config.strict !== void 0) {
60
72
  metadata.strict = config.strict;
61
73
  }
@@ -101,9 +113,12 @@ function metadataConfigOf(entry) {
101
113
  ...typeof entry.config.tokenizer === "string" ? { tokenizer: entry.config.tokenizer } : {},
102
114
  ...typeof entry.config.stopWords === "string" ? { stopWords: entry.config.stopWords } : {},
103
115
  ...entry.config.stopWords instanceof Set ? { stopWordList: [...entry.config.stopWords].sort(compareCodePoints) } : {},
116
+ ...entry.analysisGeneration > 0 ? { analysisGeneration: entry.analysisGeneration } : {},
117
+ ...entry.pendingAnalysis !== null ? { pendingAnalysis: entry.pendingAnalysis } : {},
104
118
  ...entry.config.partitions !== void 0 ? { partitionLimits: entry.config.partitions } : {},
105
119
  ...entry.config.defaultScoring !== void 0 ? { defaultScoring: entry.config.defaultScoring } : {},
106
120
  ...entry.config.trackPositions !== void 0 ? { trackPositions: entry.config.trackPositions } : {},
121
+ ...entry.config.textStorage !== void 0 ? { textStorage: entry.config.textStorage } : {},
107
122
  ...entry.config.strict !== void 0 ? { strict: entry.config.strict } : {},
108
123
  ...entry.config.required !== void 0 ? { required: entry.config.required } : {},
109
124
  ...entry.config.vectorPromotion !== void 0 ? { vectorPromotion: entry.config.vectorPromotion } : {},
@@ -111,6 +126,38 @@ function metadataConfigOf(entry) {
111
126
  ...entry.synonyms !== null ? { synonyms: rulesOfStoredSynonyms(entry.synonyms) } : {}
112
127
  };
113
128
  }
129
+ function durableMetadataConfigOf(entry) {
130
+ const terms = metadataConfigOf(entry);
131
+ const durable = entry.durableAnalysis;
132
+ if (durable === null) return terms;
133
+ const {
134
+ language,
135
+ tokenizer,
136
+ stopWords,
137
+ stopWordList,
138
+ analysisRevision: _analysisRevision,
139
+ arrayPositionGap: _arrayPositionGap,
140
+ analysisGeneration: _analysisGeneration,
141
+ ...rest
142
+ } = terms;
143
+ return {
144
+ ...rest,
145
+ language: durable.language,
146
+ ...durable.tokenizer !== void 0 ? { tokenizer: durable.tokenizer } : {},
147
+ ...durable.stopWords !== void 0 ? { stopWords: durable.stopWords } : {},
148
+ ...durable.stopWordList !== void 0 ? { stopWordList: durable.stopWordList } : {},
149
+ ...durable.analysisRevision !== null ? { analysisRevision: durable.analysisRevision } : {},
150
+ ...durable.arrayPositionGap !== null ? { arrayPositionGap: durable.arrayPositionGap } : {},
151
+ ...durable.generation > 0 ? { analysisGeneration: durable.generation } : {},
152
+ pendingAnalysis: {
153
+ generation: entry.analysisGeneration,
154
+ language,
155
+ ...tokenizer !== void 0 ? { tokenizer } : {},
156
+ ...stopWords !== void 0 ? { stopWords } : {},
157
+ ...stopWordList !== void 0 ? { stopWordList } : {}
158
+ }
159
+ };
160
+ }
114
161
  function indexMetadataOf(indexName, entry, partitionCount) {
115
162
  return metadataFromConfig(indexName, metadataConfigOf(entry), partitionCount);
116
163
  }
@@ -322,6 +369,19 @@ function metadataToWire(meta) {
322
369
  if (meta.stopWordList !== void 0) {
323
370
  wire.stop_word_list = meta.stopWordList;
324
371
  }
372
+ if (meta.analysisGeneration !== void 0) {
373
+ wire.analysis_generation = meta.analysisGeneration;
374
+ }
375
+ if (meta.pendingAnalysis !== void 0) {
376
+ const pending = meta.pendingAnalysis;
377
+ wire.pending_analysis = {
378
+ generation: pending.generation,
379
+ language: pending.language,
380
+ ...pending.tokenizer !== void 0 ? { tokenizer: pending.tokenizer } : {},
381
+ ...pending.stopWords !== void 0 ? { stop_words: pending.stopWords } : {},
382
+ ...pending.stopWordList !== void 0 ? { stop_word_list: pending.stopWordList } : {}
383
+ };
384
+ }
325
385
  if (meta.partitionLimits !== void 0) {
326
386
  wire.partition_limits = {
327
387
  ...meta.partitionLimits.maxDocsPerPartition !== void 0 ? { max_docs_per_partition: meta.partitionLimits.maxDocsPerPartition } : {},
@@ -335,6 +395,9 @@ function metadataToWire(meta) {
335
395
  if (meta.trackPositions !== void 0) {
336
396
  wire.track_positions = meta.trackPositions;
337
397
  }
398
+ if (meta.textStorage !== void 0) {
399
+ wire.text_storage = meta.textStorage;
400
+ }
338
401
  if (meta.strict !== void 0) {
339
402
  wire.strict = meta.strict;
340
403
  }
@@ -417,6 +480,19 @@ function wireToMetadata(raw) {
417
480
  if (Array.isArray(raw.stop_word_list) && raw.stop_word_list.every((word) => typeof word === "string")) {
418
481
  meta.stopWordList = raw.stop_word_list;
419
482
  }
483
+ if (isPositiveInteger(raw.analysis_generation)) {
484
+ meta.analysisGeneration = raw.analysis_generation;
485
+ }
486
+ const pending = raw.pending_analysis;
487
+ if (pending && isPositiveInteger(pending.generation) && typeof pending.language === "string") {
488
+ meta.pendingAnalysis = {
489
+ generation: pending.generation,
490
+ language: pending.language,
491
+ ...typeof pending.tokenizer === "string" ? { tokenizer: pending.tokenizer } : {},
492
+ ...typeof pending.stop_words === "string" ? { stopWords: pending.stop_words } : {},
493
+ ...isStringList(pending.stop_word_list) ? { stopWordList: pending.stop_word_list } : {}
494
+ };
495
+ }
420
496
  if (raw.partition_limits && typeof raw.partition_limits === "object") {
421
497
  const limits = {
422
498
  ...isPositiveInteger(raw.partition_limits.max_docs_per_partition) ? { maxDocsPerPartition: raw.partition_limits.max_docs_per_partition } : {},
@@ -433,6 +509,14 @@ function wireToMetadata(raw) {
433
509
  if (typeof raw.track_positions === "boolean") {
434
510
  meta.trackPositions = raw.track_positions;
435
511
  }
512
+ if (raw.text_storage !== void 0) {
513
+ if (raw.text_storage !== "memory" && raw.text_storage !== "disk") {
514
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, "text_storage must be 'memory' or 'disk'", {
515
+ textStorage: raw.text_storage
516
+ });
517
+ }
518
+ meta.textStorage = raw.text_storage;
519
+ }
436
520
  if (typeof raw.strict === "boolean") {
437
521
  meta.strict = raw.strict;
438
522
  }
@@ -509,6 +593,9 @@ function wireToMetadata(raw) {
509
593
  function isNonNegativeInteger(value) {
510
594
  return typeof value === "number" && Number.isInteger(value) && value >= 0;
511
595
  }
596
+ function isStringList(value) {
597
+ return Array.isArray(value) && value.every((word) => typeof word === "string");
598
+ }
512
599
  function isPositiveInteger(value) {
513
600
  return typeof value === "number" && Number.isInteger(value) && value >= 1;
514
601
  }
@@ -691,7 +778,20 @@ function readFileRef(raw, expectedKey, what) {
691
778
  invalidManifest(`gives the ${what} "${expectedKey}" an invalid length`, { key: expectedKey, bytes });
692
779
  }
693
780
  if (!isUint32(raw.checksum)) invalidManifest(`gives the ${what} "${expectedKey}" an invalid checksum`);
694
- return { key: expectedKey, bytes, checksum: raw.checksum };
781
+ if (raw.headOffset === void 0) return { key: expectedKey, bytes, checksum: raw.checksum };
782
+ const { headOffset, headLength, headChecksum } = raw;
783
+ if (!isSequenceNumber(headOffset) || !isUint32(headLength) || !isUint32(headChecksum)) {
784
+ invalidManifest(`gives the ${what} "${expectedKey}" an invalid head`, { key: expectedKey });
785
+ }
786
+ if (headOffset + headLength > bytes - HEADER_SIZE) {
787
+ invalidManifest(`ends the head of the ${what} "${expectedKey}" outside its payload`, {
788
+ key: expectedKey,
789
+ headOffset,
790
+ headLength,
791
+ bytes
792
+ });
793
+ }
794
+ return { key: expectedKey, bytes, checksum: raw.checksum, headOffset, headLength, headChecksum };
695
795
  }
696
796
 
697
797
  // src/persistence/durability/segment/manifest-vectors-decode.ts
@@ -788,12 +888,16 @@ function decodeManifestPayload(raw, indexName) {
788
888
  if (!isRecord(schema)) invalidManifest("is missing its schema");
789
889
  requireValidFieldTypes(schema);
790
890
  if (typeof raw.language !== "string") invalidManifest("is missing its language");
891
+ if (raw.analysisGeneration !== void 0 && !isSequenceNumber(raw.analysisGeneration)) {
892
+ invalidManifest("has an invalid analysis generation");
893
+ }
791
894
  return {
792
895
  kind: "current",
793
896
  manifest: {
794
897
  version: SEGMENT_MANIFEST_VERSION,
795
898
  schema,
796
899
  language: raw.language,
900
+ ...raw.analysisGeneration ? { analysisGeneration: raw.analysisGeneration } : {},
797
901
  checkpoint: normalizeCheckpoint(raw.checkpoint),
798
902
  partitions: normalizePartitions(raw.partitions, indexName),
799
903
  vectors: normalizeVectorFields(raw.vectors, indexName)
@@ -878,100 +982,313 @@ function readNextSegmentId(value, segments, partitionId) {
878
982
  }
879
983
  return value;
880
984
  }
881
- var VECTOR_FILE_ENVELOPE_VERSION = 2;
882
- var GRAPH_PART_ENVELOPE_VERSION = 2;
883
- function liveVectorsOf(ref) {
884
- let live = 0;
885
- for (const file of ref.files) live += file.count - countDead(file.dead, file.count);
886
- return live;
887
- }
888
- async function openListedFile(store, ref, what, allowsCompression) {
889
- const bytes = await readListedFile(store, ref.key, { bytes: ref.bytes, what });
890
- const { header, payload } = await openChecksummedEnvelope(bytes, {
891
- envelopeFormatVersion: VECTOR_FILE_ENVELOPE_VERSION,
892
- what: `${what} "${ref.key}"`,
893
- allowsCompression
894
- });
895
- requireListedChecksum(what, ref.key, header.checksum, ref.checksum);
896
- return { payload, payloadLength: header.payloadLength };
897
- }
898
- async function readGraphParts(store, ref) {
899
- if (ref.graphParts.length === 0) return null;
900
- const parts = [];
901
- for (const part of ref.graphParts) {
902
- const { payload } = await openListedFile(store, part, "graph part", true);
903
- const decoded = decodeVectorGraphPartPayload(decode(payload));
904
- if (decoded.fieldName !== ref.fieldPath) {
905
- throw new NarsilError(
906
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
907
- `The graph part "${part.key}" names field "${decoded.fieldName}" where the manifest lists "${ref.fieldPath}"`,
908
- { key: part.key }
909
- );
910
- }
911
- parts.push(decoded);
912
- }
913
- return assembleGraphParts(parts);
985
+
986
+ // src/serialization/crc32-combine.ts
987
+ var IEEE_POLYNOMIAL = 3988292384;
988
+ var CRC_BITS = 32;
989
+ var zeroByteOperators = [];
990
+ function timesVector(operator, vector) {
991
+ let sum = 0;
992
+ let rest = vector >>> 0;
993
+ for (let row = 0; rest !== 0; row++, rest >>>= 1) {
994
+ if (rest & 1) sum ^= operator[row];
995
+ }
996
+ return sum >>> 0;
997
+ }
998
+ function squared(operator) {
999
+ const result = new Uint32Array(CRC_BITS);
1000
+ for (let row = 0; row < CRC_BITS; row++) result[row] = timesVector(operator, operator[row]);
1001
+ return result;
914
1002
  }
915
- async function readVectorFile(store, file, fieldPath) {
916
- const { payload: bytes, payloadLength } = await openListedFile(store, file, "vector file", false);
917
- const payload = decodeVectorFilePayload(decode(bytes));
918
- if (payload.docIds.length !== file.count) {
919
- throw new NarsilError(
920
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
921
- `The vector file "${file.key}" holds ${payload.docIds.length} vectors where the manifest counts ${file.count}`,
922
- { key: file.key, held: payload.docIds.length, count: file.count }
923
- );
924
- }
925
- if (payload.fieldName !== fieldPath) {
926
- throw new NarsilError(
927
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
928
- `The vector file "${file.key}" names field "${payload.fieldName}" where the manifest lists "${fieldPath}"`,
929
- { key: file.key }
930
- );
1003
+ function firstOperator() {
1004
+ let oneZeroBit = new Uint32Array(CRC_BITS);
1005
+ oneZeroBit[0] = IEEE_POLYNOMIAL;
1006
+ for (let row = 1; row < CRC_BITS; row++) oneZeroBit[row] = 1 << row - 1;
1007
+ for (let doubling = 0; doubling < 3; doubling++) oneZeroBit = squared(oneZeroBit);
1008
+ return oneZeroBit;
1009
+ }
1010
+ function operatorForZeroBytesPower(power) {
1011
+ if (zeroByteOperators.length === 0) zeroByteOperators.push(firstOperator());
1012
+ while (zeroByteOperators.length <= power) {
1013
+ zeroByteOperators.push(squared(zeroByteOperators[zeroByteOperators.length - 1]));
931
1014
  }
932
- return { payload, payloadLength };
1015
+ return zeroByteOperators[power];
933
1016
  }
934
- async function loadVectorField(store, ref, vectorIndex) {
935
- const restore = vectorIndex.restoreCheckpoint({
936
- liveVectors: liveVectorsOf(ref),
937
- holdsGraph: ref.graphParts.length > 0,
938
- graph: await readGraphParts(store, ref)
939
- });
940
- for (const file of ref.files) {
941
- const { payload, payloadLength } = await readVectorFile(store, file, ref.fieldPath);
942
- restore.addFile({
943
- id: file.id,
944
- key: file.key,
945
- payload,
946
- dead: file.dead,
947
- location: store.pathOf === void 0 ? null : {
948
- path: await store.pathOf(file.key),
949
- vectorsOffset: HEADER_SIZE + payloadLength - payload.vectors.byteLength
950
- }
951
- });
1017
+ function crc32Combine(first, second, secondByteLength) {
1018
+ let crc = first >>> 0;
1019
+ let rest = secondByteLength;
1020
+ for (let power = 0; rest > 0; power++) {
1021
+ if (rest % 2 === 1) crc = timesVector(operatorForZeroBytesPower(power), crc);
1022
+ rest = Math.floor(rest / 2);
952
1023
  }
953
- restore.finish();
1024
+ return (crc ^ second) >>> 0;
954
1025
  }
955
- var EMPTY_BIN = new Uint8Array(0);
956
- var EMPTY_BIN_BYTES = 2;
957
1026
  var BIN8 = 196;
958
1027
  var BIN16 = 197;
959
1028
  var BIN32 = 198;
1029
+ var ARRAY16 = 220;
1030
+ var ARRAY32 = 221;
1031
+ var FIXMAP = 128;
1032
+ var FIXARRAY = 144;
1033
+ var FIXED_LIST_MAX = 15;
1034
+ var FIXSTR = 160;
1035
+ var FIXSTR_MAX_LENGTH = 31;
1036
+ var ASCII_MAX = 127;
1037
+ var UTF8_MAX_BYTES_PER_UNIT = 3;
1038
+ var STRING_HEADER_MAX_BYTES = 5;
960
1039
  var BIN8_MAX_BYTES = 255;
961
- var BIN16_MAX_BYTES = 65535;
1040
+ var UINT16_MAX = 65535;
962
1041
  function binHeader(byteLength) {
963
1042
  if (byteLength <= BIN8_MAX_BYTES) return Uint8Array.of(BIN8, byteLength);
964
- if (byteLength <= BIN16_MAX_BYTES) return Uint8Array.of(BIN16, byteLength >> 8, byteLength & 255);
1043
+ if (byteLength <= UINT16_MAX) return Uint8Array.of(BIN16, byteLength >> 8, byteLength & 255);
965
1044
  const header = new Uint8Array(5);
966
1045
  header[0] = BIN32;
967
1046
  new DataView(header.buffer).setUint32(1, byteLength, false);
968
1047
  return header;
969
1048
  }
1049
+ function arrayHeader(count) {
1050
+ if (count <= FIXED_LIST_MAX) return Uint8Array.of(FIXARRAY | count);
1051
+ if (count <= UINT16_MAX) return Uint8Array.of(ARRAY16, count >> 8, count & 255);
1052
+ const header = new Uint8Array(5);
1053
+ header[0] = ARRAY32;
1054
+ new DataView(header.buffer).setUint32(1, count, false);
1055
+ return header;
1056
+ }
1057
+ function createPayloadBuilder() {
1058
+ const chunks = [];
1059
+ let offset = 0;
1060
+ function push(chunk) {
1061
+ chunks.push(chunk);
1062
+ offset += chunk.byteLength;
1063
+ }
1064
+ return {
1065
+ chunks,
1066
+ get offset() {
1067
+ return offset;
1068
+ },
1069
+ push,
1070
+ entry(key, value) {
1071
+ push(encode(key));
1072
+ push(encode(value));
1073
+ },
1074
+ rawEntry(key, encoded) {
1075
+ push(encode(key));
1076
+ const at = offset;
1077
+ push(encoded);
1078
+ return at;
1079
+ },
1080
+ bytesEntry(key, bytes) {
1081
+ push(encode(key));
1082
+ push(binHeader(bytes.byteLength));
1083
+ const at = offset;
1084
+ push(bytes);
1085
+ return at;
1086
+ }
1087
+ };
1088
+ }
1089
+ function pageChecksums(bytes, pageBytes) {
1090
+ const pages = Math.ceil(bytes.byteLength / pageBytes);
1091
+ const checksums = new Uint8Array(pages * 4);
1092
+ const view = new DataView(checksums.buffer);
1093
+ for (let page = 0; page < pages; page++) {
1094
+ const from = page * pageBytes;
1095
+ view.setUint32(page * 4, crc32(bytes.subarray(from, Math.min(from + pageBytes, bytes.byteLength))), true);
1096
+ }
1097
+ return checksums;
1098
+ }
1099
+ function sectionOf(name, offset, bytes, pageBytes) {
1100
+ return { name, offset, length: bytes.byteLength, pageBytes, checksums: pageChecksums(bytes, pageBytes) };
1101
+ }
1102
+ function uint32Bytes(values) {
1103
+ const bytes = new Uint8Array(values.length * 4);
1104
+ const view = new DataView(bytes.buffer);
1105
+ for (let i = 0; i < values.length; i++) view.setUint32(i * 4, values[i], true);
1106
+ return bytes;
1107
+ }
1108
+ function writtenAsShortAscii(docId, list, at) {
1109
+ if (docId.length > FIXSTR_MAX_LENGTH) return false;
1110
+ for (let unit = 0; unit < docId.length; unit++) {
1111
+ const code = docId.charCodeAt(unit);
1112
+ if (code > ASCII_MAX) return false;
1113
+ list[at + 1 + unit] = code;
1114
+ }
1115
+ list[at] = FIXSTR | docId.length;
1116
+ return true;
1117
+ }
1118
+ function encodedIdList(docIds) {
1119
+ const header = arrayHeader(docIds.length);
1120
+ let capacity = header.byteLength;
1121
+ for (const docId of docIds) capacity += docId.length * UTF8_MAX_BYTES_PER_UNIT + STRING_HEADER_MAX_BYTES;
1122
+ const list = new Uint8Array(capacity);
1123
+ list.set(header);
1124
+ const offsets = new Uint32Array(docIds.length + 1);
1125
+ let cursor = header.byteLength;
1126
+ for (let i = 0; i < docIds.length; i++) {
1127
+ offsets[i] = cursor;
1128
+ if (writtenAsShortAscii(docIds[i], list, cursor)) {
1129
+ cursor += docIds[i].length + 1;
1130
+ continue;
1131
+ }
1132
+ const encoded = encode(docIds[i]);
1133
+ list.set(encoded, cursor);
1134
+ cursor += encoded.byteLength;
1135
+ }
1136
+ offsets[docIds.length] = cursor;
1137
+ return { list: list.subarray(0, cursor), offsets };
1138
+ }
1139
+ function positionsInIdOrder(docIds) {
1140
+ const positions = Array.from(docIds.keys());
1141
+ positions.sort((left, right) => compareCodePoints(docIds[left], docIds[right]));
1142
+ return Uint32Array.from(positions);
1143
+ }
1144
+ function magnitudeBytes(magnitudes) {
1145
+ const bytes = new Uint8Array(magnitudes.length * 8);
1146
+ const view = new DataView(bytes.buffer);
1147
+ for (let i = 0; i < magnitudes.length; i++) view.setFloat64(i * 8, magnitudes[i], true);
1148
+ return bytes;
1149
+ }
1150
+ function checksumAcrossPages(checksums, byteLength, pageBytes) {
1151
+ const view = new DataView(checksums.buffer, checksums.byteOffset, checksums.byteLength);
1152
+ let whole = 0;
1153
+ for (let page = 0; page * 4 < checksums.byteLength; page++) {
1154
+ const pageLength = Math.min(pageBytes, byteLength - page * pageBytes);
1155
+ whole = crc32Combine(whole, view.getUint32(page * 4, true), pageLength);
1156
+ }
1157
+ return whole;
1158
+ }
1159
+ function encodedHeadAndVectorsPlace(sections, headOffsetAfterHeader, vectors, checksums, vectorsKeyAndHeaderBytes) {
1160
+ let headLength = 0;
1161
+ let head = new Uint8Array(0);
1162
+ let vectorsSection = {
1163
+ name: "vectors",
1164
+ offset: 0,
1165
+ length: vectors.byteLength,
1166
+ pageBytes: SEGMENT_PAGE_BYTES,
1167
+ checksums
1168
+ };
1169
+ for (let attempt = 0; attempt < STABLE_HEAD_ATTEMPTS; attempt++) {
1170
+ vectorsSection = {
1171
+ ...vectorsSection,
1172
+ offset: headOffsetAfterHeader(headLength) + headLength + vectorsKeyAndHeaderBytes
1173
+ };
1174
+ head = encode({ sections: [...sections, vectorsSection] });
1175
+ if (head.byteLength === headLength) break;
1176
+ headLength = head.byteLength;
1177
+ }
1178
+ return { head, vectorsSection };
1179
+ }
1180
+ function pagedVectorFileChunks(payload, heldMagnitudes) {
1181
+ const builder = createPayloadBuilder();
1182
+ const { codes } = payload;
1183
+ builder.push(Uint8Array.of(FIXMAP | 10));
1184
+ builder.entry("v", payload.v);
1185
+ builder.entry("fieldName", payload.fieldName);
1186
+ builder.entry("dimension", payload.dimension);
1187
+ const { list, offsets } = encodedIdList(payload.docIds);
1188
+ const docIdsAt = builder.rawEntry("docIds", list);
1189
+ const sections = [sectionOf("docIds", docIdsAt, list, SEGMENT_PAGE_BYTES)];
1190
+ builder.push(encode("codes"));
1191
+ if (codes === null) {
1192
+ builder.push(encode(null));
1193
+ } else {
1194
+ builder.push(Uint8Array.of(FIXMAP | 3));
1195
+ builder.entry("bits", codes.bits);
1196
+ builder.entry("centroid", codes.centroid);
1197
+ const recordsAt = builder.bytesEntry("records", codes.records);
1198
+ sections.push(sectionOf("records", recordsAt, codes.records, RECORD_PAGE_BYTES));
1199
+ }
1200
+ const magnitudes = magnitudeBytes(heldMagnitudes);
1201
+ sections.push(sectionOf("magnitudes", builder.bytesEntry("magnitudes", magnitudes), magnitudes, SEGMENT_PAGE_BYTES));
1202
+ const idOffsets = uint32Bytes(offsets);
1203
+ sections.push(sectionOf("idOffsets", builder.bytesEntry("idOffsets", idOffsets), idOffsets, SEGMENT_PAGE_BYTES));
1204
+ const idOrder = uint32Bytes(positionsInIdOrder(payload.docIds));
1205
+ sections.push(sectionOf("idOrder", builder.bytesEntry("idOrder", idOrder), idOrder, SEGMENT_PAGE_BYTES));
1206
+ const headKey = encode("head");
1207
+ const vectorsKey = encode("vectors");
1208
+ const vectorsHeader = binHeader(payload.vectors.byteLength);
1209
+ const beforeHead = builder.offset + headKey.byteLength;
1210
+ const vectorChecksums = pageChecksums(payload.vectors, SEGMENT_PAGE_BYTES);
1211
+ const { head } = encodedHeadAndVectorsPlace(
1212
+ sections,
1213
+ (headLength) => beforeHead + binHeader(headLength).byteLength,
1214
+ payload.vectors,
1215
+ vectorChecksums,
1216
+ vectorsKey.byteLength + vectorsHeader.byteLength
1217
+ );
1218
+ const headOffset = builder.bytesEntry("head", head);
1219
+ builder.push(vectorsKey);
1220
+ builder.push(vectorsHeader);
1221
+ const beforeVectors = [...builder.chunks];
1222
+ builder.push(payload.vectors);
1223
+ return {
1224
+ chunks: builder.chunks,
1225
+ headOffset,
1226
+ headLength: head.byteLength,
1227
+ headChecksum: crc32(head),
1228
+ beforeVectors,
1229
+ vectors: {
1230
+ byteLength: payload.vectors.byteLength,
1231
+ checksum: checksumAcrossPages(vectorChecksums, payload.vectors.byteLength, SEGMENT_PAGE_BYTES)
1232
+ }
1233
+ };
1234
+ }
1235
+ function nodeOffsetsOf(graph) {
1236
+ const offsets = new Uint32Array(graph.levels.length + 1);
1237
+ const view = new DataView(graph.neighbours.buffer, graph.neighbours.byteOffset, graph.neighbours.byteLength);
1238
+ let cursor = 0;
1239
+ for (let at = 0; at < graph.levels.length; at++) {
1240
+ offsets[at] = cursor;
1241
+ for (let layer = 0; layer < graph.levels[at]; layer++) cursor += (view.getUint32(cursor, true) + 1) * 4;
1242
+ }
1243
+ offsets[graph.levels.length] = cursor;
1244
+ return uint32Bytes(offsets);
1245
+ }
1246
+ function pagedGraphPartChunks(part) {
1247
+ const builder = createPayloadBuilder();
1248
+ builder.push(Uint8Array.of(FIXMAP | 4));
1249
+ builder.entry("v", part.v);
1250
+ builder.entry("fieldName", part.fieldName);
1251
+ builder.push(encode("graphs"));
1252
+ builder.push(arrayHeader(part.graphs.length));
1253
+ const layouts = [];
1254
+ for (const graph of part.graphs) {
1255
+ const { levels, neighbours, ...header } = graph;
1256
+ const headerEntries = Object.entries(header).filter(([, value]) => value !== void 0);
1257
+ builder.push(Uint8Array.of(FIXMAP | headerEntries.length + 2));
1258
+ for (const [key, value] of headerEntries) builder.entry(key, value);
1259
+ const levelsAt = builder.bytesEntry("levels", levels);
1260
+ const neighboursAt = builder.bytesEntry("neighbours", neighbours);
1261
+ layouts.push({
1262
+ ...header,
1263
+ levels: sectionOf("levels", levelsAt, levels, SEGMENT_PAGE_BYTES),
1264
+ nodeOffsets: nodeOffsetsOf(graph),
1265
+ neighbours: sectionOf("neighbours", neighboursAt, neighbours, RECORD_PAGE_BYTES)
1266
+ });
1267
+ }
1268
+ const head = encode({ graphs: layouts });
1269
+ const headOffset = builder.bytesEntry("head", head);
1270
+ return { chunks: builder.chunks, headOffset, headLength: head.byteLength, headChecksum: crc32(head) };
1271
+ }
1272
+ var EMPTY_BIN = new Uint8Array(0);
1273
+ var EMPTY_BIN_BYTES = 2;
1274
+ var BIN82 = 196;
1275
+ var BIN162 = 197;
1276
+ var BIN322 = 198;
1277
+ var BIN8_MAX_BYTES2 = 255;
1278
+ var BIN16_MAX_BYTES = 65535;
1279
+ function binHeader2(byteLength) {
1280
+ if (byteLength <= BIN8_MAX_BYTES2) return Uint8Array.of(BIN82, byteLength);
1281
+ if (byteLength <= BIN16_MAX_BYTES) return Uint8Array.of(BIN162, byteLength >> 8, byteLength & 255);
1282
+ const header = new Uint8Array(5);
1283
+ header[0] = BIN322;
1284
+ new DataView(header.buffer).setUint32(1, byteLength, false);
1285
+ return header;
1286
+ }
970
1287
  function chunksEndingInBytes(withTheBytes, withEmptyBytesInTheirPlace, bytes) {
971
1288
  const head = encode(withEmptyBytesInTheirPlace);
972
- const endsWithTheEmptyBin = head[head.length - EMPTY_BIN_BYTES] === BIN8 && head[head.length - 1] === 0;
1289
+ const endsWithTheEmptyBin = head[head.length - EMPTY_BIN_BYTES] === BIN82 && head[head.length - 1] === 0;
973
1290
  if (!endsWithTheEmptyBin) return [encode(withTheBytes)];
974
- return [head.subarray(0, head.length - EMPTY_BIN_BYTES), binHeader(bytes.byteLength), bytes];
1291
+ return [head.subarray(0, head.length - EMPTY_BIN_BYTES), binHeader2(bytes.byteLength), bytes];
975
1292
  }
976
1293
  function vectorPartChunks(part) {
977
1294
  return chunksEndingInBytes(part, { ...part, vectors: EMPTY_BIN }, part.vectors);
@@ -987,20 +1304,27 @@ function keptFileRef(input, prior, kept) {
987
1304
  { key: kept.key }
988
1305
  );
989
1306
  }
990
- return { ...kept, bytes: listed.bytes, checksum: listed.checksum };
1307
+ return { ...kept, bytes: listed.bytes, checksum: listed.checksum, ...headFieldsOf(listed) };
1308
+ }
1309
+ function headFieldsOf(ref) {
1310
+ return ref.headOffset === void 0 ? {} : { headOffset: ref.headOffset, headLength: ref.headLength, headChecksum: ref.headChecksum };
1311
+ }
1312
+ function headRefOf(paged) {
1313
+ return paged === null ? {} : { headOffset: paged.headOffset, headLength: paged.headLength, headChecksum: paged.headChecksum };
991
1314
  }
992
1315
  async function writeGraphParts(input, fieldPath, plan, generation) {
993
1316
  const parts = plan.graphParts ?? [];
994
1317
  const refs = [];
995
1318
  for (let part = 0; part < parts.length; part++) {
996
1319
  const key = graphPartKey(input.indexName, fieldPath, generation, part);
1320
+ const paged = plan.storesHeads ? pagedGraphPartChunks(parts[part]) : null;
997
1321
  const packed = await packChecksummedEnvelope(
998
- [encode(parts[part])],
1322
+ paged === null ? [encode(parts[part])] : paged.chunks,
999
1323
  GRAPH_PART_ENVELOPE_VERSION,
1000
1324
  `graph part "${key}"`
1001
1325
  );
1002
1326
  await input.store.write(key, packed.chunks);
1003
- refs.push({ key, bytes: packed.byteLength, checksum: packed.checksum });
1327
+ refs.push({ key, bytes: packed.byteLength, checksum: packed.checksum, ...headRefOf(paged) });
1004
1328
  }
1005
1329
  return refs;
1006
1330
  }
@@ -1014,17 +1338,28 @@ async function writeField(input, fieldPath, plan, outcome) {
1014
1338
  const files = plan.kept.map((kept) => keptFileRef(input, prior, kept));
1015
1339
  const written = [];
1016
1340
  for (let index = 0; index < plan.newFiles; index++) {
1017
- const { payload, ordinals, dead } = plan.readNewFile(index);
1341
+ const { payload, ordinals, dead, magnitudes } = plan.readNewFile(index);
1018
1342
  const id = nextFileId;
1019
1343
  nextFileId += 1;
1020
1344
  const key = vectorFileKey(input.indexName, fieldPath, id);
1021
- const packed = await packChecksummedEnvelope(
1022
- vectorPartChunks(payload),
1345
+ const paged = plan.storesHeads && magnitudes !== null ? pagedVectorFileChunks(payload, magnitudes) : null;
1346
+ const what = `vector file "${key}"`;
1347
+ const packed = paged === null ? await packChecksummedEnvelope(vectorPartChunks(payload), VECTOR_FILE_ENVELOPE_VERSION, what) : packEnvelopeWithChecksum(
1348
+ paged.chunks,
1023
1349
  VECTOR_FILE_ENVELOPE_VERSION,
1024
- `vector file "${key}"`
1350
+ what,
1351
+ crc32Combine(await checksumOfChunks(paged.beforeVectors), paged.vectors.checksum, paged.vectors.byteLength)
1025
1352
  );
1026
1353
  await input.store.write(key, packed.chunks);
1027
- files.push({ id, key, count: payload.docIds.length, dead, bytes: packed.byteLength, checksum: packed.checksum });
1354
+ files.push({
1355
+ id,
1356
+ key,
1357
+ count: payload.docIds.length,
1358
+ dead,
1359
+ bytes: packed.byteLength,
1360
+ checksum: packed.checksum,
1361
+ ...headRefOf(paged)
1362
+ });
1028
1363
  written.push({ id, key, ordinals });
1029
1364
  outcome.layouts.push({
1030
1365
  fieldPath,
@@ -1101,8 +1436,41 @@ function removeWhatTheSegmentReplaces(manager, segment, contents) {
1101
1436
  function yieldToEventLoop() {
1102
1437
  return new Promise((resolve) => setTimeout(resolve, 0));
1103
1438
  }
1439
+ function removeFromPartition(manager, partitionId, docIds) {
1440
+ const { composite, docPartitionMap, insertOptions } = manager.diskSegmentAccess(partitionId);
1441
+ composite.beginBatch();
1442
+ try {
1443
+ for (const docId of docIds) {
1444
+ if (!composite.has(docId)) continue;
1445
+ composite.remove(docId, manager.config.schema, manager.language, insertOptions);
1446
+ docPartitionMap.delete(docId);
1447
+ }
1448
+ } finally {
1449
+ composite.endBatch();
1450
+ }
1451
+ }
1452
+ async function attachPagedSegment(directory, ref, partitionId, manager, options) {
1453
+ const { checkPagedSegmentFully, createPagedFrozenSegment, openPagedSnapshot, pagedListingOf } = await import('./module-IYPLWHGC.mjs');
1454
+ if (options.decodesEveryDocument) await checkPagedSegmentFully(directory, ref);
1455
+ const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
1456
+ requireAgreedSchema(options.schema, snapshot.schema, ref.key);
1457
+ const segment = createPagedFrozenSegment(snapshot);
1458
+ removeFromPartition(manager, partitionId, snapshot.tombstones);
1459
+ options.writeRecords?.loadPagedSegment(manager.indexName, partitionId, segment);
1460
+ if (snapshot.documentCount > 0) manager.attachFrozenSegment(partitionId, segment);
1461
+ else segment.paged?.close();
1462
+ }
1104
1463
  async function loadPartitionSegments(store, partition, manager, options) {
1464
+ const { directory } = store;
1105
1465
  for (const ref of partition.segments) {
1466
+ if (manager.config.textStorage === "disk" && directory !== void 0) {
1467
+ const { isPagedSegmentFile } = await import('./module-IYPLWHGC.mjs');
1468
+ if (await isPagedSegmentFile(directory, ref.key)) {
1469
+ await attachPagedSegment(directory, ref, partition.partitionId, manager, options);
1470
+ await yieldToEventLoop();
1471
+ continue;
1472
+ }
1473
+ }
1106
1474
  const contents = await readSegmentFile(store, ref);
1107
1475
  requireAgreedSchema(options.schema, contents.schema, ref.key);
1108
1476
  if (options.decodesEveryDocument) decodeEveryStoredDocument(contents.arrays.documentTable);
@@ -1147,6 +1515,14 @@ async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodes
1147
1515
  if (partition.partitionId > highestPartitionId) highestPartitionId = partition.partitionId;
1148
1516
  }
1149
1517
  growPartitionsTo(deps.manager, highestPartitionId);
1518
+ const { directory } = store;
1519
+ const opensVectorsOnDemand = manifest.vectors.some(
1520
+ (field) => deps.vectorIndexes.get(field.fieldPath)?.storage === "disk" && field.files.some((file) => file.headOffset !== void 0)
1521
+ );
1522
+ if (!decodesEveryDocument && directory !== void 0 && opensVectorsOnDemand) {
1523
+ const { startReadingOpeningSectionsAhead } = await import('./module-IYPLWHGC.mjs');
1524
+ await startReadingOpeningSectionsAhead(directory, manifest);
1525
+ }
1150
1526
  for (const partition of manifest.partitions) {
1151
1527
  if (partition.partitionId >= deps.manager.partitionCount) {
1152
1528
  throw new NarsilError(
@@ -1164,7 +1540,12 @@ async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodes
1164
1540
  for (const vector of manifest.vectors) {
1165
1541
  const vectorIndex = deps.vectorIndexes.get(vector.fieldPath);
1166
1542
  if (vectorIndex === void 0) continue;
1167
- await loadVectorField(store, vector, vectorIndex);
1543
+ await loadVectorField(
1544
+ store,
1545
+ vector,
1546
+ vectorIndex,
1547
+ decodesEveryDocument ? null : (docId) => deps.manager.partitionIdOf(docId)
1548
+ );
1168
1549
  }
1169
1550
  return manifest.checkpoint;
1170
1551
  }
@@ -1174,4 +1555,4 @@ async function reclaimUnlistedKeys(store, indexName, referenced) {
1174
1555
  }
1175
1556
  }
1176
1557
 
1177
- export { SINGLE_NODE_PRIMARY_TERM, VECTOR_FILE_ENVELOPE_VERSION, createSeqOwner, decodeManifestPayload, decodeSegmentManifest, deserializePayloadV1, dropStoredVectorValues, growPartitionsTo, indexMetadataOf, invalidManifest, isId, isRecord, isSequenceNumber, isUint32, listedKeysOf, loadPartitionSegments, loadSegmentedCheckpoint, metadataConfigOf, metadataFromConfig, openManifestPayload, patternIndexesFromWire, readCurrentManifest, readGraphParts, readMetadataEnvelope, readSegmentManifest, readVectorFile, reclaimUnlistedKeys, sanitizeSurfaceForms, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, writeLiveVectors, writeMetadataEnvelope };
1558
+ export { SINGLE_NODE_PRIMARY_TERM, createSeqOwner, decodeManifestPayload, decodeSegmentManifest, deserializePayloadV1, dropStoredVectorValues, durableMetadataConfigOf, growPartitionsTo, indexMetadataOf, invalidManifest, isId, isRecord, isSequenceNumber, isUint32, listedKeysOf, loadPartitionSegments, loadSegmentedCheckpoint, metadataConfigOf, metadataFromConfig, openManifestPayload, patternIndexesFromWire, readCurrentManifest, readMetadataEnvelope, readSegmentManifest, reclaimUnlistedKeys, sanitizeSurfaceForms, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, writeLiveVectors, writeMetadataEnvelope };