@delali/narsil 0.2.4-next.59 → 0.3.1-next.101

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. package/README.md +9 -8
  2. package/dist/adapters/filesystem.d.ts +1 -1
  3. package/dist/adapters/filesystem.mjs +1 -1
  4. package/dist/adapters/indexeddb.d.ts +1 -1
  5. package/dist/adapters/indexeddb.mjs +1 -1
  6. package/dist/adapters/memory.d.ts +1 -1
  7. package/dist/{adapters-79OIa4xm.d.ts → adapters-BQFQ4PRz.d.ts} +27 -1
  8. package/dist/browser/chunk-BDOSMEEP.mjs +493 -0
  9. package/dist/browser/chunk-EJDUFWSO.mjs +602 -0
  10. package/dist/browser/chunk-HOZW3PKP.mjs +9 -0
  11. package/dist/browser/chunk-JOVZ6HMT.mjs +4358 -0
  12. package/dist/browser/chunk-JPTQZWJQ.mjs +10 -0
  13. package/dist/browser/chunk-KPJMDPHG.mjs +24558 -0
  14. package/dist/browser/chunk-PGIM3HIM.mjs +2743 -0
  15. package/dist/browser/chunk-RHG4O77A.mjs +206 -0
  16. package/dist/browser/chunk-RIJSLY2I.mjs +360 -0
  17. package/dist/browser/durability-wiring-SRUYT3VV.mjs +3875 -0
  18. package/dist/browser/module-CBB2ETYP.mjs +3111 -0
  19. package/dist/browser/release-76MXRPGV.mjs +33 -0
  20. package/dist/browser/replace-RURXTGVG.mjs +57 -0
  21. package/dist/browser/restore-queue-EEZQO6RF.mjs +35 -0
  22. package/dist/browser/runtime-7ZAHCHWR.mjs +557 -0
  23. package/dist/browser/snapshot-OAAA5UJ7.mjs +1210 -0
  24. package/dist/browser/write-7D6VIK6O.mjs +208 -0
  25. package/dist/{capabilities-DSsOGz6F.d.ts → capabilities-DiGlB6Sr.d.ts} +68 -7
  26. package/dist/chunk-2RIRGOUL.mjs +113 -0
  27. package/dist/chunk-37CD3OKS.mjs +87 -0
  28. package/dist/chunk-44W3CBNG.mjs +7 -0
  29. package/dist/chunk-46D6JQIG.mjs +426 -0
  30. package/dist/{chunk-VMRR3DQU.mjs → chunk-5JNA5SKS.mjs} +546 -35
  31. package/dist/chunk-747OHOUA.mjs +11 -0
  32. package/dist/chunk-75XIEF6H.mjs +126 -0
  33. package/dist/chunk-7X5JST5L.mjs +114 -0
  34. package/dist/chunk-BOFBLJFZ.mjs +209 -0
  35. package/dist/{chunk-USKFU7J5.mjs → chunk-BWJVNZPR.mjs} +8 -1
  36. package/dist/chunk-C7SLA6CM.mjs +19 -0
  37. package/dist/{chunk-POWOTUKJ.mjs → chunk-CQUMNIIG.mjs} +3 -6
  38. package/dist/chunk-CVXLLEQ3.mjs +1011 -0
  39. package/dist/chunk-DJHHF2L5.mjs +17 -0
  40. package/dist/chunk-E74YLAIG.mjs +24 -0
  41. package/dist/{chunk-UXNWQZST.mjs → chunk-EMRMGYRE.mjs} +236 -18
  42. package/dist/chunk-EWZLIYHV.mjs +57 -0
  43. package/dist/chunk-GBEEG5AN.mjs +35 -0
  44. package/dist/chunk-GES7U2PM.mjs +68 -0
  45. package/dist/chunk-HJSN7UYG.mjs +10 -0
  46. package/dist/chunk-HLPRHOSO.mjs +17 -0
  47. package/dist/chunk-HMFNFVP7.mjs +14 -0
  48. package/dist/chunk-J5QP52HA.mjs +1751 -0
  49. package/dist/{chunk-YNVIUGF5.mjs → chunk-JZVARCRZ.mjs} +3 -28
  50. package/dist/chunk-KOKM4PQN.mjs +52 -0
  51. package/dist/{chunk-B2TFROFX.mjs → chunk-KWTW6VTG.mjs} +465 -250
  52. package/dist/chunk-KZ3NX2P7.mjs +21 -0
  53. package/dist/chunk-L5ORED7V.mjs +25 -0
  54. package/dist/{chunk-THYRHHO2.mjs → chunk-LGAMKHA2.mjs} +2009 -2351
  55. package/dist/chunk-LKOB5COA.mjs +117 -0
  56. package/dist/chunk-LPQOEU5M.mjs +96 -0
  57. package/dist/{chunk-3LHGNRYE.mjs → chunk-LXWDQLTW.mjs} +7 -2
  58. package/dist/{chunk-L6TNPLAR.mjs → chunk-OEYI3KRB.mjs} +270 -108
  59. package/dist/chunk-OM362BRO.mjs +948 -0
  60. package/dist/chunk-OZDFA4WC.mjs +67 -0
  61. package/dist/chunk-P36MUFEJ.mjs +10 -0
  62. package/dist/chunk-PC2G76VW.mjs +2540 -0
  63. package/dist/{chunk-GCM3JFVA.mjs → chunk-PNEYIHA4.mjs} +11 -2
  64. package/dist/chunk-PNFKUCG3.mjs +1130 -0
  65. package/dist/chunk-Q6XWVUZX.mjs +44 -0
  66. package/dist/chunk-R27EMH5M.mjs +617 -0
  67. package/dist/{chunk-YMZCMH5M.mjs → chunk-R6A773WK.mjs} +19 -24
  68. package/dist/chunk-RF7PLG43.mjs +106 -0
  69. package/dist/chunk-RFBLQLLZ.mjs +16 -0
  70. package/dist/{chunk-Y3L5MDRV.mjs → chunk-RFVUDM5E.mjs} +17 -2
  71. package/dist/chunk-S7FL6UAH.mjs +521 -0
  72. package/dist/chunk-S7GNDRPS.mjs +651 -0
  73. package/dist/chunk-SP3NM3ZG.mjs +3537 -0
  74. package/dist/chunk-U476IXKQ.mjs +10 -0
  75. package/dist/chunk-UA3AUGNV.mjs +462 -0
  76. package/dist/chunk-UTG5YNXV.mjs +507 -0
  77. package/dist/chunk-VI4R6W25.mjs +8484 -0
  78. package/dist/chunk-WQBCBKY3.mjs +8 -0
  79. package/dist/{chunk-24RXHMEL.mjs → chunk-WVCOCKBU.mjs} +3794 -7570
  80. package/dist/chunk-X5CV2E2B.mjs +1 -0
  81. package/dist/chunk-XADZPPZI.mjs +614 -0
  82. package/dist/chunk-YEEQHT4V.mjs +212 -0
  83. package/dist/chunk-YTJCGKM4.mjs +56 -0
  84. package/dist/chunk-YYQIODMF.mjs +275 -0
  85. package/dist/chunk-ZI34TVOA.mjs +144 -0
  86. package/dist/{chunk-OOSIDHA3.mjs → chunk-ZKWYWW4Y.mjs} +1 -1
  87. package/dist/chunk-ZM7KU4YW.mjs +35 -0
  88. package/dist/{client-BQK6Tf6A.d.ts → client-BOStAkc5.d.ts} +160 -37
  89. package/dist/client.d.ts +11 -10
  90. package/dist/client.mjs +209 -45
  91. package/dist/{config-DmXULPa-.d.ts → config-Cjvzbs_j.d.ts} +51 -3
  92. package/dist/config-DffQlmRq.d.ts +421 -0
  93. package/dist/direct-executor-ZD64PPEZ.mjs +24 -0
  94. package/dist/distribution/coordinator/etcd.d.ts +79 -18
  95. package/dist/distribution/coordinator/etcd.mjs +553 -109
  96. package/dist/distribution/coordinator/in-memory.d.ts +4 -3
  97. package/dist/distribution/coordinator/in-memory.mjs +22 -3
  98. package/dist/distribution/transport/grpc.d.ts +2 -2
  99. package/dist/distribution/transport/grpc.mjs +79 -14
  100. package/dist/distribution/transport/in-memory.d.ts +2 -2
  101. package/dist/distribution/transport/in-memory.mjs +12 -7
  102. package/dist/distribution/transport/tcp.d.ts +2 -2
  103. package/dist/distribution/transport/tcp.mjs +50 -28
  104. package/dist/distribution.d.ts +144 -23
  105. package/dist/distribution.mjs +14479 -8173
  106. package/dist/durability-wiring-FEUBR5BT.mjs +3330 -0
  107. package/dist/embeddings/openai.d.ts +2 -2
  108. package/dist/embeddings/openai.mjs +22 -133
  109. package/dist/{engine-CIOyFozK.d.ts → engine-rC0vjjY-.d.ts} +165 -141
  110. package/dist/{memory-yFaVn2xZ.d.ts → events-DqOIg5xa.d.ts} +645 -136
  111. package/dist/{types-DAK7mnUZ.d.ts → index-Bkdt8nDw.d.ts} +182 -45
  112. package/dist/index.browser.mjs +4 -32180
  113. package/dist/index.d.ts +91 -20
  114. package/dist/index.mjs +33 -16
  115. package/dist/invalidation/broadcast-channel.d.ts +1 -1
  116. package/dist/invalidation/filesystem.d.ts +1 -1
  117. package/dist/invalidation/filesystem.mjs +2 -1
  118. package/dist/invalidation/noop.d.ts +1 -1
  119. package/dist/module-3S65MVCM.mjs +2151 -0
  120. package/dist/partial-words-DTIHO8G4.d.ts +19 -0
  121. package/dist/partial-words.d.ts +22 -0
  122. package/dist/partial-words.mjs +17 -0
  123. package/dist/pattern-Cledccbl.d.ts +18 -0
  124. package/dist/pattern.d.ts +22 -0
  125. package/dist/pattern.mjs +12 -0
  126. package/dist/persistence/durability/checkpoint-worker.d.ts +119 -72
  127. package/dist/persistence/durability/checkpoint-worker.mjs +51 -18
  128. package/dist/react.d.ts +279 -183
  129. package/dist/react.mjs +201 -48
  130. package/dist/release-BH3DKW3T.mjs +33 -0
  131. package/dist/replace-CEO5KJJ3.mjs +57 -0
  132. package/dist/rerank-K7C4WZFG.mjs +60 -0
  133. package/dist/rerankers/http.d.ts +50 -0
  134. package/dist/rerankers/http.mjs +104 -0
  135. package/dist/restore-queue-KQHA7ZV2.mjs +35 -0
  136. package/dist/runtime-6Y4ZYXOG.mjs +177 -0
  137. package/dist/{schema-d5Afu03A.d.ts → schema-4W_7tk1f.d.ts} +81 -11
  138. package/dist/server.d.ts +13 -12
  139. package/dist/server.mjs +611 -80
  140. package/dist/snapshot-QGZPODM5.mjs +799 -0
  141. package/dist/synonyms-CpOAHPa-.d.ts +52 -0
  142. package/dist/synonyms.d.ts +22 -0
  143. package/dist/synonyms.mjs +12 -0
  144. package/dist/{tasks-CH2Dod8w.d.ts → tasks-CwPTz-tP.d.ts} +10 -3
  145. package/dist/thread-N3IUGUS3.mjs +697 -0
  146. package/dist/{types-BQDlIvfS.d.ts → types-2qoitTXg.d.ts} +1 -1
  147. package/dist/{types-D5OwhtsQ.d.ts → types-Cz5okLml.d.ts} +2 -2
  148. package/dist/vector/search-worker.d.ts +46 -4
  149. package/dist/vector/search-worker.mjs +21 -11
  150. package/dist/workers/entry.mjs +9 -6
  151. package/dist/write-U5RI7MAX.mjs +241 -0
  152. package/package.json +55 -9
  153. package/dist/chunk-3L4U6EJQ.mjs +0 -49
  154. package/dist/chunk-IFBKLBGB.mjs +0 -7
  155. package/dist/chunk-ORBLD3EL.mjs +0 -6
  156. package/dist/chunk-U7KYW4BA.mjs +0 -135
  157. package/dist/chunk-ZJ4CSAB2.mjs +0 -13
  158. package/dist/chunk-ZNZBXMP5.mjs +0 -2117
  159. package/dist/chunk-ZZT3EAUC.mjs +0 -11168
  160. package/dist/config-BhldZgaK.d.ts +0 -194
  161. package/dist/direct-executor-ALAVE7FC.mjs +0 -12
  162. package/dist/serialization/crc32-worker.d.ts +0 -20
  163. package/dist/serialization/crc32-worker.mjs +0 -62
  164. package/dist/simd-DW5AtJ6B.d.ts +0 -41
  165. package/dist/thread-DYHEQRVY.mjs +0 -402
@@ -0,0 +1,1130 @@
1
+ import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-YYQIODMF.mjs';
2
+ import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, readSegmentFile, manifestKey, segmentsPrefix, readListedFile, requireListedChecksum, vectorFileKey, graphPartKey } from './chunk-J5QP52HA.mjs';
3
+ import { VERSION, rulesOfStoredSynonyms, openChecksummedEnvelope, readHeader, HEADER_SIZE, crc32, engineVersionParts, writeHeader, packChecksummedEnvelope } from './chunk-WVCOCKBU.mjs';
4
+ import { decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays } from './chunk-SP3NM3ZG.mjs';
5
+ import { flattenSchema, requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-OEYI3KRB.mjs';
6
+ import { MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-U476IXKQ.mjs';
7
+ import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
8
+ import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
9
+ import { NarsilError, ErrorCodes } from './chunk-BWJVNZPR.mjs';
10
+ import { decode, encode } from '@msgpack/msgpack';
11
+
12
+ // src/engine/durability-metadata.ts
13
+ function metadataFromConfig(indexName, config, partitionCount, documentCount) {
14
+ const metadata = {
15
+ indexName,
16
+ schema: config.schema,
17
+ language: config.language,
18
+ partitionCount: partitionCount ?? config.partitionCount,
19
+ bm25Params: { k1: config.k1, b: config.b },
20
+ createdAt: Date.now(),
21
+ engineVersion: VERSION
22
+ };
23
+ metadata.documentCount = documentCount ?? config.documentCount;
24
+ if (config.embedding) {
25
+ metadata.embedding = config.embedding;
26
+ }
27
+ if (config.surfaceForms !== void 0) {
28
+ metadata.surfaceForms = config.surfaceForms;
29
+ }
30
+ if (config.analysisRevision !== void 0) {
31
+ metadata.analysisRevision = config.analysisRevision;
32
+ }
33
+ if (config.tokenizer !== void 0) {
34
+ metadata.tokenizer = config.tokenizer;
35
+ }
36
+ if (config.stopWords !== void 0) {
37
+ metadata.stopWords = config.stopWords;
38
+ }
39
+ if (config.stopWordList !== void 0) {
40
+ metadata.stopWordList = config.stopWordList;
41
+ }
42
+ if (config.partitionLimits !== void 0) {
43
+ metadata.partitionLimits = config.partitionLimits;
44
+ }
45
+ if (config.defaultScoring !== void 0) {
46
+ metadata.defaultScoring = config.defaultScoring;
47
+ }
48
+ if (config.trackPositions !== void 0) {
49
+ metadata.trackPositions = config.trackPositions;
50
+ }
51
+ if (config.strict !== void 0) {
52
+ metadata.strict = config.strict;
53
+ }
54
+ if (config.required !== void 0) {
55
+ metadata.required = config.required;
56
+ }
57
+ if (config.vectorPromotion !== void 0) {
58
+ metadata.vectorPromotion = config.vectorPromotion;
59
+ }
60
+ if (config.patternValueLimit !== void 0) {
61
+ metadata.patternValueLimit = config.patternValueLimit;
62
+ }
63
+ if (config.indexUuid !== void 0) {
64
+ metadata.indexUuid = config.indexUuid;
65
+ }
66
+ if (config.synonyms !== void 0) {
67
+ metadata.synonyms = config.synonyms;
68
+ }
69
+ if (config.heldPartitions !== void 0) {
70
+ metadata.heldPartitions = config.heldPartitions;
71
+ }
72
+ return metadata;
73
+ }
74
+ function metadataConfigOf(entry) {
75
+ const embedding = entry.config.embedding ? {
76
+ fields: entry.config.embedding.fields,
77
+ ...entry.embeddingAdapterName !== null ? { adapter: entry.embeddingAdapterName } : {}
78
+ } : void 0;
79
+ return {
80
+ ...entry.indexUuid !== null ? { indexUuid: entry.indexUuid } : {},
81
+ ...entry.heldPartitions !== null ? { heldPartitions: entry.heldPartitions } : {},
82
+ documentCount: entry.documentCount,
83
+ partitionCount: entry.partitionCount,
84
+ schema: flattenSchema(entry.config.schema),
85
+ language: entry.language.name,
86
+ k1: entry.config.bm25?.k1 ?? 1.2,
87
+ b: entry.config.bm25?.b ?? 0.75,
88
+ ...embedding !== void 0 ? { embedding } : {},
89
+ surfaceForms: entry.config.surfaceForms !== false,
90
+ ...entry.analysisRevision !== null ? { analysisRevision: entry.analysisRevision } : {},
91
+ ...typeof entry.config.tokenizer === "string" ? { tokenizer: entry.config.tokenizer } : {},
92
+ ...typeof entry.config.stopWords === "string" ? { stopWords: entry.config.stopWords } : {},
93
+ ...entry.config.stopWords instanceof Set ? { stopWordList: [...entry.config.stopWords].sort(compareCodePoints) } : {},
94
+ ...entry.config.partitions !== void 0 ? { partitionLimits: entry.config.partitions } : {},
95
+ ...entry.config.defaultScoring !== void 0 ? { defaultScoring: entry.config.defaultScoring } : {},
96
+ ...entry.config.trackPositions !== void 0 ? { trackPositions: entry.config.trackPositions } : {},
97
+ ...entry.config.strict !== void 0 ? { strict: entry.config.strict } : {},
98
+ ...entry.config.required !== void 0 ? { required: entry.config.required } : {},
99
+ ...entry.config.vectorPromotion !== void 0 ? { vectorPromotion: entry.config.vectorPromotion } : {},
100
+ ...entry.config.patternValueLimit !== void 0 ? { patternValueLimit: entry.config.patternValueLimit } : {},
101
+ ...entry.synonyms !== null ? { synonyms: rulesOfStoredSynonyms(entry.synonyms) } : {}
102
+ };
103
+ }
104
+ function indexMetadataOf(indexName, entry, partitionCount) {
105
+ return metadataFromConfig(indexName, metadataConfigOf(entry), partitionCount);
106
+ }
107
+
108
+ // src/serialization/stored-vector-values.ts
109
+ function dropStoredVectorValues(documents, schema) {
110
+ const vectorFieldPaths = [];
111
+ for (const [fieldPath, fieldType] of Object.entries(schema)) {
112
+ if (typeof fieldType === "string" && VECTOR_PATTERN.test(fieldType)) {
113
+ vectorFieldPaths.push(fieldPath);
114
+ }
115
+ }
116
+ if (vectorFieldPaths.length === 0) {
117
+ return;
118
+ }
119
+ for (const document of Object.values(documents)) {
120
+ for (const fieldPath of vectorFieldPaths) {
121
+ removeFieldAtPath(document.fields, fieldPath);
122
+ }
123
+ }
124
+ }
125
+ function removeFieldAtPath(fields, fieldPath) {
126
+ if (!fieldPath.includes(".")) {
127
+ delete fields[fieldPath];
128
+ return;
129
+ }
130
+ const segments = fieldPath.split(".");
131
+ let current = fields;
132
+ for (let i = 0; i < segments.length - 1; i += 1) {
133
+ const next = current[segments[i]];
134
+ if (next === null || typeof next !== "object") {
135
+ return;
136
+ }
137
+ current = next;
138
+ }
139
+ delete current[segments[segments.length - 1]];
140
+ }
141
+
142
+ // src/serialization/payload-v1/partition.ts
143
+ function sanitizeSurfaceForms(raw) {
144
+ if (raw === void 0 || raw === null || typeof raw !== "object" || Array.isArray(raw)) return void 0;
145
+ const result = /* @__PURE__ */ Object.create(null);
146
+ let count = 0;
147
+ for (const [surface, value] of Object.entries(raw)) {
148
+ if (typeof value === "number") {
149
+ result[surface] = value;
150
+ count++;
151
+ } else if (Array.isArray(value) && typeof value[0] === "number" && typeof value[1] === "string") {
152
+ result[surface] = [value[0], value[1]];
153
+ count++;
154
+ }
155
+ }
156
+ return count > 0 ? result : void 0;
157
+ }
158
+ function isWireRecord(value) {
159
+ return value !== null && typeof value === "object" && !Array.isArray(value);
160
+ }
161
+ function corruptPatternEntry(field) {
162
+ return new NarsilError(
163
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
164
+ `The pattern entry of field "${field}" in this partition must hold a list of document IDs and a map of runs`,
165
+ { field }
166
+ );
167
+ }
168
+ function patternIndexesFromWire(raw) {
169
+ if (raw === void 0 || raw === null) return {};
170
+ if (!isWireRecord(raw)) {
171
+ throw new NarsilError(
172
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
173
+ "The pattern entry of this partition must be a map of fields"
174
+ );
175
+ }
176
+ const pattern = /* @__PURE__ */ Object.create(null);
177
+ for (const [field, index] of Object.entries(raw)) {
178
+ if (!isWireRecord(index) || !isWireRecord(index.runs) || !Array.isArray(index.doc_ids)) {
179
+ throw corruptPatternEntry(field);
180
+ }
181
+ const docIds = [];
182
+ for (const docId of index.doc_ids) {
183
+ if (typeof docId !== "string") throw corruptPatternEntry(field);
184
+ docIds.push(docId);
185
+ }
186
+ pattern[field] = { docIds, runs: index.runs };
187
+ }
188
+ return { pattern };
189
+ }
190
+ function validateHnswMetric(value) {
191
+ if (value === void 0 || value === null) return void 0;
192
+ if (typeof value === "string" && VALID_HNSW_METRICS.has(value)) {
193
+ return value;
194
+ }
195
+ return void 0;
196
+ }
197
+ function wireToPartition(raw) {
198
+ const documents = {};
199
+ for (const [docId, doc] of Object.entries(raw.documents ?? {})) {
200
+ documents[docId] = {
201
+ fields: doc.fields,
202
+ fieldLengths: doc.field_lengths ?? {}
203
+ };
204
+ }
205
+ requireValidFieldTypes(raw.schema ?? {});
206
+ dropStoredVectorValues(documents, raw.schema ?? {});
207
+ const invertedIndex = {};
208
+ for (const [token, list] of Object.entries(raw.inverted_index ?? {})) {
209
+ invertedIndex[token] = {
210
+ docFrequency: list.doc_freq,
211
+ postings: (list.postings ?? []).map((p) => ({
212
+ docId: p.doc_id,
213
+ termFrequency: p.term_freq,
214
+ field: p.field,
215
+ positions: p.positions ?? []
216
+ }))
217
+ };
218
+ }
219
+ const numeric = {};
220
+ for (const [field, entries] of Object.entries(raw.field_indexes?.numeric ?? {})) {
221
+ numeric[field] = entries.map((e) => ({ value: e.value, docId: e.doc_id }));
222
+ }
223
+ const boolean = {};
224
+ for (const [field, idx] of Object.entries(raw.field_indexes?.boolean ?? {})) {
225
+ boolean[field] = { trueDocs: idx.true_docs ?? [], falseDocs: idx.false_docs ?? [] };
226
+ }
227
+ const geopoint = {};
228
+ for (const [field, entries] of Object.entries(raw.field_indexes?.geopoint ?? {})) {
229
+ geopoint[field] = entries.map((e) => ({ lat: e.lat, lon: e.lon, docId: e.doc_id }));
230
+ }
231
+ const vectorData = {};
232
+ for (const [field, data] of Object.entries(raw.vector_data ?? {})) {
233
+ vectorData[field] = {
234
+ dimension: data.dimension,
235
+ vectors: (data.vectors ?? []).map((v) => ({ docId: v.doc_id, vector: v.vector })),
236
+ hnswGraph: data.hnsw_graph ? {
237
+ entryPoint: data.hnsw_graph.entry_point,
238
+ maxLayer: data.hnsw_graph.max_layer,
239
+ m: data.hnsw_graph.m,
240
+ efConstruction: data.hnsw_graph.ef_construction,
241
+ metric: validateHnswMetric(data.hnsw_graph.metric),
242
+ nodes: data.hnsw_graph.nodes
243
+ } : null,
244
+ codes: data.codes ? { bits: data.codes.bits, centroid: data.codes.centroid, records: data.codes.records } : null
245
+ };
246
+ }
247
+ return {
248
+ indexName: raw.index_name,
249
+ partitionId: raw.partition_id,
250
+ totalPartitions: raw.total_partitions,
251
+ language: raw.language ?? "english",
252
+ schema: raw.schema ?? {},
253
+ docCount: raw.doc_count ?? 0,
254
+ avgDocLength: raw.avg_doc_length ?? 0,
255
+ documents,
256
+ invertedIndex,
257
+ fieldIndexes: {
258
+ numeric,
259
+ boolean,
260
+ enum: raw.field_indexes?.enum ?? {},
261
+ geopoint,
262
+ ...patternIndexesFromWire(raw.field_indexes?.pattern)
263
+ },
264
+ surfaceForms: sanitizeSurfaceForms(raw.surface_forms),
265
+ vectorData,
266
+ statistics: {
267
+ totalDocuments: raw.statistics?.total_documents ?? 0,
268
+ totalFieldLengths: raw.statistics?.total_field_lengths ?? {},
269
+ averageFieldLengths: raw.statistics?.average_field_lengths ?? {},
270
+ docFrequencies: raw.statistics?.doc_frequencies ?? {}
271
+ }
272
+ };
273
+ }
274
+ function deserializePayloadV1(data) {
275
+ const raw = decode(data);
276
+ return wireToPartition(raw);
277
+ }
278
+ function metadataToWire(meta) {
279
+ const wire = {
280
+ index_name: meta.indexName,
281
+ schema: meta.schema,
282
+ language: meta.language,
283
+ partition_count: meta.partitionCount,
284
+ bm25_params: meta.bm25Params,
285
+ created_at: meta.createdAt,
286
+ engine_version: meta.engineVersion
287
+ };
288
+ if (meta.vectorFields) {
289
+ wire.vector_fields = meta.vectorFields;
290
+ }
291
+ if (isNonNegativeInteger(meta.documentCount) && Number.isSafeInteger(meta.documentCount)) {
292
+ wire.document_count = meta.documentCount;
293
+ }
294
+ if (meta.embedding) {
295
+ wire.embedding = meta.embedding.adapter !== void 0 ? { adapter: meta.embedding.adapter, fields: meta.embedding.fields } : { fields: meta.embedding.fields };
296
+ }
297
+ if (meta.surfaceForms !== void 0) {
298
+ wire.surface_forms_enabled = meta.surfaceForms;
299
+ }
300
+ if (meta.analysisRevision !== void 0) {
301
+ wire.analysis_revision = meta.analysisRevision;
302
+ }
303
+ if (meta.tokenizer !== void 0) {
304
+ wire.tokenizer = meta.tokenizer;
305
+ }
306
+ if (meta.stopWords !== void 0) {
307
+ wire.stop_words = meta.stopWords;
308
+ }
309
+ if (meta.stopWordList !== void 0) {
310
+ wire.stop_word_list = meta.stopWordList;
311
+ }
312
+ if (meta.partitionLimits !== void 0) {
313
+ wire.partition_limits = {
314
+ ...meta.partitionLimits.maxDocsPerPartition !== void 0 ? { max_docs_per_partition: meta.partitionLimits.maxDocsPerPartition } : {},
315
+ ...meta.partitionLimits.maxPartitions !== void 0 ? { max_partitions: meta.partitionLimits.maxPartitions } : {},
316
+ ...meta.partitionLimits.watermark !== void 0 ? { watermark: meta.partitionLimits.watermark } : {}
317
+ };
318
+ }
319
+ if (meta.defaultScoring !== void 0) {
320
+ wire.default_scoring = meta.defaultScoring;
321
+ }
322
+ if (meta.trackPositions !== void 0) {
323
+ wire.track_positions = meta.trackPositions;
324
+ }
325
+ if (meta.strict !== void 0) {
326
+ wire.strict = meta.strict;
327
+ }
328
+ if (meta.required !== void 0) {
329
+ wire.required = meta.required;
330
+ }
331
+ if (meta.vectorPromotion !== void 0) {
332
+ const promotion = meta.vectorPromotion;
333
+ wire.vector_promotion = {
334
+ ...promotion.threshold !== void 0 ? { threshold: promotion.threshold } : {},
335
+ ...promotion.filterThreshold !== void 0 ? { filter_threshold: promotion.filterThreshold } : {},
336
+ ...promotion.hnswConfig !== void 0 ? {
337
+ hnsw_config: {
338
+ ...promotion.hnswConfig.m !== void 0 ? { m: promotion.hnswConfig.m } : {},
339
+ ...promotion.hnswConfig.efConstruction !== void 0 ? { ef_construction: promotion.hnswConfig.efConstruction } : {},
340
+ ...promotion.hnswConfig.metric !== void 0 ? { metric: promotion.hnswConfig.metric } : {}
341
+ }
342
+ } : {},
343
+ ...promotion.quantization !== void 0 ? { quantization: promotion.quantization } : {},
344
+ ...promotion.storage !== void 0 ? { storage: promotion.storage } : {}
345
+ };
346
+ }
347
+ if (meta.patternValueLimit !== void 0) {
348
+ wire.pattern_value_limit = meta.patternValueLimit;
349
+ }
350
+ if (meta.indexUuid !== void 0) {
351
+ wire.index_uuid = meta.indexUuid;
352
+ }
353
+ if (meta.heldPartitions !== void 0) {
354
+ wire.held_partitions = meta.heldPartitions;
355
+ }
356
+ if (Array.isArray(meta.synonyms) && meta.synonyms.length > 0) {
357
+ wire.synonyms = meta.synonyms;
358
+ }
359
+ return wire;
360
+ }
361
+ function wireToMetadata(raw) {
362
+ const meta = {
363
+ indexName: raw.index_name,
364
+ schema: raw.schema ?? {},
365
+ language: raw.language ?? "english",
366
+ partitionCount: raw.partition_count ?? 1,
367
+ bm25Params: raw.bm25_params ?? { k1: 1.2, b: 0.75 },
368
+ createdAt: raw.created_at ?? 0,
369
+ engineVersion: raw.engine_version ?? "0.0.0"
370
+ };
371
+ if (raw.vector_fields) {
372
+ for (const [fieldPath, field] of Object.entries(raw.vector_fields)) {
373
+ if (!isQuantizationMode(field.quantization)) {
374
+ throw new NarsilError(
375
+ ErrorCodes.CONFIG_INVALID,
376
+ `Index metadata names a vector quantization "${String(field.quantization)}" for "${fieldPath}" this engine does not read`,
377
+ { fieldPath, quantization: field.quantization }
378
+ );
379
+ }
380
+ }
381
+ meta.vectorFields = raw.vector_fields;
382
+ }
383
+ if (isNonNegativeInteger(raw.document_count) && Number.isSafeInteger(raw.document_count)) {
384
+ meta.documentCount = raw.document_count;
385
+ }
386
+ if (raw.embedding && typeof raw.embedding === "object" && typeof raw.embedding.fields === "object") {
387
+ meta.embedding = typeof raw.embedding.adapter === "string" ? { adapter: raw.embedding.adapter, fields: raw.embedding.fields } : { fields: raw.embedding.fields };
388
+ }
389
+ if (typeof raw.surface_forms_enabled === "boolean") {
390
+ meta.surfaceForms = raw.surface_forms_enabled;
391
+ }
392
+ if (typeof raw.analysis_revision === "string") {
393
+ meta.analysisRevision = raw.analysis_revision;
394
+ }
395
+ if (typeof raw.tokenizer === "string") {
396
+ meta.tokenizer = raw.tokenizer;
397
+ }
398
+ if (typeof raw.stop_words === "string") {
399
+ meta.stopWords = raw.stop_words;
400
+ }
401
+ if (Array.isArray(raw.stop_word_list) && raw.stop_word_list.every((word) => typeof word === "string")) {
402
+ meta.stopWordList = raw.stop_word_list;
403
+ }
404
+ if (raw.partition_limits && typeof raw.partition_limits === "object") {
405
+ const limits = {
406
+ ...isPositiveInteger(raw.partition_limits.max_docs_per_partition) ? { maxDocsPerPartition: raw.partition_limits.max_docs_per_partition } : {},
407
+ ...isPositiveInteger(raw.partition_limits.max_partitions) ? { maxPartitions: raw.partition_limits.max_partitions } : {},
408
+ ...isWatermarkFraction(raw.partition_limits.watermark) ? { watermark: raw.partition_limits.watermark } : {}
409
+ };
410
+ if (Object.keys(limits).length > 0) {
411
+ meta.partitionLimits = limits;
412
+ }
413
+ }
414
+ if (raw.default_scoring === "local" || raw.default_scoring === "dfs" || raw.default_scoring === "broadcast") {
415
+ meta.defaultScoring = raw.default_scoring;
416
+ }
417
+ if (typeof raw.track_positions === "boolean") {
418
+ meta.trackPositions = raw.track_positions;
419
+ }
420
+ if (typeof raw.strict === "boolean") {
421
+ meta.strict = raw.strict;
422
+ }
423
+ if (Array.isArray(raw.required) && raw.required.every((field) => typeof field === "string")) {
424
+ meta.required = raw.required;
425
+ }
426
+ if (raw.vector_promotion && typeof raw.vector_promotion === "object") {
427
+ const promotion = raw.vector_promotion;
428
+ const hnsw = promotion.hnsw_config && typeof promotion.hnsw_config === "object" ? promotion.hnsw_config : void 0;
429
+ const metric = hnsw?.metric;
430
+ const restored = {};
431
+ const hnswConfig = {};
432
+ if (isPositiveInteger(hnsw?.m)) {
433
+ hnswConfig.m = hnsw?.m;
434
+ }
435
+ if (isPositiveInteger(hnsw?.ef_construction)) {
436
+ hnswConfig.efConstruction = hnsw?.ef_construction;
437
+ }
438
+ if (metric === "cosine" || metric === "dotProduct" || metric === "euclidean") {
439
+ hnswConfig.metric = metric;
440
+ }
441
+ if (isPositiveInteger(promotion.threshold)) {
442
+ restored.threshold = promotion.threshold;
443
+ }
444
+ if (isSelectivityRatio(promotion.filter_threshold)) {
445
+ restored.filterThreshold = promotion.filter_threshold;
446
+ }
447
+ if (Object.keys(hnswConfig).length > 0) {
448
+ restored.hnswConfig = hnswConfig;
449
+ }
450
+ if (promotion.quantization !== void 0) {
451
+ if (!isQuantizationMode(promotion.quantization)) {
452
+ throw new NarsilError(
453
+ ErrorCodes.CONFIG_INVALID,
454
+ `Index metadata names a vector quantization "${String(promotion.quantization)}" this engine does not read`,
455
+ { quantization: promotion.quantization }
456
+ );
457
+ }
458
+ restored.quantization = promotion.quantization;
459
+ }
460
+ if (promotion.storage !== void 0) {
461
+ if (!isStorageMode(promotion.storage)) {
462
+ throw new NarsilError(
463
+ ErrorCodes.CONFIG_INVALID,
464
+ `Index metadata names a vector storage "${String(promotion.storage)}" this engine does not read`,
465
+ { storage: promotion.storage }
466
+ );
467
+ }
468
+ restored.storage = promotion.storage;
469
+ }
470
+ if (Object.keys(restored).length > 0) {
471
+ meta.vectorPromotion = restored;
472
+ }
473
+ }
474
+ if (raw.pattern_value_limit !== void 0) {
475
+ if (!isPositiveInteger(raw.pattern_value_limit)) {
476
+ throw new NarsilError(
477
+ ErrorCodes.CONFIG_INVALID,
478
+ `The pattern value limit in the index metadata is ${String(raw.pattern_value_limit)}, but the limit must be a whole number of code points`,
479
+ { patternValueLimit: String(raw.pattern_value_limit) }
480
+ );
481
+ }
482
+ meta.patternValueLimit = raw.pattern_value_limit;
483
+ }
484
+ if (typeof raw.index_uuid === "string" && raw.index_uuid.length > 0) {
485
+ meta.indexUuid = raw.index_uuid;
486
+ }
487
+ if (Array.isArray(raw.held_partitions) && raw.held_partitions.every(isNonNegativeInteger)) {
488
+ meta.heldPartitions = raw.held_partitions;
489
+ }
490
+ if (raw.synonyms !== void 0 && raw.synonyms !== null) meta.synonyms = raw.synonyms;
491
+ return meta;
492
+ }
493
+ function isNonNegativeInteger(value) {
494
+ return typeof value === "number" && Number.isInteger(value) && value >= 0;
495
+ }
496
+ function isPositiveInteger(value) {
497
+ return typeof value === "number" && Number.isInteger(value) && value >= 1;
498
+ }
499
+ function isSelectivityRatio(value) {
500
+ return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1;
501
+ }
502
+ function isWatermarkFraction(value) {
503
+ return typeof value === "number" && value > 0 && value <= 1;
504
+ }
505
+ function serializeMetadata(meta) {
506
+ const wire = metadataToWire(meta);
507
+ return encode(wire);
508
+ }
509
+ function deserializeMetadata(data) {
510
+ const raw = decode(data);
511
+ return wireToMetadata(raw);
512
+ }
513
+
514
+ // src/serialization/envelope.ts
515
+ var CURRENT_ENVELOPE_VERSION = 1;
516
+ var SNAPSHOT_ENVELOPE_VERSION = 2;
517
+ var INDEX_SNAPSHOT_ENVELOPE_VERSION = 3;
518
+ async function collectStream(stream) {
519
+ const reader = stream.getReader();
520
+ const chunks = [];
521
+ let totalLength = 0;
522
+ for (; ; ) {
523
+ const { done, value } = await reader.read();
524
+ if (done) break;
525
+ chunks.push(value);
526
+ totalLength += value.length;
527
+ }
528
+ const result = new Uint8Array(totalLength);
529
+ let offset = 0;
530
+ for (const chunk of chunks) {
531
+ result.set(chunk, offset);
532
+ offset += chunk.length;
533
+ }
534
+ return result;
535
+ }
536
+ function toArrayBuffer(data) {
537
+ return data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength);
538
+ }
539
+ async function compressGzip(data) {
540
+ const blob = new Blob([toArrayBuffer(data)]);
541
+ const compressed = blob.stream().pipeThrough(new CompressionStream("gzip"));
542
+ return collectStream(compressed);
543
+ }
544
+ async function decompressGzip(data) {
545
+ const blob = new Blob([toArrayBuffer(data)]);
546
+ const decompressed = blob.stream().pipeThrough(new DecompressionStream("gzip"));
547
+ return collectStream(decompressed);
548
+ }
549
+ function buildFlags(options) {
550
+ const useCompression = options.compression === "gzip";
551
+ return {
552
+ compressionEnabled: useCompression,
553
+ compressionAlgorithm: useCompression ? "gzip" : "none",
554
+ checksumPresent: options.checksum ?? false,
555
+ encryptionEnabled: false
556
+ };
557
+ }
558
+ async function packEnvelope(payloadBytes, options) {
559
+ const flags = buildFlags(options);
560
+ let finalPayload = payloadBytes;
561
+ if (flags.compressionEnabled) {
562
+ finalPayload = await compressGzip(payloadBytes);
563
+ }
564
+ const checksum = flags.checksumPresent ? crc32(finalPayload) : 0;
565
+ const [major, minor, patch] = engineVersionParts();
566
+ const envelopeFormatVersion = options.envelopeFormatVersion ?? CURRENT_ENVELOPE_VERSION;
567
+ const header = {
568
+ magic: "NRSL",
569
+ envelopeFormatVersion,
570
+ engineVersionMajor: major,
571
+ engineVersionMinor: minor,
572
+ engineVersionPatch: patch,
573
+ payloadLength: finalPayload.length,
574
+ flags,
575
+ checksum,
576
+ reserved: new Uint8Array(14)
577
+ };
578
+ const headerBytes = writeHeader(header);
579
+ const envelope = new Uint8Array(HEADER_SIZE + finalPayload.length);
580
+ envelope.set(headerBytes, 0);
581
+ envelope.set(finalPayload, HEADER_SIZE);
582
+ return envelope;
583
+ }
584
+ async function unpackEnvelope(data, maxAcceptedVersion = CURRENT_ENVELOPE_VERSION) {
585
+ const header = readHeader(data);
586
+ if (header.envelopeFormatVersion > maxAcceptedVersion) {
587
+ throw new NarsilError(
588
+ ErrorCodes.ENVELOPE_VERSION_MISMATCH,
589
+ `This data holds Narsil envelope format v${header.envelopeFormatVersion}, while this build of @delali/narsil reads up to v${maxAcceptedVersion}, so upgrade the package to read it.`
590
+ );
591
+ }
592
+ if (header.flags.encryptionEnabled) {
593
+ throw new NarsilError(
594
+ ErrorCodes.ENVELOPE_VERSION_MISMATCH,
595
+ "Encrypted envelopes are not supported in this version of Narsil"
596
+ );
597
+ }
598
+ const compAlgo = header.flags.compressionAlgorithm;
599
+ if (header.flags.compressionEnabled && compAlgo !== "gzip" && compAlgo !== "none") {
600
+ throw new NarsilError(
601
+ ErrorCodes.ENVELOPE_VERSION_MISMATCH,
602
+ `Compression algorithm "${compAlgo}" is not supported in this version of Narsil`
603
+ );
604
+ }
605
+ const availablePayload = data.length - HEADER_SIZE;
606
+ if (availablePayload < header.payloadLength) {
607
+ throw new NarsilError(
608
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
609
+ `Envelope data is truncated: header declares ${header.payloadLength} bytes of payload but only ${availablePayload} bytes are available`
610
+ );
611
+ }
612
+ let payloadBytes = new Uint8Array(data.slice(HEADER_SIZE, HEADER_SIZE + header.payloadLength));
613
+ if (header.flags.checksumPresent) {
614
+ const computed = crc32(payloadBytes);
615
+ if (computed !== header.checksum) {
616
+ throw new NarsilError(ErrorCodes.PERSISTENCE_CRC_MISMATCH, "CRC32 checksum mismatch: data may be corrupted", {
617
+ expected: header.checksum,
618
+ computed
619
+ });
620
+ }
621
+ }
622
+ if (header.flags.compressionEnabled) {
623
+ payloadBytes = await decompressGzip(payloadBytes);
624
+ }
625
+ return { header, payloadBytes };
626
+ }
627
+ async function unpackEnvelopeBytes(data) {
628
+ return unpackEnvelope(data, SNAPSHOT_ENVELOPE_VERSION);
629
+ }
630
+ async function unpackIndexSnapshotEnvelope(data) {
631
+ const { header, payloadBytes } = await unpackEnvelope(data, INDEX_SNAPSHOT_ENVELOPE_VERSION);
632
+ if (header.envelopeFormatVersion !== INDEX_SNAPSHOT_ENVELOPE_VERSION) {
633
+ throw new NarsilError(
634
+ ErrorCodes.ENVELOPE_VERSION_MISMATCH,
635
+ `Expected an index snapshot envelope of format v${INDEX_SNAPSHOT_ENVELOPE_VERSION}, received format v${header.envelopeFormatVersion}`
636
+ );
637
+ }
638
+ return payloadBytes;
639
+ }
640
+ async function writeMetadataEnvelope(meta, options = {}) {
641
+ const payloadBytes = serializeMetadata(meta);
642
+ return packEnvelope(payloadBytes, options);
643
+ }
644
+ async function readMetadataEnvelope(data) {
645
+ const { header, payloadBytes } = await unpackEnvelope(data);
646
+ const metadata = deserializeMetadata(payloadBytes);
647
+ return { header, metadata };
648
+ }
649
+
650
+ // src/persistence/durability/segment/manifest-fields.ts
651
+ function invalidManifest(message, details) {
652
+ throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Segment manifest ${message}`, details);
653
+ }
654
+ function isRecord(value) {
655
+ return typeof value === "object" && value !== null && !Array.isArray(value) && !(value instanceof Uint8Array);
656
+ }
657
+ function isId(value) {
658
+ return typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= MAX_SEGMENT_ID;
659
+ }
660
+ function isUint32(value) {
661
+ return typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= 4294967295;
662
+ }
663
+ function isSequenceNumber(value) {
664
+ return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
665
+ }
666
+ function readFileRef(raw, expectedKey, what) {
667
+ if (raw.key !== expectedKey) {
668
+ invalidManifest(`lists the ${what} under the key "${String(raw.key)}" where it belongs at "${expectedKey}"`, {
669
+ key: raw.key,
670
+ expected: expectedKey
671
+ });
672
+ }
673
+ const bytes = raw.bytes;
674
+ if (typeof bytes !== "number" || !Number.isInteger(bytes) || bytes < HEADER_SIZE || bytes > MAX_FILE_BYTES) {
675
+ invalidManifest(`gives the ${what} "${expectedKey}" an invalid length`, { key: expectedKey, bytes });
676
+ }
677
+ if (!isUint32(raw.checksum)) invalidManifest(`gives the ${what} "${expectedKey}" an invalid checksum`);
678
+ return { key: expectedKey, bytes, checksum: raw.checksum };
679
+ }
680
+
681
+ // src/persistence/durability/segment/manifest-vectors-decode.ts
682
+ function normalizeFile(raw, indexName, fieldPath, nextFileId, seenIds) {
683
+ if (!isRecord(raw) || !isId(raw.id)) invalidManifest("has an invalid vector file reference", { fieldPath });
684
+ if (raw.id >= nextFileId || seenIds.has(raw.id)) {
685
+ invalidManifest(`lists vector file ${raw.id} of "${fieldPath}" twice or at its next file id`, { fieldPath });
686
+ }
687
+ seenIds.add(raw.id);
688
+ const count = raw.count;
689
+ if (typeof count !== "number" || !Number.isInteger(count) || count < 0 || count > VECTOR_FILE_MAX_VECTORS) {
690
+ invalidManifest(`gives vector file ${raw.id} of "${fieldPath}" an invalid count`, { fieldPath, count });
691
+ }
692
+ const dead = raw.dead ?? null;
693
+ if (dead !== null && !(dead instanceof Uint8Array)) invalidManifest("has dead vectors that are not bytes");
694
+ if (dead !== null && dead.byteLength !== deadBytesFor(count)) {
695
+ invalidManifest(`marks the dead vectors of file ${raw.id} of "${fieldPath}" with the wrong number of bytes`);
696
+ }
697
+ const ref = readFileRef(raw, vectorFileKey(indexName, fieldPath, raw.id), "vector file");
698
+ return { ...ref, id: raw.id, count, dead };
699
+ }
700
+ function normalizeGraphParts(raw, indexName, fieldPath, generation) {
701
+ if (!Array.isArray(raw)) invalidManifest(`gives "${fieldPath}" no list of graph parts`, { fieldPath });
702
+ if (raw.length > 0 && generation === 0) invalidManifest(`lists graph parts of "${fieldPath}" at generation 0`);
703
+ return raw.map((part, at) => {
704
+ if (!isRecord(part)) invalidManifest(`has an invalid graph part of "${fieldPath}"`, { fieldPath });
705
+ return readFileRef(part, graphPartKey(indexName, fieldPath, generation, at), "graph part");
706
+ });
707
+ }
708
+ function normalizeField(raw, indexName, seenFields) {
709
+ if (!isRecord(raw) || typeof raw.fieldPath !== "string") invalidManifest("has an invalid vector reference");
710
+ const fieldPath = raw.fieldPath;
711
+ if (seenFields.has(fieldPath)) invalidManifest(`lists the vector field "${fieldPath}" twice`, { fieldPath });
712
+ seenFields.add(fieldPath);
713
+ if (!isId(raw.nextFileId)) invalidManifest(`gives "${fieldPath}" an invalid next file id`, { fieldPath });
714
+ if (!Array.isArray(raw.files)) invalidManifest(`gives "${fieldPath}" no list of vector files`, { fieldPath });
715
+ if (!isId(raw.graphGeneration)) invalidManifest(`gives "${fieldPath}" an invalid graph generation`, { fieldPath });
716
+ const nextFileId = raw.nextFileId;
717
+ const seenIds = /* @__PURE__ */ new Set();
718
+ return {
719
+ fieldPath,
720
+ nextFileId,
721
+ files: raw.files.map((file) => normalizeFile(file, indexName, fieldPath, nextFileId, seenIds)),
722
+ graphGeneration: raw.graphGeneration,
723
+ graphParts: normalizeGraphParts(raw.graphParts, indexName, fieldPath, raw.graphGeneration)
724
+ };
725
+ }
726
+ function normalizeVectorFields(raw, indexName) {
727
+ if (!Array.isArray(raw)) invalidManifest("has malformed vector references");
728
+ const seenFields = /* @__PURE__ */ new Set();
729
+ return raw.map((field) => normalizeField(field, indexName, seenFields));
730
+ }
731
+
732
+ // src/persistence/durability/segment/manifest-decode.ts
733
+ async function openManifestPayload(data) {
734
+ const { payload } = await openChecksummedEnvelope(data, {
735
+ envelopeFormatVersion: MANIFEST_ENVELOPE_VERSION,
736
+ what: "segment manifest",
737
+ allowsCompression: true
738
+ });
739
+ const raw = decode(payload);
740
+ if (!isRecord(raw)) invalidManifest("is not a map");
741
+ return raw;
742
+ }
743
+ function listedKeysOf(raw) {
744
+ const keys = [];
745
+ for (const partition of Array.isArray(raw.partitions) ? raw.partitions : []) {
746
+ if (!isRecord(partition) || !Array.isArray(partition.segments)) continue;
747
+ for (const segment of partition.segments)
748
+ if (isRecord(segment) && typeof segment.key === "string") keys.push(segment.key);
749
+ }
750
+ for (const field of Array.isArray(raw.vectors) ? raw.vectors : []) {
751
+ if (!isRecord(field)) continue;
752
+ for (const list of [field.files, field.graphParts]) {
753
+ if (!Array.isArray(list)) continue;
754
+ for (const file of list) if (isRecord(file) && typeof file.key === "string") keys.push(file.key);
755
+ }
756
+ }
757
+ return keys;
758
+ }
759
+ async function decodeSegmentManifest(data, indexName) {
760
+ return decodeManifestPayload(await openManifestPayload(data), indexName);
761
+ }
762
+ function decodeManifestPayload(raw, indexName) {
763
+ if (raw.version === EARLIER_SEGMENT_MANIFEST_VERSION) return { kind: "earlier", payload: raw };
764
+ if (raw.version !== SEGMENT_MANIFEST_VERSION) {
765
+ throw new NarsilError(
766
+ ErrorCodes.ENVELOPE_VERSION_MISMATCH,
767
+ `The segment manifest of index "${indexName}" holds layout ${String(raw.version)}, which this release cannot open. Export the index with the release that wrote it and import the export`,
768
+ { indexName, version: raw.version }
769
+ );
770
+ }
771
+ const schema = raw.schema;
772
+ if (!isRecord(schema)) invalidManifest("is missing its schema");
773
+ requireValidFieldTypes(schema);
774
+ if (typeof raw.language !== "string") invalidManifest("is missing its language");
775
+ return {
776
+ kind: "current",
777
+ manifest: {
778
+ version: SEGMENT_MANIFEST_VERSION,
779
+ schema,
780
+ language: raw.language,
781
+ checkpoint: normalizeCheckpoint(raw.checkpoint),
782
+ partitions: normalizePartitions(raw.partitions, indexName),
783
+ vectors: normalizeVectorFields(raw.vectors, indexName)
784
+ }
785
+ };
786
+ }
787
+ function normalizeCheckpoint(raw) {
788
+ if (!Array.isArray(raw)) invalidManifest("is missing its checkpoint list");
789
+ const seen = /* @__PURE__ */ new Set();
790
+ return raw.map((entry) => {
791
+ if (!isRecord(entry) || !isUint32(entry.partitionId) || !isSequenceNumber(entry.lastSeqNo) || !isSequenceNumber(entry.primaryTerm) || seen.has(entry.partitionId)) {
792
+ invalidManifest("has an invalid partition checkpoint", { entry });
793
+ }
794
+ seen.add(entry.partitionId);
795
+ return { partitionId: entry.partitionId, lastSeqNo: entry.lastSeqNo, primaryTerm: entry.primaryTerm };
796
+ });
797
+ }
798
+ function normalizePartitions(raw, indexName) {
799
+ if (!Array.isArray(raw)) invalidManifest("is missing partition entries");
800
+ const seen = /* @__PURE__ */ new Set();
801
+ return raw.map((entry) => {
802
+ if (!isRecord(entry) || !isUint32(entry.partitionId) || seen.has(entry.partitionId)) {
803
+ invalidManifest("has a partition without a valid id", { partitionId: isRecord(entry) ? entry.partitionId : null });
804
+ }
805
+ seen.add(entry.partitionId);
806
+ const segments = normalizeSegments(entry.segments, indexName, entry.partitionId);
807
+ return {
808
+ partitionId: entry.partitionId,
809
+ nextSegmentId: readNextSegmentId(entry.nextSegmentId, segments, entry.partitionId),
810
+ segments
811
+ };
812
+ });
813
+ }
814
+ function normalizeSegments(raw, indexName, partitionId) {
815
+ if (!Array.isArray(raw)) invalidManifest("has malformed segment references", { partitionId });
816
+ if (raw.length > MAX_SEGMENTS_PER_PARTITION) {
817
+ invalidManifest(`lists ${raw.length} segments for partition ${partitionId}, above ${MAX_SEGMENTS_PER_PARTITION}`);
818
+ }
819
+ const seenIds = /* @__PURE__ */ new Set();
820
+ return raw.map((segment) => {
821
+ if (!isRecord(segment) || !isId(segment.id) || !isUint32(segment.docCount) || !isUint32(segment.tombstoneCount)) {
822
+ invalidManifest("has an invalid segment reference", { partitionId });
823
+ }
824
+ if (seenIds.has(segment.id)) invalidManifest(`reuses segment id ${segment.id} in partition ${partitionId}`);
825
+ seenIds.add(segment.id);
826
+ const ref = readFileRef(segment, segmentKey(indexName, partitionId, segment.id), "segment");
827
+ return { ...ref, id: segment.id, docCount: segment.docCount, tombstoneCount: segment.tombstoneCount };
828
+ });
829
+ }
830
+ function readNextSegmentId(value, segments, partitionId) {
831
+ if (!isId(value)) invalidManifest("has an invalid next segment id", { partitionId });
832
+ for (const segment of segments) {
833
+ if (segment.id >= value) {
834
+ invalidManifest(`next segment id ${value} does not exceed segment id ${segment.id}`, { partitionId });
835
+ }
836
+ }
837
+ return value;
838
+ }
839
+ var VECTOR_FILE_ENVELOPE_VERSION = 2;
840
+ var GRAPH_PART_ENVELOPE_VERSION = 2;
841
+ function liveVectorsOf(ref) {
842
+ let live = 0;
843
+ for (const file of ref.files) live += file.count - countDead(file.dead, file.count);
844
+ return live;
845
+ }
846
+ async function openListedFile(store, ref, what, allowsCompression) {
847
+ const bytes = await readListedFile(store, ref.key, { bytes: ref.bytes, what });
848
+ const { header, payload } = await openChecksummedEnvelope(bytes, {
849
+ envelopeFormatVersion: VECTOR_FILE_ENVELOPE_VERSION,
850
+ what: `${what} "${ref.key}"`,
851
+ allowsCompression
852
+ });
853
+ requireListedChecksum(what, ref.key, header.checksum, ref.checksum);
854
+ return { payload, payloadLength: header.payloadLength };
855
+ }
856
+ async function readGraphParts(store, ref) {
857
+ if (ref.graphParts.length === 0) return null;
858
+ const parts = [];
859
+ for (const part of ref.graphParts) {
860
+ const { payload } = await openListedFile(store, part, "graph part", true);
861
+ const decoded = decodeVectorGraphPartPayload(decode(payload));
862
+ if (decoded.fieldName !== ref.fieldPath) {
863
+ throw new NarsilError(
864
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
865
+ `The graph part "${part.key}" names field "${decoded.fieldName}" where the manifest lists "${ref.fieldPath}"`,
866
+ { key: part.key }
867
+ );
868
+ }
869
+ parts.push(decoded);
870
+ }
871
+ return assembleGraphParts(parts);
872
+ }
873
+ async function readVectorFile(store, file, fieldPath) {
874
+ const { payload: bytes, payloadLength } = await openListedFile(store, file, "vector file", false);
875
+ const payload = decodeVectorFilePayload(decode(bytes));
876
+ if (payload.docIds.length !== file.count) {
877
+ throw new NarsilError(
878
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
879
+ `The vector file "${file.key}" holds ${payload.docIds.length} vectors where the manifest counts ${file.count}`,
880
+ { key: file.key, held: payload.docIds.length, count: file.count }
881
+ );
882
+ }
883
+ if (payload.fieldName !== fieldPath) {
884
+ throw new NarsilError(
885
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
886
+ `The vector file "${file.key}" names field "${payload.fieldName}" where the manifest lists "${fieldPath}"`,
887
+ { key: file.key }
888
+ );
889
+ }
890
+ return { payload, payloadLength };
891
+ }
892
+ async function loadVectorField(store, ref, vectorIndex) {
893
+ const restore = vectorIndex.restoreCheckpoint({
894
+ liveVectors: liveVectorsOf(ref),
895
+ holdsGraph: ref.graphParts.length > 0,
896
+ graph: await readGraphParts(store, ref)
897
+ });
898
+ for (const file of ref.files) {
899
+ const { payload, payloadLength } = await readVectorFile(store, file, ref.fieldPath);
900
+ restore.addFile({
901
+ id: file.id,
902
+ key: file.key,
903
+ payload,
904
+ dead: file.dead,
905
+ location: store.pathOf === void 0 ? null : {
906
+ path: await store.pathOf(file.key),
907
+ vectorsOffset: HEADER_SIZE + payloadLength - payload.vectors.byteLength
908
+ }
909
+ });
910
+ }
911
+ restore.finish();
912
+ }
913
+ var EMPTY_BIN = new Uint8Array(0);
914
+ var EMPTY_BIN_BYTES = 2;
915
+ var BIN8 = 196;
916
+ var BIN16 = 197;
917
+ var BIN32 = 198;
918
+ var BIN8_MAX_BYTES = 255;
919
+ var BIN16_MAX_BYTES = 65535;
920
+ function binHeader(byteLength) {
921
+ if (byteLength <= BIN8_MAX_BYTES) return Uint8Array.of(BIN8, byteLength);
922
+ if (byteLength <= BIN16_MAX_BYTES) return Uint8Array.of(BIN16, byteLength >> 8, byteLength & 255);
923
+ const header = new Uint8Array(5);
924
+ header[0] = BIN32;
925
+ new DataView(header.buffer).setUint32(1, byteLength, false);
926
+ return header;
927
+ }
928
+ function chunksEndingInBytes(withTheBytes, withEmptyBytesInTheirPlace, bytes) {
929
+ const head = encode(withEmptyBytesInTheirPlace);
930
+ const endsWithTheEmptyBin = head[head.length - EMPTY_BIN_BYTES] === BIN8 && head[head.length - 1] === 0;
931
+ if (!endsWithTheEmptyBin) return [encode(withTheBytes)];
932
+ return [head.subarray(0, head.length - EMPTY_BIN_BYTES), binHeader(bytes.byteLength), bytes];
933
+ }
934
+ function vectorPartChunks(part) {
935
+ return chunksEndingInBytes(part, { ...part, vectors: EMPTY_BIN }, part.vectors);
936
+ }
937
+
938
+ // src/persistence/durability/segment/vector-write.ts
939
+ function keptFileRef(input, prior, kept) {
940
+ const listed = prior?.files.find((file) => file.key === kept.key) ?? input.priorFileRefs?.get(kept.key);
941
+ if (listed === void 0) {
942
+ throw new NarsilError(
943
+ ErrorCodes.PERSISTENCE_SAVE_FAILED,
944
+ `The checkpoint keeps the vector file "${kept.key}", which the previous manifest does not list`,
945
+ { key: kept.key }
946
+ );
947
+ }
948
+ return { ...kept, bytes: listed.bytes, checksum: listed.checksum };
949
+ }
950
+ async function writeGraphParts(input, fieldPath, plan, generation) {
951
+ const parts = plan.graphParts ?? [];
952
+ const refs = [];
953
+ for (let part = 0; part < parts.length; part++) {
954
+ const key = graphPartKey(input.indexName, fieldPath, generation, part);
955
+ const packed = await packChecksummedEnvelope(
956
+ [encode(parts[part])],
957
+ GRAPH_PART_ENVELOPE_VERSION,
958
+ `graph part "${key}"`
959
+ );
960
+ await input.store.write(key, packed.chunks);
961
+ refs.push({ key, bytes: packed.byteLength, checksum: packed.checksum });
962
+ }
963
+ return refs;
964
+ }
965
+ async function writeField(input, fieldPath, plan, outcome) {
966
+ const prior = input.priorVectors.find((ref) => ref.fieldPath === fieldPath);
967
+ if (plan.unchanged && prior !== void 0) {
968
+ outcome.refs.push(prior);
969
+ return;
970
+ }
971
+ let nextFileId = prior?.nextFileId ?? 0;
972
+ const files = plan.kept.map((kept) => keptFileRef(input, prior, kept));
973
+ const written = [];
974
+ for (let index = 0; index < plan.newFiles; index++) {
975
+ const { payload, ordinals, dead } = plan.readNewFile(index);
976
+ const id = nextFileId;
977
+ nextFileId += 1;
978
+ const key = vectorFileKey(input.indexName, fieldPath, id);
979
+ const packed = await packChecksummedEnvelope(
980
+ vectorPartChunks(payload),
981
+ VECTOR_FILE_ENVELOPE_VERSION,
982
+ `vector file "${key}"`
983
+ );
984
+ await input.store.write(key, packed.chunks);
985
+ files.push({ id, key, count: payload.docIds.length, dead, bytes: packed.byteLength, checksum: packed.checksum });
986
+ written.push({ id, key, ordinals });
987
+ outcome.layouts.push({
988
+ fieldPath,
989
+ key,
990
+ docIds: payload.docIds,
991
+ dead,
992
+ vectorsOffset: packed.byteLength - payload.vectors.byteLength
993
+ });
994
+ }
995
+ const priorGeneration = prior?.graphGeneration ?? 0;
996
+ const holdsGraph = plan.graphParts !== null && plan.graphParts.length > 0;
997
+ const graphGeneration = holdsGraph ? priorGeneration + 1 : priorGeneration;
998
+ const graphParts = holdsGraph ? await writeGraphParts(input, fieldPath, plan, graphGeneration) : [];
999
+ outcome.refs.push({ fieldPath, nextFileId, files, graphGeneration, graphParts });
1000
+ outcome.written.push({ fieldPath, plan, files: written });
1001
+ }
1002
+ async function writeLiveVectors(input) {
1003
+ const outcome = { refs: [], layouts: [], written: [] };
1004
+ for (const [fieldPath, plan] of input.plans) await writeField(input, fieldPath, plan, outcome);
1005
+ return outcome;
1006
+ }
1007
+
1008
+ // src/persistence/durability/seq-owner.ts
1009
+ var SINGLE_NODE_PRIMARY_TERM = 1;
1010
+ function createSeqOwner(startSeqNo, primaryTerm = SINGLE_NODE_PRIMARY_TERM) {
1011
+ let current = startSeqNo;
1012
+ let term = primaryTerm;
1013
+ return {
1014
+ next() {
1015
+ current += 1;
1016
+ return current;
1017
+ },
1018
+ advanceTo(seqNo, nextPrimaryTerm) {
1019
+ current = seqNo;
1020
+ term = nextPrimaryTerm;
1021
+ },
1022
+ get current() {
1023
+ return current;
1024
+ },
1025
+ get primaryTerm() {
1026
+ return term;
1027
+ }
1028
+ };
1029
+ }
1030
+
1031
+ // src/persistence/durability/segment/load-partition.ts
1032
+ function requireAgreedSchema(agreed, schema, key) {
1033
+ for (const [field, type] of Object.entries(schema)) {
1034
+ const existing = agreed[field];
1035
+ if (existing !== void 0 && existing !== type) {
1036
+ throw new NarsilError(
1037
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
1038
+ `The segment "${key}" gives field "${field}" the type "${type}", while the index gives it "${existing}"`,
1039
+ { key, field, existing, found: type }
1040
+ );
1041
+ }
1042
+ }
1043
+ }
1044
+ function removeWhatTheSegmentReplaces(manager, segment, contents) {
1045
+ manager.beginBatchRemove();
1046
+ try {
1047
+ for (const docId of contents.tombstones) {
1048
+ if (manager.has(docId)) manager.remove(docId);
1049
+ }
1050
+ const ids = segment.arrays.idTable;
1051
+ for (let ordinal = 0; ordinal < ids.count; ordinal++) {
1052
+ const docId = ids.idAt(ordinal);
1053
+ if (manager.has(docId)) manager.remove(docId);
1054
+ }
1055
+ } finally {
1056
+ manager.endBatchRemove();
1057
+ }
1058
+ }
1059
+ function yieldToEventLoop() {
1060
+ return new Promise((resolve) => setTimeout(resolve, 0));
1061
+ }
1062
+ async function loadPartitionSegments(store, partition, manager, options) {
1063
+ for (const ref of partition.segments) {
1064
+ const contents = await readSegmentFile(store, ref);
1065
+ requireAgreedSchema(options.schema, contents.schema, ref.key);
1066
+ if (options.decodesEveryDocument) decodeEveryStoredDocument(contents.arrays.documentTable);
1067
+ const segment = createSharedFrozenSegment(shareSegmentArrays(contents.arrays, manager.partialWordAnalysis));
1068
+ removeWhatTheSegmentReplaces(manager, segment, contents);
1069
+ if (contents.arrays.documentCount > 0) manager.attachFrozenSegment(partition.partitionId, segment);
1070
+ await yieldToEventLoop();
1071
+ }
1072
+ }
1073
+
1074
+ // src/persistence/durability/segment/load.ts
1075
+ async function readSegmentManifest(store, indexName) {
1076
+ const bytes = await store.read(manifestKey(indexName));
1077
+ if (bytes === null) return null;
1078
+ return decodeSegmentManifest(bytes, indexName);
1079
+ }
1080
+ async function readCurrentManifest(store, indexName) {
1081
+ const decoded = await readSegmentManifest(store, indexName);
1082
+ if (decoded === null) return null;
1083
+ if (decoded.kind === "current") return decoded.manifest;
1084
+ throw new NarsilError(
1085
+ ErrorCodes.PERSISTENCE_SAVE_FAILED,
1086
+ `Index "${indexName}" still holds the layout of an earlier release, which recovery upgrades before any checkpoint`,
1087
+ { indexName }
1088
+ );
1089
+ }
1090
+ function growPartitionsTo(manager, highestPartitionId) {
1091
+ while (manager.partitionCount <= highestPartitionId) {
1092
+ const before = manager.partitionCount;
1093
+ try {
1094
+ manager.addPartition();
1095
+ } catch {
1096
+ break;
1097
+ }
1098
+ if (manager.partitionCount === before) break;
1099
+ }
1100
+ }
1101
+ async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodesEveryDocument = false) {
1102
+ let highestPartitionId = manifest.partitions.length - 1;
1103
+ for (const partition of manifest.partitions) {
1104
+ if (partition.partitionId > highestPartitionId) highestPartitionId = partition.partitionId;
1105
+ }
1106
+ growPartitionsTo(deps.manager, highestPartitionId);
1107
+ for (const partition of manifest.partitions) {
1108
+ if (partition.partitionId >= deps.manager.partitionCount) {
1109
+ throw new NarsilError(
1110
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
1111
+ `Segment manifest references partition ${partition.partitionId} beyond the partition count ${deps.manager.partitionCount}`,
1112
+ { indexName, partitionId: partition.partitionId, partitionCount: deps.manager.partitionCount }
1113
+ );
1114
+ }
1115
+ await loadPartitionSegments(store, partition, deps.manager, { schema: manifest.schema, decodesEveryDocument });
1116
+ }
1117
+ for (const vector of manifest.vectors) {
1118
+ const vectorIndex = deps.vectorIndexes.get(vector.fieldPath);
1119
+ if (vectorIndex === void 0) continue;
1120
+ await loadVectorField(store, vector, vectorIndex);
1121
+ }
1122
+ return manifest.checkpoint;
1123
+ }
1124
+ async function reclaimUnlistedKeys(store, indexName, referenced) {
1125
+ for (const key of await store.list(segmentsPrefix(indexName))) {
1126
+ if (!referenced.has(key)) await store.remove(key);
1127
+ }
1128
+ }
1129
+
1130
+ export { SINGLE_NODE_PRIMARY_TERM, VECTOR_FILE_ENVELOPE_VERSION, createSeqOwner, decodeManifestPayload, decodeSegmentManifest, deserializePayloadV1, dropStoredVectorValues, growPartitionsTo, indexMetadataOf, invalidManifest, isId, isRecord, isSequenceNumber, isUint32, listedKeysOf, loadPartitionSegments, loadSegmentedCheckpoint, metadataConfigOf, metadataFromConfig, openManifestPayload, patternIndexesFromWire, readCurrentManifest, readGraphParts, readMetadataEnvelope, readSegmentManifest, readVectorFile, reclaimUnlistedKeys, sanitizeSurfaceForms, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, writeLiveVectors, writeMetadataEnvelope };