@delali/narsil 0.3.1-next.86 → 0.3.1-next.87

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/dist/browser/chunk-3BXQPIXK.mjs +602 -0
  2. package/dist/browser/chunk-3ZRMZIH4.mjs +3959 -0
  3. package/dist/browser/chunk-BBDAYUCO.mjs +2703 -0
  4. package/dist/browser/chunk-FHFFWLEK.mjs +669 -0
  5. package/dist/browser/chunk-HOZW3PKP.mjs +9 -0
  6. package/dist/browser/chunk-NQW2YXEX.mjs +346 -0
  7. package/dist/browser/chunk-TBMNPGFO.mjs +22910 -0
  8. package/dist/browser/durability-wiring-7RUN7LNS.mjs +3250 -0
  9. package/dist/browser/module-E25UDADM.mjs +2766 -0
  10. package/dist/browser/snapshot-A42DRERF.mjs +888 -0
  11. package/dist/browser/write-4U6Q26TJ.mjs +206 -0
  12. package/dist/{capabilities-CAdkp2P-.d.ts → capabilities-SJPttcgQ.d.ts} +2 -2
  13. package/dist/chunk-37CD3OKS.mjs +87 -0
  14. package/dist/chunk-AQU2QHSY.mjs +7247 -0
  15. package/dist/{chunk-OLXM65YH.mjs → chunk-AV4IJKD2.mjs} +1 -18
  16. package/dist/{chunk-4D6OH4EF.mjs → chunk-AXDAS2VA.mjs} +7 -5
  17. package/dist/{chunk-LS5IRM23.mjs → chunk-BVOJIYRV.mjs} +2 -25
  18. package/dist/chunk-C7SLA6CM.mjs +19 -0
  19. package/dist/{chunk-KJ7LB3TR.mjs → chunk-CVQWQVFX.mjs} +3 -22
  20. package/dist/chunk-DJHHF2L5.mjs +17 -0
  21. package/dist/chunk-FGMGUDHX.mjs +21 -0
  22. package/dist/chunk-IKST36MA.mjs +614 -0
  23. package/dist/{chunk-2453RJYK.mjs → chunk-J2BPXTD7.mjs} +167 -176
  24. package/dist/chunk-JFJBA3PD.mjs +522 -0
  25. package/dist/chunk-K4OWQ2M3.mjs +462 -0
  26. package/dist/chunk-L5ORED7V.mjs +25 -0
  27. package/dist/chunk-MKK4OZ5W.mjs +703 -0
  28. package/dist/chunk-MXS5HWIH.mjs +515 -0
  29. package/dist/chunk-P36MUFEJ.mjs +10 -0
  30. package/dist/chunk-PR4O7CAP.mjs +18 -0
  31. package/dist/{chunk-4VBG6PEH.mjs → chunk-PSUJCOSE.mjs} +3 -2
  32. package/dist/{chunk-ZCKPRCYI.mjs → chunk-Q5DC6WAB.mjs} +1 -1
  33. package/dist/{chunk-NFF5TDRG.mjs → chunk-QZN3MIJ3.mjs} +1 -16
  34. package/dist/chunk-R7OSDAFK.mjs +44 -0
  35. package/dist/chunk-RVM7AODE.mjs +73 -0
  36. package/dist/{chunk-XYDP7CCJ.mjs → chunk-SYRFSF6V.mjs} +6 -4
  37. package/dist/{chunk-SREPGC6O.mjs → chunk-T6CNO72J.mjs} +1 -1
  38. package/dist/chunk-T7DMUVTL.mjs +1714 -0
  39. package/dist/chunk-UKSUENHK.mjs +997 -0
  40. package/dist/{chunk-B3F7LZXM.mjs → chunk-ULRPNYAG.mjs} +1220 -2318
  41. package/dist/{chunk-GNYXCLJS.mjs → chunk-URGLSNKR.mjs} +4 -3
  42. package/dist/chunk-WQSHDZ7Z.mjs +631 -0
  43. package/dist/{chunk-XPK3GB2P.mjs → chunk-XBXKPWMR.mjs} +20 -2664
  44. package/dist/{chunk-RM6FRYEC.mjs → chunk-XSZVS4S3.mjs} +3 -2
  45. package/dist/chunk-Y2F66IQJ.mjs +74 -0
  46. package/dist/chunk-YCJLVNWR.mjs +262 -0
  47. package/dist/{client-CKUtkLz3.d.ts → client-BMK53IR2.d.ts} +3 -3
  48. package/dist/client.d.ts +8 -8
  49. package/dist/{config-CSDEPraV.d.ts → config-BUR-7eVQ.d.ts} +2 -2
  50. package/dist/{config-BuIkeM5C.d.ts → config-UJx1dyX1.d.ts} +2 -2
  51. package/dist/direct-executor-LD6JW57R.mjs +18 -0
  52. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  53. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  54. package/dist/distribution.d.ts +8 -8
  55. package/dist/distribution.mjs +30 -15
  56. package/dist/durability-wiring-5S32KDDC.mjs +2797 -0
  57. package/dist/{engine-DMNZJAUq.d.ts → engine-9X6W5naj.d.ts} +2 -2
  58. package/dist/{index-DzONZUKE.d.ts → index-vCMu8OeQ.d.ts} +1 -1
  59. package/dist/index.browser.mjs +2 -38128
  60. package/dist/index.d.ts +23 -9
  61. package/dist/index.mjs +18 -14
  62. package/dist/{memory-BPKNx9Tl.d.ts → memory-CV9NAxLY.d.ts} +1 -1
  63. package/dist/module-WIDYIJLY.mjs +1819 -0
  64. package/dist/partial-words.mjs +5 -3
  65. package/dist/pattern.mjs +3 -2
  66. package/dist/persistence/durability/checkpoint-worker.d.ts +1 -1
  67. package/dist/persistence/durability/checkpoint-worker.mjs +23 -13
  68. package/dist/react.d.ts +7 -7
  69. package/dist/{schema-DgyDijhf.d.ts → schema-DtsgmNfU.d.ts} +1 -1
  70. package/dist/server.d.ts +10 -10
  71. package/dist/server.mjs +14 -9
  72. package/dist/snapshot-6BYSUPWL.mjs +469 -0
  73. package/dist/{tasks-DGyz586z.d.ts → tasks-BGaZJyoY.d.ts} +1 -1
  74. package/dist/{thread-EUUNJNKU.mjs → thread-GS65E6RM.mjs} +12 -9
  75. package/dist/vector/search-worker.d.ts +30 -3
  76. package/dist/vector/search-worker.mjs +6 -2
  77. package/dist/workers/entry.mjs +5 -3
  78. package/dist/write-HLEZTL2L.mjs +233 -0
  79. package/package.json +11 -7
  80. package/dist/chunk-IDLP54FH.mjs +0 -15198
  81. package/dist/direct-executor-SI2QV4YI.mjs +0 -19
  82. package/dist/simd-DW5AtJ6B.d.ts +0 -41
@@ -0,0 +1,703 @@
1
+ import { decodeSegmentManifest, loadPartitionSegments, loadVectorField } from './chunk-JFJBA3PD.mjs';
2
+ import { manifestKey, segmentsPrefix } from './chunk-T7DMUVTL.mjs';
3
+ import { readHeader, HEADER_SIZE, crc32, engineVersionParts, writeHeader } from './chunk-AQU2QHSY.mjs';
4
+ import { VALID_HNSW_METRICS } from './chunk-RVM7AODE.mjs';
5
+ import { requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-CVQWQVFX.mjs';
6
+ import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
7
+ import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
8
+ import { decode, encode } from '@msgpack/msgpack';
9
+
10
+ // src/serialization/stored-vector-values.ts
11
+ function dropStoredVectorValues(documents, schema) {
12
+ const vectorFieldPaths = [];
13
+ for (const [fieldPath, fieldType] of Object.entries(schema)) {
14
+ if (typeof fieldType === "string" && VECTOR_PATTERN.test(fieldType)) {
15
+ vectorFieldPaths.push(fieldPath);
16
+ }
17
+ }
18
+ if (vectorFieldPaths.length === 0) {
19
+ return;
20
+ }
21
+ for (const document of Object.values(documents)) {
22
+ for (const fieldPath of vectorFieldPaths) {
23
+ removeFieldAtPath(document.fields, fieldPath);
24
+ }
25
+ }
26
+ }
27
+ function removeFieldAtPath(fields, fieldPath) {
28
+ if (!fieldPath.includes(".")) {
29
+ delete fields[fieldPath];
30
+ return;
31
+ }
32
+ const segments = fieldPath.split(".");
33
+ let current = fields;
34
+ for (let i = 0; i < segments.length - 1; i += 1) {
35
+ const next = current[segments[i]];
36
+ if (next === null || typeof next !== "object") {
37
+ return;
38
+ }
39
+ current = next;
40
+ }
41
+ delete current[segments[segments.length - 1]];
42
+ }
43
+
44
+ // src/serialization/payload-v1/partition.ts
45
+ function sanitizeSurfaceForms(raw) {
46
+ if (raw === void 0 || raw === null || typeof raw !== "object" || Array.isArray(raw)) return void 0;
47
+ const result = /* @__PURE__ */ Object.create(null);
48
+ let count = 0;
49
+ for (const [surface, value] of Object.entries(raw)) {
50
+ if (typeof value === "number") {
51
+ result[surface] = value;
52
+ count++;
53
+ } else if (Array.isArray(value) && typeof value[0] === "number" && typeof value[1] === "string") {
54
+ result[surface] = [value[0], value[1]];
55
+ count++;
56
+ }
57
+ }
58
+ return count > 0 ? result : void 0;
59
+ }
60
+ function isWireRecord(value) {
61
+ return value !== null && typeof value === "object" && !Array.isArray(value);
62
+ }
63
+ function corruptPatternEntry(field) {
64
+ return new NarsilError(
65
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
66
+ `The pattern entry of field "${field}" in this partition must hold a list of document IDs and a map of runs`,
67
+ { field }
68
+ );
69
+ }
70
+ function patternIndexesFromWire(raw) {
71
+ if (raw === void 0 || raw === null) return {};
72
+ if (!isWireRecord(raw)) {
73
+ throw new NarsilError(
74
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
75
+ "The pattern entry of this partition must be a map of fields"
76
+ );
77
+ }
78
+ const pattern = /* @__PURE__ */ Object.create(null);
79
+ for (const [field, index] of Object.entries(raw)) {
80
+ if (!isWireRecord(index) || !isWireRecord(index.runs) || !Array.isArray(index.doc_ids)) {
81
+ throw corruptPatternEntry(field);
82
+ }
83
+ const docIds = [];
84
+ for (const docId of index.doc_ids) {
85
+ if (typeof docId !== "string") throw corruptPatternEntry(field);
86
+ docIds.push(docId);
87
+ }
88
+ pattern[field] = { docIds, runs: index.runs };
89
+ }
90
+ return { pattern };
91
+ }
92
+ function validateHnswMetric(value) {
93
+ if (value === void 0 || value === null) return void 0;
94
+ if (typeof value === "string" && VALID_HNSW_METRICS.has(value)) {
95
+ return value;
96
+ }
97
+ return void 0;
98
+ }
99
+ function wireToPartition(raw) {
100
+ const documents = {};
101
+ for (const [docId, doc] of Object.entries(raw.documents ?? {})) {
102
+ documents[docId] = {
103
+ fields: doc.fields,
104
+ fieldLengths: doc.field_lengths ?? {}
105
+ };
106
+ }
107
+ requireValidFieldTypes(raw.schema ?? {});
108
+ dropStoredVectorValues(documents, raw.schema ?? {});
109
+ const invertedIndex = {};
110
+ for (const [token, list] of Object.entries(raw.inverted_index ?? {})) {
111
+ invertedIndex[token] = {
112
+ docFrequency: list.doc_freq,
113
+ postings: (list.postings ?? []).map((p) => ({
114
+ docId: p.doc_id,
115
+ termFrequency: p.term_freq,
116
+ field: p.field,
117
+ positions: p.positions ?? []
118
+ }))
119
+ };
120
+ }
121
+ const numeric = {};
122
+ for (const [field, entries] of Object.entries(raw.field_indexes?.numeric ?? {})) {
123
+ numeric[field] = entries.map((e) => ({ value: e.value, docId: e.doc_id }));
124
+ }
125
+ const boolean = {};
126
+ for (const [field, idx] of Object.entries(raw.field_indexes?.boolean ?? {})) {
127
+ boolean[field] = { trueDocs: idx.true_docs ?? [], falseDocs: idx.false_docs ?? [] };
128
+ }
129
+ const geopoint = {};
130
+ for (const [field, entries] of Object.entries(raw.field_indexes?.geopoint ?? {})) {
131
+ geopoint[field] = entries.map((e) => ({ lat: e.lat, lon: e.lon, docId: e.doc_id }));
132
+ }
133
+ const vectorData = {};
134
+ for (const [field, data] of Object.entries(raw.vector_data ?? {})) {
135
+ vectorData[field] = {
136
+ dimension: data.dimension,
137
+ vectors: (data.vectors ?? []).map((v) => ({ docId: v.doc_id, vector: v.vector })),
138
+ hnswGraph: data.hnsw_graph ? {
139
+ entryPoint: data.hnsw_graph.entry_point,
140
+ maxLayer: data.hnsw_graph.max_layer,
141
+ m: data.hnsw_graph.m,
142
+ efConstruction: data.hnsw_graph.ef_construction,
143
+ metric: validateHnswMetric(data.hnsw_graph.metric),
144
+ nodes: data.hnsw_graph.nodes
145
+ } : null,
146
+ codes: data.codes ? { bits: data.codes.bits, centroid: data.codes.centroid, records: data.codes.records } : null
147
+ };
148
+ }
149
+ return {
150
+ indexName: raw.index_name,
151
+ partitionId: raw.partition_id,
152
+ totalPartitions: raw.total_partitions,
153
+ language: raw.language ?? "english",
154
+ schema: raw.schema ?? {},
155
+ docCount: raw.doc_count ?? 0,
156
+ avgDocLength: raw.avg_doc_length ?? 0,
157
+ documents,
158
+ invertedIndex,
159
+ fieldIndexes: {
160
+ numeric,
161
+ boolean,
162
+ enum: raw.field_indexes?.enum ?? {},
163
+ geopoint,
164
+ ...patternIndexesFromWire(raw.field_indexes?.pattern)
165
+ },
166
+ surfaceForms: sanitizeSurfaceForms(raw.surface_forms),
167
+ vectorData,
168
+ statistics: {
169
+ totalDocuments: raw.statistics?.total_documents ?? 0,
170
+ totalFieldLengths: raw.statistics?.total_field_lengths ?? {},
171
+ averageFieldLengths: raw.statistics?.average_field_lengths ?? {},
172
+ docFrequencies: raw.statistics?.doc_frequencies ?? {}
173
+ }
174
+ };
175
+ }
176
+ function deserializePayloadV1(data) {
177
+ const raw = decode(data);
178
+ return wireToPartition(raw);
179
+ }
180
+ function metadataToWire(meta) {
181
+ const wire = {
182
+ index_name: meta.indexName,
183
+ schema: meta.schema,
184
+ language: meta.language,
185
+ partition_count: meta.partitionCount,
186
+ bm25_params: meta.bm25Params,
187
+ created_at: meta.createdAt,
188
+ engine_version: meta.engineVersion
189
+ };
190
+ if (meta.vectorFields) {
191
+ wire.vector_fields = meta.vectorFields;
192
+ }
193
+ if (isNonNegativeInteger(meta.documentCount) && Number.isSafeInteger(meta.documentCount)) {
194
+ wire.document_count = meta.documentCount;
195
+ }
196
+ if (meta.embedding) {
197
+ wire.embedding = meta.embedding.adapter !== void 0 ? { adapter: meta.embedding.adapter, fields: meta.embedding.fields } : { fields: meta.embedding.fields };
198
+ }
199
+ if (meta.surfaceForms !== void 0) {
200
+ wire.surface_forms_enabled = meta.surfaceForms;
201
+ }
202
+ if (meta.analysisRevision !== void 0) {
203
+ wire.analysis_revision = meta.analysisRevision;
204
+ }
205
+ if (meta.tokenizer !== void 0) {
206
+ wire.tokenizer = meta.tokenizer;
207
+ }
208
+ if (meta.stopWords !== void 0) {
209
+ wire.stop_words = meta.stopWords;
210
+ }
211
+ if (meta.stopWordList !== void 0) {
212
+ wire.stop_word_list = meta.stopWordList;
213
+ }
214
+ if (meta.partitionLimits !== void 0) {
215
+ wire.partition_limits = {
216
+ ...meta.partitionLimits.maxDocsPerPartition !== void 0 ? { max_docs_per_partition: meta.partitionLimits.maxDocsPerPartition } : {},
217
+ ...meta.partitionLimits.maxPartitions !== void 0 ? { max_partitions: meta.partitionLimits.maxPartitions } : {},
218
+ ...meta.partitionLimits.watermark !== void 0 ? { watermark: meta.partitionLimits.watermark } : {}
219
+ };
220
+ }
221
+ if (meta.defaultScoring !== void 0) {
222
+ wire.default_scoring = meta.defaultScoring;
223
+ }
224
+ if (meta.trackPositions !== void 0) {
225
+ wire.track_positions = meta.trackPositions;
226
+ }
227
+ if (meta.strict !== void 0) {
228
+ wire.strict = meta.strict;
229
+ }
230
+ if (meta.required !== void 0) {
231
+ wire.required = meta.required;
232
+ }
233
+ if (meta.vectorPromotion !== void 0) {
234
+ const promotion = meta.vectorPromotion;
235
+ wire.vector_promotion = {
236
+ ...promotion.threshold !== void 0 ? { threshold: promotion.threshold } : {},
237
+ ...promotion.filterThreshold !== void 0 ? { filter_threshold: promotion.filterThreshold } : {},
238
+ ...promotion.hnswConfig !== void 0 ? {
239
+ hnsw_config: {
240
+ ...promotion.hnswConfig.m !== void 0 ? { m: promotion.hnswConfig.m } : {},
241
+ ...promotion.hnswConfig.efConstruction !== void 0 ? { ef_construction: promotion.hnswConfig.efConstruction } : {},
242
+ ...promotion.hnswConfig.metric !== void 0 ? { metric: promotion.hnswConfig.metric } : {}
243
+ }
244
+ } : {},
245
+ ...promotion.quantization !== void 0 ? { quantization: promotion.quantization } : {},
246
+ ...promotion.storage !== void 0 ? { storage: promotion.storage } : {}
247
+ };
248
+ }
249
+ if (meta.patternValueLimit !== void 0) {
250
+ wire.pattern_value_limit = meta.patternValueLimit;
251
+ }
252
+ if (meta.indexUuid !== void 0) {
253
+ wire.index_uuid = meta.indexUuid;
254
+ }
255
+ if (meta.heldPartitions !== void 0) {
256
+ wire.held_partitions = meta.heldPartitions;
257
+ }
258
+ return wire;
259
+ }
260
+ function wireToMetadata(raw) {
261
+ const meta = {
262
+ indexName: raw.index_name,
263
+ schema: raw.schema ?? {},
264
+ language: raw.language ?? "english",
265
+ partitionCount: raw.partition_count ?? 1,
266
+ bm25Params: raw.bm25_params ?? { k1: 1.2, b: 0.75 },
267
+ createdAt: raw.created_at ?? 0,
268
+ engineVersion: raw.engine_version ?? "0.0.0"
269
+ };
270
+ if (raw.vector_fields) {
271
+ for (const [fieldPath, field] of Object.entries(raw.vector_fields)) {
272
+ if (!isQuantizationMode(field.quantization)) {
273
+ throw new NarsilError(
274
+ ErrorCodes.CONFIG_INVALID,
275
+ `Index metadata names a vector quantization "${String(field.quantization)}" for "${fieldPath}" this engine does not read`,
276
+ { fieldPath, quantization: field.quantization }
277
+ );
278
+ }
279
+ }
280
+ meta.vectorFields = raw.vector_fields;
281
+ }
282
+ if (isNonNegativeInteger(raw.document_count) && Number.isSafeInteger(raw.document_count)) {
283
+ meta.documentCount = raw.document_count;
284
+ }
285
+ if (raw.embedding && typeof raw.embedding === "object" && typeof raw.embedding.fields === "object") {
286
+ meta.embedding = typeof raw.embedding.adapter === "string" ? { adapter: raw.embedding.adapter, fields: raw.embedding.fields } : { fields: raw.embedding.fields };
287
+ }
288
+ if (typeof raw.surface_forms_enabled === "boolean") {
289
+ meta.surfaceForms = raw.surface_forms_enabled;
290
+ }
291
+ if (typeof raw.analysis_revision === "string") {
292
+ meta.analysisRevision = raw.analysis_revision;
293
+ }
294
+ if (typeof raw.tokenizer === "string") {
295
+ meta.tokenizer = raw.tokenizer;
296
+ }
297
+ if (typeof raw.stop_words === "string") {
298
+ meta.stopWords = raw.stop_words;
299
+ }
300
+ if (Array.isArray(raw.stop_word_list) && raw.stop_word_list.every((word) => typeof word === "string")) {
301
+ meta.stopWordList = raw.stop_word_list;
302
+ }
303
+ if (raw.partition_limits && typeof raw.partition_limits === "object") {
304
+ const limits = {
305
+ ...isPositiveInteger(raw.partition_limits.max_docs_per_partition) ? { maxDocsPerPartition: raw.partition_limits.max_docs_per_partition } : {},
306
+ ...isPositiveInteger(raw.partition_limits.max_partitions) ? { maxPartitions: raw.partition_limits.max_partitions } : {},
307
+ ...isWatermarkFraction(raw.partition_limits.watermark) ? { watermark: raw.partition_limits.watermark } : {}
308
+ };
309
+ if (Object.keys(limits).length > 0) {
310
+ meta.partitionLimits = limits;
311
+ }
312
+ }
313
+ if (raw.default_scoring === "local" || raw.default_scoring === "dfs" || raw.default_scoring === "broadcast") {
314
+ meta.defaultScoring = raw.default_scoring;
315
+ }
316
+ if (typeof raw.track_positions === "boolean") {
317
+ meta.trackPositions = raw.track_positions;
318
+ }
319
+ if (typeof raw.strict === "boolean") {
320
+ meta.strict = raw.strict;
321
+ }
322
+ if (Array.isArray(raw.required) && raw.required.every((field) => typeof field === "string")) {
323
+ meta.required = raw.required;
324
+ }
325
+ if (raw.vector_promotion && typeof raw.vector_promotion === "object") {
326
+ const promotion = raw.vector_promotion;
327
+ const hnsw = promotion.hnsw_config && typeof promotion.hnsw_config === "object" ? promotion.hnsw_config : void 0;
328
+ const metric = hnsw?.metric;
329
+ const restored = {};
330
+ const hnswConfig = {};
331
+ if (isPositiveInteger(hnsw?.m)) {
332
+ hnswConfig.m = hnsw?.m;
333
+ }
334
+ if (isPositiveInteger(hnsw?.ef_construction)) {
335
+ hnswConfig.efConstruction = hnsw?.ef_construction;
336
+ }
337
+ if (metric === "cosine" || metric === "dotProduct" || metric === "euclidean") {
338
+ hnswConfig.metric = metric;
339
+ }
340
+ if (isPositiveInteger(promotion.threshold)) {
341
+ restored.threshold = promotion.threshold;
342
+ }
343
+ if (isSelectivityRatio(promotion.filter_threshold)) {
344
+ restored.filterThreshold = promotion.filter_threshold;
345
+ }
346
+ if (Object.keys(hnswConfig).length > 0) {
347
+ restored.hnswConfig = hnswConfig;
348
+ }
349
+ if (promotion.quantization !== void 0) {
350
+ if (!isQuantizationMode(promotion.quantization)) {
351
+ throw new NarsilError(
352
+ ErrorCodes.CONFIG_INVALID,
353
+ `Index metadata names a vector quantization "${String(promotion.quantization)}" this engine does not read`,
354
+ { quantization: promotion.quantization }
355
+ );
356
+ }
357
+ restored.quantization = promotion.quantization;
358
+ }
359
+ if (promotion.storage !== void 0) {
360
+ if (!isStorageMode(promotion.storage)) {
361
+ throw new NarsilError(
362
+ ErrorCodes.CONFIG_INVALID,
363
+ `Index metadata names a vector storage "${String(promotion.storage)}" this engine does not read`,
364
+ { storage: promotion.storage }
365
+ );
366
+ }
367
+ restored.storage = promotion.storage;
368
+ }
369
+ if (Object.keys(restored).length > 0) {
370
+ meta.vectorPromotion = restored;
371
+ }
372
+ }
373
+ if (raw.pattern_value_limit !== void 0) {
374
+ if (!isPositiveInteger(raw.pattern_value_limit)) {
375
+ throw new NarsilError(
376
+ ErrorCodes.CONFIG_INVALID,
377
+ `The pattern value limit in the index metadata is ${String(raw.pattern_value_limit)}, but the limit must be a whole number of code points`,
378
+ { patternValueLimit: String(raw.pattern_value_limit) }
379
+ );
380
+ }
381
+ meta.patternValueLimit = raw.pattern_value_limit;
382
+ }
383
+ if (typeof raw.index_uuid === "string" && raw.index_uuid.length > 0) {
384
+ meta.indexUuid = raw.index_uuid;
385
+ }
386
+ if (Array.isArray(raw.held_partitions) && raw.held_partitions.every(isNonNegativeInteger)) {
387
+ meta.heldPartitions = raw.held_partitions;
388
+ }
389
+ return meta;
390
+ }
391
+ function isNonNegativeInteger(value) {
392
+ return typeof value === "number" && Number.isInteger(value) && value >= 0;
393
+ }
394
+ function isPositiveInteger(value) {
395
+ return typeof value === "number" && Number.isInteger(value) && value >= 1;
396
+ }
397
+ function isSelectivityRatio(value) {
398
+ return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1;
399
+ }
400
+ function isWatermarkFraction(value) {
401
+ return typeof value === "number" && value > 0 && value <= 1;
402
+ }
403
+ function serializeMetadata(meta) {
404
+ const wire = metadataToWire(meta);
405
+ return encode(wire);
406
+ }
407
+ function deserializeMetadata(data) {
408
+ const raw = decode(data);
409
+ return wireToMetadata(raw);
410
+ }
411
+
412
+ // src/serialization/envelope.ts
413
+ var CURRENT_ENVELOPE_VERSION = 1;
414
+ var SNAPSHOT_ENVELOPE_VERSION = 2;
415
+ var INDEX_SNAPSHOT_ENVELOPE_VERSION = 3;
416
+ async function collectStream(stream) {
417
+ const reader = stream.getReader();
418
+ const chunks = [];
419
+ let totalLength = 0;
420
+ for (; ; ) {
421
+ const { done, value } = await reader.read();
422
+ if (done) break;
423
+ chunks.push(value);
424
+ totalLength += value.length;
425
+ }
426
+ const result = new Uint8Array(totalLength);
427
+ let offset = 0;
428
+ for (const chunk of chunks) {
429
+ result.set(chunk, offset);
430
+ offset += chunk.length;
431
+ }
432
+ return result;
433
+ }
434
+ function toArrayBuffer(data) {
435
+ return data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength);
436
+ }
437
+ async function compressGzip(data) {
438
+ const blob = new Blob([toArrayBuffer(data)]);
439
+ const compressed = blob.stream().pipeThrough(new CompressionStream("gzip"));
440
+ return collectStream(compressed);
441
+ }
442
+ async function decompressGzip(data) {
443
+ const blob = new Blob([toArrayBuffer(data)]);
444
+ const decompressed = blob.stream().pipeThrough(new DecompressionStream("gzip"));
445
+ return collectStream(decompressed);
446
+ }
447
+ function buildFlags(options) {
448
+ const useCompression = options.compression === "gzip";
449
+ return {
450
+ compressionEnabled: useCompression,
451
+ compressionAlgorithm: useCompression ? "gzip" : "none",
452
+ checksumPresent: options.checksum ?? false,
453
+ encryptionEnabled: false
454
+ };
455
+ }
456
+ async function packEnvelope(payloadBytes, options) {
457
+ const flags = buildFlags(options);
458
+ let finalPayload = payloadBytes;
459
+ if (flags.compressionEnabled) {
460
+ finalPayload = await compressGzip(payloadBytes);
461
+ }
462
+ const checksum = flags.checksumPresent ? crc32(finalPayload) : 0;
463
+ const [major, minor, patch] = engineVersionParts();
464
+ const envelopeFormatVersion = options.envelopeFormatVersion ?? CURRENT_ENVELOPE_VERSION;
465
+ const header = {
466
+ magic: "NRSL",
467
+ envelopeFormatVersion,
468
+ engineVersionMajor: major,
469
+ engineVersionMinor: minor,
470
+ engineVersionPatch: patch,
471
+ payloadLength: finalPayload.length,
472
+ flags,
473
+ checksum,
474
+ reserved: new Uint8Array(14)
475
+ };
476
+ const headerBytes = writeHeader(header);
477
+ const envelope = new Uint8Array(HEADER_SIZE + finalPayload.length);
478
+ envelope.set(headerBytes, 0);
479
+ envelope.set(finalPayload, HEADER_SIZE);
480
+ return envelope;
481
+ }
482
+ async function unpackEnvelope(data, maxAcceptedVersion = CURRENT_ENVELOPE_VERSION) {
483
+ const header = readHeader(data);
484
+ if (header.envelopeFormatVersion > maxAcceptedVersion) {
485
+ throw new NarsilError(
486
+ ErrorCodes.ENVELOPE_VERSION_MISMATCH,
487
+ `This data holds Narsil envelope format v${header.envelopeFormatVersion}, while this build of @delali/narsil reads up to v${maxAcceptedVersion}, so upgrade the package to read it.`
488
+ );
489
+ }
490
+ if (header.flags.encryptionEnabled) {
491
+ throw new NarsilError(
492
+ ErrorCodes.ENVELOPE_VERSION_MISMATCH,
493
+ "Encrypted envelopes are not supported in this version of Narsil"
494
+ );
495
+ }
496
+ const compAlgo = header.flags.compressionAlgorithm;
497
+ if (header.flags.compressionEnabled && compAlgo !== "gzip" && compAlgo !== "none") {
498
+ throw new NarsilError(
499
+ ErrorCodes.ENVELOPE_VERSION_MISMATCH,
500
+ `Compression algorithm "${compAlgo}" is not supported in this version of Narsil`
501
+ );
502
+ }
503
+ const availablePayload = data.length - HEADER_SIZE;
504
+ if (availablePayload < header.payloadLength) {
505
+ throw new NarsilError(
506
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
507
+ `Envelope data is truncated: header declares ${header.payloadLength} bytes of payload but only ${availablePayload} bytes are available`
508
+ );
509
+ }
510
+ let payloadBytes = new Uint8Array(data.slice(HEADER_SIZE, HEADER_SIZE + header.payloadLength));
511
+ if (header.flags.checksumPresent) {
512
+ const computed = crc32(payloadBytes);
513
+ if (computed !== header.checksum) {
514
+ throw new NarsilError(ErrorCodes.PERSISTENCE_CRC_MISMATCH, "CRC32 checksum mismatch: data may be corrupted", {
515
+ expected: header.checksum,
516
+ computed
517
+ });
518
+ }
519
+ }
520
+ if (header.flags.compressionEnabled) {
521
+ payloadBytes = await decompressGzip(payloadBytes);
522
+ }
523
+ return { header, payloadBytes };
524
+ }
525
+ async function unpackEnvelopeBytes(data) {
526
+ return unpackEnvelope(data, SNAPSHOT_ENVELOPE_VERSION);
527
+ }
528
+ async function unpackIndexSnapshotEnvelope(data) {
529
+ const { header, payloadBytes } = await unpackEnvelope(data, INDEX_SNAPSHOT_ENVELOPE_VERSION);
530
+ if (header.envelopeFormatVersion !== INDEX_SNAPSHOT_ENVELOPE_VERSION) {
531
+ throw new NarsilError(
532
+ ErrorCodes.ENVELOPE_VERSION_MISMATCH,
533
+ `Expected an index snapshot envelope of format v${INDEX_SNAPSHOT_ENVELOPE_VERSION}, received format v${header.envelopeFormatVersion}`
534
+ );
535
+ }
536
+ return payloadBytes;
537
+ }
538
+ async function writeMetadataEnvelope(meta, options = {}) {
539
+ const payloadBytes = serializeMetadata(meta);
540
+ return packEnvelope(payloadBytes, options);
541
+ }
542
+ async function readMetadataEnvelope(data) {
543
+ const { header, payloadBytes } = await unpackEnvelope(data);
544
+ const metadata = deserializeMetadata(payloadBytes);
545
+ return { header, metadata };
546
+ }
547
+
548
+ // src/persistence/durability/segment/load.ts
549
+ async function readSegmentManifest(store, indexName) {
550
+ const bytes = await store.read(manifestKey(indexName));
551
+ if (bytes === null) return null;
552
+ return decodeSegmentManifest(bytes, indexName);
553
+ }
554
+ async function readCurrentManifest(store, indexName) {
555
+ const decoded = await readSegmentManifest(store, indexName);
556
+ if (decoded === null) return null;
557
+ if (decoded.kind === "current") return decoded.manifest;
558
+ throw new NarsilError(
559
+ ErrorCodes.PERSISTENCE_SAVE_FAILED,
560
+ `Index "${indexName}" still holds the layout of an earlier release, which recovery upgrades before any checkpoint`,
561
+ { indexName }
562
+ );
563
+ }
564
+ function growPartitionsTo(manager, highestPartitionId) {
565
+ while (manager.partitionCount <= highestPartitionId) {
566
+ const before = manager.partitionCount;
567
+ try {
568
+ manager.addPartition();
569
+ } catch {
570
+ break;
571
+ }
572
+ if (manager.partitionCount === before) break;
573
+ }
574
+ }
575
+ async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodesEveryDocument = false) {
576
+ let highestPartitionId = manifest.partitions.length - 1;
577
+ for (const partition of manifest.partitions) {
578
+ if (partition.partitionId > highestPartitionId) highestPartitionId = partition.partitionId;
579
+ }
580
+ growPartitionsTo(deps.manager, highestPartitionId);
581
+ for (const partition of manifest.partitions) {
582
+ if (partition.partitionId >= deps.manager.partitionCount) {
583
+ throw new NarsilError(
584
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
585
+ `Segment manifest references partition ${partition.partitionId} beyond the partition count ${deps.manager.partitionCount}`,
586
+ { indexName, partitionId: partition.partitionId, partitionCount: deps.manager.partitionCount }
587
+ );
588
+ }
589
+ await loadPartitionSegments(store, partition, deps.manager, { schema: manifest.schema, decodesEveryDocument });
590
+ }
591
+ for (const vector of manifest.vectors) {
592
+ const vectorIndex = deps.vectorIndexes.get(vector.fieldPath);
593
+ if (vectorIndex === void 0) continue;
594
+ await loadVectorField(store, vector, vectorIndex);
595
+ }
596
+ return manifest.checkpoint;
597
+ }
598
+ async function reclaimUnlistedKeys(store, indexName, referenced) {
599
+ for (const key of await store.list(segmentsPrefix(indexName))) {
600
+ if (!referenced.has(key)) await store.remove(key);
601
+ }
602
+ }
603
+ function validateHnswMetric2(value) {
604
+ if (value === void 0 || value === null) return void 0;
605
+ if (typeof value === "string" && VALID_HNSW_METRICS.has(value)) {
606
+ return value;
607
+ }
608
+ return void 0;
609
+ }
610
+ function storedDocumentsOf(raw) {
611
+ requireValidFieldTypes(raw.schema ?? {});
612
+ const documents = {};
613
+ for (const [docId, doc] of Object.entries(raw.documents ?? {})) {
614
+ documents[docId] = {
615
+ fields: doc.fields,
616
+ fieldLengths: doc.field_lengths ?? {}
617
+ };
618
+ }
619
+ dropStoredVectorValues(documents, raw.schema ?? {});
620
+ return documents;
621
+ }
622
+ function documentsOfPayloadV2(data) {
623
+ const raw = decode(data);
624
+ return { documents: storedDocumentsOf(raw), schema: raw.schema ?? {} };
625
+ }
626
+ function deserializePayloadV2(data) {
627
+ const raw = decode(data);
628
+ const documents = storedDocumentsOf(raw);
629
+ const fieldNames = raw.inverted_index?.field_names ?? [];
630
+ const invertedIndex = {};
631
+ for (const [token, list] of Object.entries(raw.inverted_index?.entries ?? {})) {
632
+ const count = list.ids.length;
633
+ const postings = new Array(count);
634
+ for (let i = 0; i < count; i++) {
635
+ postings[i] = {
636
+ docId: list.ids[i],
637
+ termFrequency: list.tf[i],
638
+ field: fieldNames[list.fi[i]] ?? "",
639
+ positions: list.pos ? list.pos[i] : []
640
+ };
641
+ }
642
+ invertedIndex[token] = {
643
+ docFrequency: list.df,
644
+ postings
645
+ };
646
+ }
647
+ const numeric = {};
648
+ for (const [field, entries] of Object.entries(raw.field_indexes?.numeric ?? {})) {
649
+ numeric[field] = entries.map((e) => ({ value: e.value, docId: e.doc_id }));
650
+ }
651
+ const boolean = {};
652
+ for (const [field, idx] of Object.entries(raw.field_indexes?.boolean ?? {})) {
653
+ boolean[field] = { trueDocs: idx.true_docs ?? [], falseDocs: idx.false_docs ?? [] };
654
+ }
655
+ const geopoint = {};
656
+ for (const [field, entries] of Object.entries(raw.field_indexes?.geopoint ?? {})) {
657
+ geopoint[field] = entries.map((e) => ({ lat: e.lat, lon: e.lon, docId: e.doc_id }));
658
+ }
659
+ const vectorData = {};
660
+ for (const [field, data2] of Object.entries(raw.vector_data ?? {})) {
661
+ vectorData[field] = {
662
+ dimension: data2.dimension,
663
+ vectors: (data2.vectors ?? []).map((v) => ({ docId: v.doc_id, vector: v.vector })),
664
+ hnswGraph: data2.hnsw_graph ? {
665
+ entryPoint: data2.hnsw_graph.entry_point,
666
+ maxLayer: data2.hnsw_graph.max_layer,
667
+ m: data2.hnsw_graph.m,
668
+ efConstruction: data2.hnsw_graph.ef_construction,
669
+ metric: validateHnswMetric2(data2.hnsw_graph.metric),
670
+ nodes: data2.hnsw_graph.nodes
671
+ } : null,
672
+ codes: data2.codes ? { bits: data2.codes.bits, centroid: data2.codes.centroid, records: data2.codes.records } : null
673
+ };
674
+ }
675
+ return {
676
+ indexName: raw.index_name,
677
+ partitionId: raw.partition_id,
678
+ totalPartitions: raw.total_partitions,
679
+ language: raw.language ?? "english",
680
+ schema: raw.schema ?? {},
681
+ docCount: raw.doc_count ?? 0,
682
+ avgDocLength: raw.avg_doc_length ?? 0,
683
+ documents,
684
+ invertedIndex,
685
+ fieldIndexes: {
686
+ numeric,
687
+ boolean,
688
+ enum: raw.field_indexes?.enum ?? {},
689
+ geopoint,
690
+ ...patternIndexesFromWire(raw.field_indexes?.pattern)
691
+ },
692
+ surfaceForms: sanitizeSurfaceForms(raw.surface_forms),
693
+ vectorData,
694
+ statistics: {
695
+ totalDocuments: raw.statistics?.total_documents ?? 0,
696
+ totalFieldLengths: raw.statistics?.total_field_lengths ?? {},
697
+ averageFieldLengths: raw.statistics?.average_field_lengths ?? {},
698
+ docFrequencies: raw.statistics?.doc_frequencies ?? {}
699
+ }
700
+ };
701
+ }
702
+
703
+ export { deserializePayloadV1, deserializePayloadV2, documentsOfPayloadV2, growPartitionsTo, loadSegmentedCheckpoint, readCurrentManifest, readMetadataEnvelope, readSegmentManifest, reclaimUnlistedKeys, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, writeMetadataEnvelope };