@delali/narsil 0.3.1-next.91 → 0.3.1-next.95

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/dist/adapters/filesystem.mjs +1 -1
  2. package/dist/adapters/indexeddb.mjs +1 -1
  3. package/dist/browser/{chunk-66KYAOIK.mjs → chunk-3U636ADD.mjs} +2 -2
  4. package/dist/browser/{chunk-NOUZ6RH3.mjs → chunk-D3CC2NMZ.mjs} +238 -114
  5. package/dist/browser/{chunk-3ZRMZIH4.mjs → chunk-FYFQJKFW.mjs} +208 -78
  6. package/dist/browser/{chunk-FHFFWLEK.mjs → chunk-HARBSUWP.mjs} +3 -4
  7. package/dist/browser/{chunk-B6XE7KXY.mjs → chunk-LULHQXCK.mjs} +4 -4
  8. package/dist/browser/{chunk-NQW2YXEX.mjs → chunk-WKPJ4E73.mjs} +1 -1
  9. package/dist/browser/{durability-wiring-F4OGQUSU.mjs → durability-wiring-3ZRKSPJD.mjs} +498 -71
  10. package/dist/browser/{module-E25UDADM.mjs → module-IEOQI5GI.mjs} +102 -54
  11. package/dist/browser/release-76MXRPGV.mjs +33 -0
  12. package/dist/browser/replace-RURXTGVG.mjs +57 -0
  13. package/dist/browser/restore-queue-RBULSOVX.mjs +35 -0
  14. package/dist/browser/{snapshot-5EJ7I7QY.mjs → snapshot-Y3NGUJHX.mjs} +362 -103
  15. package/dist/browser/{write-K5V4XQ72.mjs → write-ZUBDNLCH.mjs} +4 -4
  16. package/dist/{capabilities-CvExBThq.d.ts → capabilities-BvxvFzVA.d.ts} +2 -2
  17. package/dist/{chunk-NVL4ZR3K.mjs → chunk-2GZGBOQI.mjs} +2 -2
  18. package/dist/{chunk-XLJ2S2BH.mjs → chunk-3LYY62M6.mjs} +40 -21
  19. package/dist/{chunk-CVQWQVFX.mjs → chunk-4ZJWRETC.mjs} +2 -2
  20. package/dist/{chunk-T4DB5YFB.mjs → chunk-54HYC5FT.mjs} +6 -6
  21. package/dist/{chunk-ZIJ63TBT.mjs → chunk-7KC56574.mjs} +1 -1
  22. package/dist/{chunk-67Z2V6TC.mjs → chunk-7QONURHW.mjs} +3 -3
  23. package/dist/{chunk-BVOJIYRV.mjs → chunk-BB7MASRQ.mjs} +1 -1
  24. package/dist/{chunk-6VZ3Q73N.mjs → chunk-CHOGQSHK.mjs} +44 -25
  25. package/dist/{chunk-QZN3MIJ3.mjs → chunk-ES33L3HO.mjs} +2 -2
  26. package/dist/{chunk-YCJLVNWR.mjs → chunk-F52QGNZ2.mjs} +1 -1
  27. package/dist/{chunk-43ORORN6.mjs → chunk-GJZHU4FC.mjs} +3 -3
  28. package/dist/{chunk-R7OSDAFK.mjs → chunk-HBVLF3HA.mjs} +1 -1
  29. package/dist/{chunk-ULRPNYAG.mjs → chunk-ICL2AELD.mjs} +120 -62
  30. package/dist/{chunk-3RDS4UHC.mjs → chunk-J7SDAOTP.mjs} +4 -4
  31. package/dist/{chunk-A4G7JSYO.mjs → chunk-JFUE6F2Y.mjs} +98 -49
  32. package/dist/chunk-KOKM4PQN.mjs +52 -0
  33. package/dist/{chunk-7OQHSDK6.mjs → chunk-L26LEDHO.mjs} +7 -7
  34. package/dist/{chunk-EONIHG3V.mjs → chunk-MASE2LTJ.mjs} +7 -7
  35. package/dist/{chunk-MXS5HWIH.mjs → chunk-MWT5WCJ5.mjs} +89 -17
  36. package/dist/{chunk-3TTYK5A7.mjs → chunk-Q4UTU5ZE.mjs} +9 -9
  37. package/dist/{chunk-T6CNO72J.mjs → chunk-QM3VPX4Z.mjs} +2 -2
  38. package/dist/chunk-QY7GC2MW.mjs +33 -0
  39. package/dist/{chunk-BJ2YKRNX.mjs → chunk-R36MIGBU.mjs} +86 -54
  40. package/dist/{chunk-RVM7AODE.mjs → chunk-RDJKYKMQ.mjs} +2 -2
  41. package/dist/{chunk-UXQE2ROB.mjs → chunk-RUIXODYU.mjs} +3 -2
  42. package/dist/{chunk-5ER245FI.mjs → chunk-SBCEHIPL.mjs} +3 -3
  43. package/dist/{chunk-TPJU7NDJ.mjs → chunk-SYWGORQB.mjs} +1 -0
  44. package/dist/{chunk-UKSUENHK.mjs → chunk-T2QN2FA4.mjs} +40 -26
  45. package/dist/{chunk-CO4MRKST.mjs → chunk-WLOF6HSP.mjs} +2 -2
  46. package/dist/{chunk-MKWUXWQH.mjs → chunk-ZHS44YKF.mjs} +1 -1
  47. package/dist/{client-Cc42b42j.d.ts → client-DKwterMb.d.ts} +2 -2
  48. package/dist/client.d.ts +6 -6
  49. package/dist/client.mjs +1 -0
  50. package/dist/{config-DyRowvVP.d.ts → config-6YBtxekG.d.ts} +1 -1
  51. package/dist/{config-BYne-yCI.d.ts → config-DPphZfuM.d.ts} +1 -1
  52. package/dist/direct-executor-5KJGVQHM.mjs +22 -0
  53. package/dist/distribution/coordinator/etcd.mjs +1 -1
  54. package/dist/distribution/transport/grpc.mjs +1 -1
  55. package/dist/distribution.d.ts +5 -5
  56. package/dist/distribution.mjs +26 -26
  57. package/dist/{durability-wiring-6AN4QTT3.mjs → durability-wiring-FRPV6JLZ.mjs} +518 -91
  58. package/dist/embeddings/openai.mjs +1 -1
  59. package/dist/{engine-96_kLtSY.d.ts → engine-BGt9lI1V.d.ts} +11 -11
  60. package/dist/index.browser.mjs +2 -2
  61. package/dist/index.d.ts +5 -5
  62. package/dist/index.mjs +18 -18
  63. package/dist/{memory-dd3GESdc.d.ts → memory-COdoGthn.d.ts} +96 -76
  64. package/dist/{module-WIDYIJLY.mjs → module-IUTNXWMV.mjs} +72 -38
  65. package/dist/partial-words.mjs +5 -5
  66. package/dist/pattern.mjs +3 -3
  67. package/dist/persistence/durability/checkpoint-worker.mjs +21 -21
  68. package/dist/react.d.ts +6 -6
  69. package/dist/react.mjs +1 -0
  70. package/dist/release-BH3DKW3T.mjs +33 -0
  71. package/dist/replace-CEO5KJJ3.mjs +57 -0
  72. package/dist/restore-queue-I5SI6AUQ.mjs +35 -0
  73. package/dist/server.d.ts +8 -8
  74. package/dist/server.mjs +10 -10
  75. package/dist/snapshot-YC6TS4OP.mjs +738 -0
  76. package/dist/synonyms.mjs +3 -3
  77. package/dist/{tasks-DA1Ff0G4.d.ts → tasks-BJmamiC4.d.ts} +1 -1
  78. package/dist/{thread-VTDLLYXC.mjs → thread-U7XYJWGG.mjs} +15 -12
  79. package/dist/vector/search-worker.mjs +6 -6
  80. package/dist/workers/entry.mjs +5 -5
  81. package/dist/{write-OJR3H4BI.mjs → write-M2OALSA4.mjs} +20 -20
  82. package/package.json +7 -7
  83. package/dist/chunk-Y2F66IQJ.mjs +0 -74
  84. package/dist/chunk-ZJ4CSAB2.mjs +0 -13
  85. package/dist/direct-executor-CHXVCBN7.mjs +0 -22
  86. package/dist/snapshot-JFZVOGZV.mjs +0 -479
@@ -1,479 +0,0 @@
1
- import { unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-T4DB5YFB.mjs';
2
- import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive } from './chunk-7OQHSDK6.mjs';
3
- import './chunk-MXS5HWIH.mjs';
4
- import { indexMetadataOf, readVectorFile, readGraphParts } from './chunk-EONIHG3V.mjs';
5
- import { decodeVectorIndexParts, bytesToVectors } from './chunk-YCJLVNWR.mjs';
6
- import './chunk-ULRPNYAG.mjs';
7
- import './chunk-37CD3OKS.mjs';
8
- import './chunk-DJHHF2L5.mjs';
9
- import { manifestKey, encodeSegmentManifest, readSegmentFile } from './chunk-3TTYK5A7.mjs';
10
- import './chunk-3RDS4UHC.mjs';
11
- import './chunk-OHP7QR33.mjs';
12
- import './chunk-P36MUFEJ.mjs';
13
- import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, validateRequiredFieldsInSchema, validateEmbeddingConfig, decodeIndexConfiguration, prepareStoredSynonyms, insertDocumentVectors } from './chunk-A4G7JSYO.mjs';
14
- import { MAX_FILE_BYTES } from './chunk-RVM7AODE.mjs';
15
- import { decodeEveryStoredDocument } from './chunk-XLJ2S2BH.mjs';
16
- import { validatePatternValueLimit, requireSurfaceFormsForPartialFields } from './chunk-BJ2YKRNX.mjs';
17
- import { setNestedValue } from './chunk-6VZ3Q73N.mjs';
18
- import { restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-CO4MRKST.mjs';
19
- import './chunk-AV4IJKD2.mjs';
20
- import { getLanguage } from './chunk-MKWUXWQH.mjs';
21
- import { BATCH_CHUNK_SIZE } from './chunk-Q5DC6WAB.mjs';
22
- import './chunk-TGP5M5R7.mjs';
23
- import './chunk-U2BKYEC7.mjs';
24
- import './chunk-Y2F66IQJ.mjs';
25
- import './chunk-ZJ4CSAB2.mjs';
26
- import './chunk-QZN3MIJ3.mjs';
27
- import './chunk-T6CNO72J.mjs';
28
- import './chunk-WQBCBKY3.mjs';
29
- import { parseVectorDimension } from './chunk-5ER245FI.mjs';
30
- import { raiseEarlierReleaseGraphSetting, validateSchema, validateVectorPromotion, validateVectorStorage, flattenSchema } from './chunk-CVQWQVFX.mjs';
31
- import './chunk-FGMGUDHX.mjs';
32
- import './chunk-KEHK2O6P.mjs';
33
- import './chunk-GES7U2PM.mjs';
34
- import './chunk-BVOJIYRV.mjs';
35
- import { resolveIndexAnalysis } from './chunk-NVL4ZR3K.mjs';
36
- import './chunk-X5CV2E2B.mjs';
37
- import './chunk-7X5JST5L.mjs';
38
- import './chunk-ZIJ63TBT.mjs';
39
- import './chunk-R6A773WK.mjs';
40
- import './chunk-YTJCGKM4.mjs';
41
- import './chunk-R6PR4WYZ.mjs';
42
- import { loadVectorCodeFor } from './chunk-R7OSDAFK.mjs';
43
- import './chunk-L5ORED7V.mjs';
44
- import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
45
- import { decode } from '@msgpack/msgpack';
46
-
47
- // src/persistence/archive/validate.ts
48
- function requireDimension(key, found, expected) {
49
- if (expected === found) return;
50
- throw new NarsilError(
51
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
52
- `The vector file "${key}" holds vectors of ${found} dimensions where the schema gives ${String(expected)}`,
53
- { key, found, expected }
54
- );
55
- }
56
- function requireSchemaOfManifest(key, schema, manifest) {
57
- for (const [field, type] of Object.entries(schema)) {
58
- const listed = manifest.schema[field];
59
- if (listed === void 0 || listed === type) continue;
60
- throw new NarsilError(
61
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
62
- `The segment "${key}" gives field "${field}" the type "${type}", while the manifest gives it "${listed}"`,
63
- { key, field }
64
- );
65
- }
66
- }
67
- async function validateStagedCheckpoint(store, manifest) {
68
- for (const partition of manifest.partitions) {
69
- for (const ref of partition.segments) {
70
- const contents = await readSegmentFile(store, ref);
71
- requireSchemaOfManifest(ref.key, contents.schema, manifest);
72
- decodeEveryStoredDocument(contents.arrays.documentTable);
73
- }
74
- }
75
- for (const field of manifest.vectors) {
76
- const fieldType = manifest.schema[field.fieldPath];
77
- const dimension = fieldType === void 0 ? null : parseVectorDimension(fieldType);
78
- for (const file of field.files) {
79
- const { payload } = await readVectorFile(store, file, field.fieldPath);
80
- requireDimension(file.key, payload.dimension, dimension);
81
- }
82
- await readGraphParts(store, field);
83
- }
84
- }
85
-
86
- // src/engine/snapshot/restore-setup.ts
87
- function prepareRestoredIndex(configuration, deps, reject) {
88
- const configFields = restoredConfigFields(configuration, reject);
89
- const embedding = restoredEmbedding(configuration.embedding, reject);
90
- const languageName = configuration.language ?? "";
91
- const language = getLanguage(languageName);
92
- const schema = validateSchema(configuration.schema);
93
- validatePatternValueLimit(configFields.patternValueLimit);
94
- requireSurfaceFormsForPartialFields(schema, configFields.surfaceForms);
95
- validateVectorPromotion(configFields.vectorPromotion);
96
- validateVectorStorage(configFields.vectorPromotion, deps.filesystemDurability);
97
- if (configFields.required !== void 0 && configFields.required.length > 0) {
98
- validateRequiredFieldsInSchema(configFields.required, schema);
99
- }
100
- const config = {
101
- schema,
102
- language: languageName,
103
- ...configFields,
104
- ...embedding !== void 0 ? {
105
- embedding: {
106
- fields: embedding.fields,
107
- ...embedding.adapter !== void 0 ? { adapter: embedding.adapter } : {}
108
- }
109
- } : {}
110
- };
111
- resolveIndexAnalysis(config);
112
- const adapterName = embedding?.adapter ?? null;
113
- let embeddingAdapter = null;
114
- if (embedding !== void 0) {
115
- const candidate = adapterName !== null ? deps.embeddingAdapters.get(adapterName) ?? null : deps.defaultEmbeddingAdapter;
116
- if (candidate) {
117
- validateEmbeddingConfig({ fields: embedding.fields, adapter: candidate }, schema, void 0);
118
- embeddingAdapter = candidate;
119
- }
120
- }
121
- const storedRevision = typeof configuration.analysisRevision === "string" ? configuration.analysisRevision : null;
122
- const synonyms = restoredSynonyms(configuration.synonyms, config, language, reject ?? rejectSnapshot);
123
- return { config, language, embeddingAdapter, adapterName, storedRevision, synonyms };
124
- }
125
- function restoredSynonyms(raw, config, language, reject) {
126
- if (raw === void 0 || raw === null) return null;
127
- return prepareStoredSynonyms(raw, config, language, (message) => reject(`synonyms: ${message}`));
128
- }
129
- async function createRestoredIndex(indexName, prepared, partitionCount, deps) {
130
- const loadingVectorCode = loadVectorCodeFor(prepared.config.schema);
131
- if (loadingVectorCode !== null) await loadingVectorCode;
132
- if (deps.indexRegistry.has(indexName)) {
133
- await deps.dropIndex(indexName);
134
- }
135
- deps.clearAnalysisStale(indexName);
136
- if (prepared.storedRevision !== prepared.language.revision) {
137
- deps.markAnalysisStale({
138
- indexName,
139
- language: prepared.language.name,
140
- storedRevision: prepared.storedRevision,
141
- currentRevision: prepared.language.revision
142
- });
143
- }
144
- deps.executor.createIndex(indexName, prepared.config, prepared.language, false, prepared.synonyms?.lookup ?? null);
145
- deps.indexRegistry.set(indexName, {
146
- config: prepared.config,
147
- language: prepared.language,
148
- embeddingAdapter: prepared.embeddingAdapter,
149
- embeddingAdapterName: prepared.adapterName,
150
- vectorFieldPaths: deps.getVectorFieldPaths(prepared.config.schema),
151
- indexUuid: null,
152
- heldPartitions: null,
153
- documentCount: 0,
154
- partitionCount,
155
- analysisRevision: prepared.storedRevision,
156
- synonyms: prepared.synonyms?.encoded ?? null
157
- });
158
- await deps.indexState.registerOpen(indexName);
159
- const release = await deps.indexState.acquire(indexName, false);
160
- const manager = deps.requireManager(indexName);
161
- while (manager.partitionCount < partitionCount) manager.addPartition();
162
- return { manager, release };
163
- }
164
- async function abandonRestoredIndex(indexName, deps) {
165
- try {
166
- deps.executor.dropIndex(indexName);
167
- } catch (_) {
168
- }
169
- deps.indexRegistry.delete(indexName);
170
- deps.indexState.forget(indexName);
171
- if (deps.durability) {
172
- await deps.durability.manager.removeIndex(indexName).catch(() => void 0);
173
- }
174
- }
175
- function recordRestoredCounts(indexName, manager, deps) {
176
- const entry = deps.indexRegistry.get(indexName);
177
- if (entry !== void 0) {
178
- entry.documentCount = manager.countDocuments();
179
- entry.partitionCount = manager.partitionCount;
180
- }
181
- }
182
-
183
- // src/engine/snapshot/archive-restore.ts
184
- function rejectConfiguration(message) {
185
- throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Invalid index configuration: ${message}`);
186
- }
187
- function renamed(manifest, sourceName, indexName) {
188
- const rename = (key) => `${indexName}/${key.slice(sourceName.length + 1)}`;
189
- return {
190
- ...manifest,
191
- partitions: manifest.partitions.map((partition) => ({
192
- ...partition,
193
- segments: partition.segments.map((segment) => ({ ...segment, key: rename(segment.key) }))
194
- })),
195
- vectors: manifest.vectors.map((field) => ({
196
- ...field,
197
- files: field.files.map((file) => ({ ...file, key: rename(file.key) })),
198
- graphParts: field.graphParts.map((part) => ({ ...part, key: rename(part.key) }))
199
- }))
200
- };
201
- }
202
- function requireSameSchema(prepared, manifest) {
203
- const configured = flattenSchema(prepared.config.schema);
204
- const keys = /* @__PURE__ */ new Set([...Object.keys(configured), ...Object.keys(manifest.schema)]);
205
- for (const field of keys) {
206
- if (configured[field] === manifest.schema[field]) continue;
207
- throw new NarsilError(
208
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
209
- `The archive's configuration gives field "${field}" another type than its manifest does`,
210
- { field }
211
- );
212
- }
213
- }
214
- function partitionCountOf(manifest) {
215
- let count = 1;
216
- for (const partition of manifest.partitions) count = Math.max(count, partition.partitionId + 1);
217
- return count;
218
- }
219
- function missingPart(what) {
220
- throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The archive holds no ${what}`);
221
- }
222
- async function stageArchive(indexName, reader, staging, deps) {
223
- const staged = { prepared: null, manifest: null };
224
- await readSnapshotArchive(reader, {
225
- config: async (bytes) => {
226
- staged.prepared = prepareRestoredIndex(await decodeIndexConfiguration(bytes), deps, rejectConfiguration);
227
- },
228
- manifest: async (manifest2, sourceName) => {
229
- if (staged.prepared === null) missingPart("configuration");
230
- requireSameSchema(staged.prepared, manifest2);
231
- staged.manifest = renamed(manifest2, sourceName, indexName);
232
- },
233
- file: async (entryName, _ref, bytes) => {
234
- if (staged.manifest === null) missingPart("manifest");
235
- await staging.write(`${indexName}/${entryName}`, [bytes]);
236
- }
237
- });
238
- const { prepared, manifest } = staged;
239
- if (prepared === null) missingPart("configuration");
240
- if (manifest === null) missingPart("manifest");
241
- await validateStagedCheckpoint(staging, manifest);
242
- return { prepared, manifest };
243
- }
244
- async function replaceWithStaged(indexName, staged, staging, memory, deps) {
245
- const { prepared, manifest } = staged;
246
- const durable = deps.durability?.manager;
247
- const created = await createRestoredIndex(indexName, prepared, partitionCountOf(manifest), deps);
248
- try {
249
- if (staging !== null && durable !== void 0) await durable.persistMetadata(indexName);
250
- const store = staging === null ? memory : await staging.commit();
251
- await store.write(manifestKey(indexName), (await encodeSegmentManifest(manifest)).chunks);
252
- await loadSegmentedCheckpoint(
253
- store,
254
- indexName,
255
- manifest,
256
- { manager: created.manager, vectorIndexes: created.manager.getVectorIndexes() },
257
- true
258
- );
259
- recordRestoredCounts(indexName, created.manager, deps);
260
- if (durable?.adoptRestoredCheckpoint !== void 0 && staging !== null) {
261
- durable.adoptRestoredCheckpoint(indexName, manifest.checkpoint);
262
- await durable.persistMetadata(indexName);
263
- } else if (durable !== void 0) {
264
- await durable.persistMetadata(indexName);
265
- await durable.checkpoint(indexName);
266
- }
267
- await deps.announceIndexCreated(indexName, prepared.config);
268
- return created;
269
- } catch (error) {
270
- created.release();
271
- await abandonRestoredIndex(indexName, deps);
272
- throw error;
273
- }
274
- }
275
- async function restoreArchive(indexName, reader, deps) {
276
- const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
277
- const memory = memoryCheckpointStore();
278
- try {
279
- const staged = await stageArchive(indexName, reader, staging?.store ?? memory, deps);
280
- const created = await replaceWithStaged(indexName, staged, staging, memory, deps);
281
- created.release();
282
- } finally {
283
- await staging?.discard().catch(() => void 0);
284
- await reader.close();
285
- }
286
- }
287
- function invalidSnapshot(message, details) {
288
- throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, message, details);
289
- }
290
- async function decodeEarlierSnapshot(data) {
291
- const payloadBytes = await unpackIndexSnapshotEnvelope(data);
292
- let decoded;
293
- try {
294
- decoded = decode(payloadBytes);
295
- } catch (err) {
296
- invalidSnapshot(`Snapshot data is not a Narsil snapshot: ${err instanceof Error ? err.message : String(err)}`, {
297
- bytes: data.length
298
- });
299
- }
300
- if (typeof decoded !== "object" || decoded === null || Array.isArray(decoded)) {
301
- invalidSnapshot("Snapshot data does not hold a snapshot envelope", { bytes: data.length });
302
- }
303
- const envelope = decoded;
304
- if (envelope.version !== 1 && envelope.version !== 2) {
305
- invalidSnapshot(`Unsupported snapshot version: ${envelope.version}. Expected version 1 or 2`, {
306
- version: envelope.version
307
- });
308
- }
309
- if (!envelope.schema || typeof envelope.schema !== "object")
310
- invalidSnapshot("Invalid snapshot: missing or invalid schema");
311
- if (!envelope.language || typeof envelope.language !== "string") {
312
- invalidSnapshot("Invalid snapshot: missing or invalid language");
313
- }
314
- if (!Array.isArray(envelope.partitions)) invalidSnapshot("Invalid snapshot: missing partitions");
315
- return { ...envelope, vectorPromotion: raiseEarlierReleaseGraphSetting(envelope.vectorPromotion) };
316
- }
317
- function vectorsOfSnapshot(envelope) {
318
- const byDocument = /* @__PURE__ */ new Map();
319
- for (const [fieldPath, rawParts] of Object.entries(envelope.vectorIndexes ?? {})) {
320
- for (const part of decodeVectorIndexParts(rawParts)) {
321
- const values = bytesToVectors(part.vectors);
322
- for (let i = 0; i < part.docIds.length; i++) {
323
- let fields = byDocument.get(part.docIds[i]);
324
- if (fields === void 0) {
325
- fields = /* @__PURE__ */ new Map();
326
- byDocument.set(part.docIds[i], fields);
327
- }
328
- fields.set(fieldPath, values.slice(i * part.dimension, (i + 1) * part.dimension));
329
- }
330
- }
331
- }
332
- return byDocument;
333
- }
334
- function documentsOfPartition(envelope, payload) {
335
- return envelope.version === 2 ? documentsOfPayloadV2(payload).documents : deserializePayloadV1(payload).documents;
336
- }
337
- function withVectors(fields, vectors) {
338
- if (vectors === void 0 || vectors.size === 0) return fields;
339
- const nested = [...vectors.keys()].some((fieldPath) => fieldPath.includes("."));
340
- const document = nested ? structuredClone(fields) : { ...fields };
341
- for (const [fieldPath, vector] of vectors) setNestedValue(document, fieldPath, vector);
342
- return document;
343
- }
344
- function insertRestoredDocument(manager, docId, fields, vectors) {
345
- manager.insert(docId, fields, { skipClone: true });
346
- if (vectors === void 0) return;
347
- try {
348
- insertDocumentVectors(docId, vectors, manager.getVectorIndexes(), manager.partitionIdOf(docId));
349
- } catch (err) {
350
- manager.remove(docId);
351
- throw err;
352
- }
353
- }
354
- async function insertRestoredChunk(indexName, manager, chunk, vectors, deps) {
355
- if (deps.durability === null) {
356
- for (const [docId, fields] of chunk) insertRestoredDocument(manager, docId, fields, vectors.get(docId));
357
- return;
358
- }
359
- const mutations = chunk.map(([docId, fields]) => ({
360
- docId,
361
- document: withVectors(fields, vectors.get(docId)),
362
- apply: () => insertRestoredDocument(manager, docId, fields, vectors.get(docId))
363
- }));
364
- for (const outcome of await deps.durability.recordMutationBatch(indexName, mutations)) {
365
- if (!outcome.ok) throw outcome.error;
366
- }
367
- }
368
- async function insertSnapshotDocuments(indexName, envelope, manager, deps) {
369
- const vectors = vectorsOfSnapshot(envelope);
370
- for (const payload of envelope.partitions ?? []) {
371
- const documents = Object.entries(documentsOfPartition(envelope, payload)).map(
372
- ([docId, stored]) => [docId, stored.fields]
373
- );
374
- for (let start = 0; start < documents.length; start += BATCH_CHUNK_SIZE) {
375
- await insertRestoredChunk(indexName, manager, documents.slice(start, start + BATCH_CHUNK_SIZE), vectors, deps);
376
- }
377
- }
378
- for (const vectorIndex of manager.getVectorIndexes().values()) vectorIndex.scheduleBuild();
379
- }
380
- async function restoreEarlierSnapshot(indexName, data, deps) {
381
- const envelope = await decodeEarlierSnapshot(data);
382
- const prepared = prepareRestoredIndex(envelope, deps);
383
- const analysedNow = { ...prepared, storedRevision: prepared.language.revision };
384
- const partitionCount = envelope.partitions?.length ?? 0;
385
- const { manager, release } = await createRestoredIndex(indexName, analysedNow, partitionCount, deps);
386
- try {
387
- await insertSnapshotDocuments(indexName, envelope, manager, deps);
388
- recordRestoredCounts(indexName, manager, deps);
389
- if (deps.durability) {
390
- await deps.durability.manager.persistMetadata(indexName);
391
- await deps.durability.manager.checkpoint(indexName);
392
- }
393
- await deps.announceIndexCreated(indexName, prepared.config);
394
- } catch (err) {
395
- await abandonRestoredIndex(indexName, deps);
396
- throw err;
397
- } finally {
398
- release();
399
- }
400
- }
401
-
402
- // src/engine/snapshot/index.ts
403
- var NRSL_MAGIC_BYTES = 4;
404
- async function checkpointForArchive(deps) {
405
- const durable = deps.durability?.manager;
406
- if (durable?.openArchiveCheckpoint !== void 0) return durable.openArchiveCheckpoint(deps.indexName);
407
- return buildArchiveFromMemory({
408
- indexName: deps.indexName,
409
- metadata: indexMetadataOf(deps.indexName, deps.entry, deps.manager.partitionCount),
410
- manager: deps.manager,
411
- vectorIndexes: deps.manager.getVectorIndexes(),
412
- appliedSeqNo: (partitionId) => durable?.highestPersistedSeqNo(deps.indexName, partitionId) ?? 0,
413
- partitionId: null
414
- });
415
- }
416
- async function createSnapshot(deps) {
417
- const config = await encodeIndexConfiguration(deps.entry);
418
- const checkpoint = await checkpointForArchive(deps);
419
- return archiveStream({ ...checkpoint, config, sourceIndexName: deps.indexName });
420
- }
421
- function requireSnapshotSource(data) {
422
- if (data instanceof Uint8Array) return data;
423
- if (data !== null && typeof data === "object") {
424
- if (typeof data.getReader === "function") return data;
425
- if (typeof data[Symbol.asyncIterator] === "function") {
426
- return data;
427
- }
428
- }
429
- throw new NarsilError(
430
- ErrorCodes.DOC_VALIDATION_FAILED,
431
- "Snapshot data must be a Uint8Array, a ReadableStream of bytes, or an async iterable of bytes"
432
- );
433
- }
434
- async function readEarlierSnapshotFile(reader) {
435
- const length = HEADER_SIZE + readHeader(await reader.peek(HEADER_SIZE)).payloadLength;
436
- if (length > MAX_FILE_BYTES) {
437
- throw new NarsilError(
438
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
439
- `The snapshot declares ${length} bytes, above the file size limit of ${MAX_FILE_BYTES}`,
440
- { bytes: length }
441
- );
442
- }
443
- const bytes = await reader.readExactly(length) ?? new Uint8Array(0);
444
- if (bytes.byteLength < length) {
445
- throw new NarsilError(
446
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
447
- "The snapshot ends before the payload its header declares"
448
- );
449
- }
450
- const trailing = await reader.readExactly(1);
451
- if (trailing !== null && trailing.byteLength > 0) {
452
- throw new NarsilError(
453
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
454
- "The snapshot holds bytes after the payload its header declares"
455
- );
456
- }
457
- return bytes;
458
- }
459
- async function restoreFromSnapshot(indexName, data, deps) {
460
- const reader = createByteReader(requireSnapshotSource(data));
461
- const head = await reader.peek(NRSL_MAGIC_BYTES);
462
- if (head.byteLength >= NRSL_MAGIC_BYTES && hasNrslMagic(head)) {
463
- let bytes;
464
- try {
465
- bytes = await readEarlierSnapshotFile(reader);
466
- } finally {
467
- await reader.close();
468
- }
469
- await restoreEarlierSnapshot(indexName, bytes, deps);
470
- return;
471
- }
472
- if (head.byteLength === 0) {
473
- await reader.close();
474
- throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Snapshot data is empty");
475
- }
476
- await restoreArchive(indexName, reader, deps);
477
- }
478
-
479
- export { checkpointForArchive, createSnapshot, restoreFromSnapshot };