@delali/narsil 0.3.1-next.90 → 0.3.1-next.94

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/dist/adapters/filesystem.mjs +1 -1
  2. package/dist/adapters/indexeddb.mjs +1 -1
  3. package/dist/browser/{chunk-3GZQKGVQ.mjs → chunk-4JTBL63P.mjs} +13 -5
  4. package/dist/browser/{chunk-3ZRMZIH4.mjs → chunk-5S74HQRT.mjs} +208 -78
  5. package/dist/browser/{chunk-FHFFWLEK.mjs → chunk-HARBSUWP.mjs} +3 -4
  6. package/dist/browser/{chunk-RAHUQOJL.mjs → chunk-HC6DMR4T.mjs} +2445 -2107
  7. package/dist/browser/{chunk-JACCR3BJ.mjs → chunk-PP6N5CCS.mjs} +2 -2
  8. package/dist/browser/{chunk-NQW2YXEX.mjs → chunk-WKPJ4E73.mjs} +1 -1
  9. package/dist/browser/{durability-wiring-VSUBLUDK.mjs → durability-wiring-7DKYHKT5.mjs} +498 -71
  10. package/dist/browser/{module-E25UDADM.mjs → module-O5WH32EL.mjs} +102 -54
  11. package/dist/browser/release-76MXRPGV.mjs +33 -0
  12. package/dist/browser/replace-RURXTGVG.mjs +57 -0
  13. package/dist/browser/restore-queue-Z7FZZNKH.mjs +35 -0
  14. package/dist/browser/{snapshot-5MXBEDXF.mjs → snapshot-PPTZ565G.mjs} +368 -103
  15. package/dist/browser/{write-6SBYTS62.mjs → write-2ZAA7UDH.mjs} +4 -4
  16. package/dist/{capabilities-SJPttcgQ.d.ts → capabilities-xsJfdFoQ.d.ts} +11 -3
  17. package/dist/chunk-2GZGBOQI.mjs +426 -0
  18. package/dist/{chunk-CVQWQVFX.mjs → chunk-4ZJWRETC.mjs} +2 -2
  19. package/dist/chunk-7KC56574.mjs +507 -0
  20. package/dist/{chunk-MXS5HWIH.mjs → chunk-7VALMHZF.mjs} +89 -17
  21. package/dist/chunk-7X5JST5L.mjs +114 -0
  22. package/dist/{chunk-UKSUENHK.mjs → chunk-AF2CIKYC.mjs} +40 -26
  23. package/dist/{chunk-BVOJIYRV.mjs → chunk-BB7MASRQ.mjs} +1 -1
  24. package/dist/{chunk-R6BMMQXH.mjs → chunk-C45FUYPI.mjs} +193 -79
  25. package/dist/{chunk-EMXAYPTQ.mjs → chunk-CHOGQSHK.mjs} +2326 -3503
  26. package/dist/{chunk-QZN3MIJ3.mjs → chunk-ES33L3HO.mjs} +2 -2
  27. package/dist/{chunk-VY4DXZ5P.mjs → chunk-EWBXO45L.mjs} +3 -3
  28. package/dist/{chunk-QAMUC2BJ.mjs → chunk-EX62BQT6.mjs} +10 -9
  29. package/dist/{chunk-MWJIAREY.mjs → chunk-EZD4MWOU.mjs} +4 -4
  30. package/dist/{chunk-YCJLVNWR.mjs → chunk-F52QGNZ2.mjs} +1 -1
  31. package/dist/{chunk-URGLSNKR.mjs → chunk-GJZHU4FC.mjs} +202 -16
  32. package/dist/{chunk-S4LQDSUL.mjs → chunk-J53IHLG7.mjs} +7 -7
  33. package/dist/{chunk-R7OSDAFK.mjs → chunk-K6A4FCVR.mjs} +1 -1
  34. package/dist/chunk-KOKM4PQN.mjs +52 -0
  35. package/dist/{chunk-GCM3JFVA.mjs → chunk-OHP7QR33.mjs} +2 -1
  36. package/dist/{chunk-T6CNO72J.mjs → chunk-QM3VPX4Z.mjs} +2 -2
  37. package/dist/{chunk-V7YUSKKZ.mjs → chunk-QWWCLRXX.mjs} +10 -6
  38. package/dist/chunk-QY7GC2MW.mjs +33 -0
  39. package/dist/{chunk-DCK4P5BI.mjs → chunk-R36MIGBU.mjs} +100 -59
  40. package/dist/{chunk-YRX6XMOQ.mjs → chunk-R6PR4WYZ.mjs} +1 -0
  41. package/dist/{chunk-RVM7AODE.mjs → chunk-RDJKYKMQ.mjs} +2 -2
  42. package/dist/{chunk-OOSIDHA3.mjs → chunk-RSRVGAIQ.mjs} +1 -1
  43. package/dist/{chunk-XSZVS4S3.mjs → chunk-RUIXODYU.mjs} +9 -4
  44. package/dist/{chunk-A3UB2MXL.mjs → chunk-SBCEHIPL.mjs} +4 -507
  45. package/dist/{chunk-TPJU7NDJ.mjs → chunk-SYWGORQB.mjs} +1 -0
  46. package/dist/chunk-UAUZGXJP.mjs +3591 -0
  47. package/dist/{chunk-CL4OM7GB.mjs → chunk-WBS3EYC4.mjs} +12 -8
  48. package/dist/{chunk-J2BPXTD7.mjs → chunk-WLOF6HSP.mjs} +3 -3
  49. package/dist/chunk-X5CV2E2B.mjs +1 -0
  50. package/dist/{chunk-ULRPNYAG.mjs → chunk-Z3MOCHG5.mjs} +120 -62
  51. package/dist/chunk-ZHS44YKF.mjs +35 -0
  52. package/dist/{client-BMK53IR2.d.ts → client-CgayPw2R.d.ts} +27 -3
  53. package/dist/client.d.ts +9 -8
  54. package/dist/client.mjs +16 -2
  55. package/dist/{config-UJx1dyX1.d.ts → config-3kBsQ0Pi.d.ts} +2 -2
  56. package/dist/{config-BUR-7eVQ.d.ts → config-B-RnO6dr.d.ts} +2 -2
  57. package/dist/direct-executor-IJTRU3YN.mjs +22 -0
  58. package/dist/distribution/coordinator/etcd.d.ts +3 -2
  59. package/dist/distribution/coordinator/etcd.mjs +5 -1
  60. package/dist/distribution/coordinator/in-memory.d.ts +3 -2
  61. package/dist/distribution/coordinator/in-memory.mjs +4 -0
  62. package/dist/distribution/transport/grpc.mjs +3 -3
  63. package/dist/distribution/transport/in-memory.mjs +1 -1
  64. package/dist/distribution/transport/tcp.mjs +2 -2
  65. package/dist/distribution.d.ts +43 -9
  66. package/dist/distribution.mjs +4620 -4293
  67. package/dist/{durability-wiring-ZLRQU6FG.mjs → durability-wiring-Y55SCPFC.mjs} +524 -93
  68. package/dist/embeddings/openai.mjs +1 -1
  69. package/dist/{engine-9X6W5naj.d.ts → engine-DSUfY2F0.d.ts} +53 -17
  70. package/dist/{index-vCMu8OeQ.d.ts → index-ONAEaqtx.d.ts} +8 -1
  71. package/dist/index.browser.mjs +2 -2
  72. package/dist/index.d.ts +44 -25
  73. package/dist/index.mjs +22 -18
  74. package/dist/{memory-CV9NAxLY.d.ts → memory-wuXcYOD1.d.ts} +99 -79
  75. package/dist/{module-WIDYIJLY.mjs → module-VITS4ZR5.mjs} +72 -38
  76. package/dist/partial-words.mjs +5 -4
  77. package/dist/pattern.mjs +3 -3
  78. package/dist/persistence/durability/checkpoint-worker.d.ts +60 -58
  79. package/dist/persistence/durability/checkpoint-worker.mjs +27 -23
  80. package/dist/react.d.ts +8 -7
  81. package/dist/react.mjs +1 -0
  82. package/dist/release-BH3DKW3T.mjs +33 -0
  83. package/dist/replace-CEO5KJJ3.mjs +57 -0
  84. package/dist/restore-queue-4SG45BLF.mjs +35 -0
  85. package/dist/{schema-DtsgmNfU.d.ts → schema-DttzM-ln.d.ts} +12 -0
  86. package/dist/server.d.ts +11 -10
  87. package/dist/server.mjs +38 -11
  88. package/dist/snapshot-XBKE7PUG.mjs +738 -0
  89. package/dist/synonyms-CpOAHPa-.d.ts +52 -0
  90. package/dist/synonyms.d.ts +22 -0
  91. package/dist/synonyms.mjs +12 -0
  92. package/dist/{tasks-BGaZJyoY.d.ts → tasks-Bj9WBlFG.d.ts} +4 -1
  93. package/dist/{thread-ZP3UMIZT.mjs → thread-U7XYJWGG.mjs} +17 -12
  94. package/dist/vector/search-worker.d.ts +2 -1
  95. package/dist/vector/search-worker.mjs +6 -6
  96. package/dist/workers/entry.mjs +6 -6
  97. package/dist/{write-PXDKPG2R.mjs → write-BMNV7P2H.mjs} +24 -20
  98. package/package.json +15 -7
  99. package/dist/chunk-PR4O7CAP.mjs +0 -18
  100. package/dist/chunk-X4SEXL4U.mjs +0 -2310
  101. package/dist/chunk-Y2F66IQJ.mjs +0 -74
  102. package/dist/chunk-ZJ4CSAB2.mjs +0 -13
  103. package/dist/direct-executor-EGESD26S.mjs +0 -18
  104. package/dist/snapshot-22CUEYYW.mjs +0 -469
@@ -1,74 +0,0 @@
1
- // src/vector/constants.ts
2
- var WASM_PAGE_BYTES = 65536;
3
- var MAX_WASM_PAGES = 65536;
4
- var VECTOR_STORE_INITIAL_CAPACITY = 16;
5
- var VECTOR_BLOCK_INITIAL_SLOTS = 256;
6
- var DISK_STORAGE_BLOCK_BYTES = 67108864;
7
- var MAX_VECTOR_ORDINALS = 16777216;
8
- var UNPLACED_FLAG_CHUNK_ORDINALS = 1024;
9
- var VECTOR_SCRATCH_SLOTS = 64;
10
- var SCRATCH_SLOTS_PER_THREAD_POOL = 31;
11
- var MAX_DOC_ID_TABLE_BYTES = 1073741824;
12
- var MAX_WORKER_COPIES = 64;
13
- var VECTOR_SEARCH_TIMEOUT_MS = 3e4;
14
- var VECTOR_SEARCH_LOAD_TIMEOUT_MS = 3e5;
15
- var VECTOR_INSERT_TIMEOUT_MS = 6e5;
16
- var NATIVE_ATTACH_RETRY_MS = 1e3;
17
- var LOCK_SPIN_ITERATIONS = 128;
18
- var LOCK_WAIT_SLICE_MS = 20;
19
- var GRAPH_LOCK_WAIT_SLICE_MS = 1;
20
- var GRAPH_BUILD_DISPATCH_CHUNK = 64;
21
- var GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER = 2;
22
-
23
- // src/runtime/simd-support.ts
24
- var SIMD_PROBE_MODULE = new Uint8Array([
25
- 0,
26
- 97,
27
- 115,
28
- 109,
29
- 1,
30
- 0,
31
- 0,
32
- 0,
33
- 1,
34
- 5,
35
- 1,
36
- 96,
37
- 0,
38
- 1,
39
- 123,
40
- 3,
41
- 2,
42
- 1,
43
- 0,
44
- 10,
45
- 10,
46
- 1,
47
- 8,
48
- 0,
49
- 65,
50
- 0,
51
- 253,
52
- 15,
53
- 253,
54
- 98,
55
- 11
56
- ]);
57
- function runtimeStartsSimdModule() {
58
- try {
59
- if (typeof WebAssembly === "undefined") return false;
60
- new WebAssembly.Instance(new WebAssembly.Module(SIMD_PROBE_MODULE));
61
- return true;
62
- } catch {
63
- return false;
64
- }
65
- }
66
- var simdAvailable = runtimeStartsSimdModule();
67
- function isSimdAvailable() {
68
- return simdAvailable;
69
- }
70
- function markSimdUnavailable() {
71
- simdAvailable = false;
72
- }
73
-
74
- export { DISK_STORAGE_BLOCK_BYTES, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, GRAPH_LOCK_WAIT_SLICE_MS, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, MAX_DOC_ID_TABLE_BYTES, MAX_VECTOR_ORDINALS, MAX_WASM_PAGES, MAX_WORKER_COPIES, NATIVE_ATTACH_RETRY_MS, SCRATCH_SLOTS_PER_THREAD_POOL, UNPLACED_FLAG_CHUNK_ORDINALS, VECTOR_BLOCK_INITIAL_SLOTS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_STORE_INITIAL_CAPACITY, WASM_PAGE_BYTES, isSimdAvailable, markSimdUnavailable };
@@ -1,13 +0,0 @@
1
- // src/vector/osq/constants.ts
2
- var OSQ_GRID = { 1: 0.798, 2: 1.493, 4: 2.514, 8: 3.922 };
3
- var OSQ_STEPS = { 1: 1, 2: 3, 4: 15, 8: 255 };
4
- var OSQ_QUERY_BITS = { 1: 4, 2: 4, 4: 4, 8: 8 };
5
- var OSQ_LAMBDA = 0.1;
6
- var OSQ_REFINE_ROUNDS = 5;
7
- var OSQ_REFINE_TOLERANCE = 1e-8;
8
- var DEFAULT_OVERSAMPLE_BY_BITS = { 1: 3, 2: 3, 4: 2, 8: 2 };
9
- var DEFAULT_OSQ4_NARROW_OVERSAMPLE = 5;
10
- var OSQ4_NARROW_DIMENSION_LIMIT = 1024;
11
- var MIN_OVERSAMPLE = 1;
12
-
13
- export { DEFAULT_OSQ4_NARROW_OVERSAMPLE, DEFAULT_OVERSAMPLE_BY_BITS, MIN_OVERSAMPLE, OSQ4_NARROW_DIMENSION_LIMIT, OSQ_GRID, OSQ_LAMBDA, OSQ_QUERY_BITS, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, OSQ_STEPS };
@@ -1,18 +0,0 @@
1
- export { createDirectExecutor } from './chunk-EMXAYPTQ.mjs';
2
- import './chunk-X4SEXL4U.mjs';
3
- import './chunk-PR4O7CAP.mjs';
4
- import './chunk-TGP5M5R7.mjs';
5
- import './chunk-T6CNO72J.mjs';
6
- import './chunk-WQBCBKY3.mjs';
7
- import './chunk-A3UB2MXL.mjs';
8
- import './chunk-CVQWQVFX.mjs';
9
- import './chunk-FGMGUDHX.mjs';
10
- import './chunk-KEHK2O6P.mjs';
11
- import './chunk-GES7U2PM.mjs';
12
- import './chunk-BVOJIYRV.mjs';
13
- import './chunk-U2BKYEC7.mjs';
14
- import './chunk-R6A773WK.mjs';
15
- import './chunk-YTJCGKM4.mjs';
16
- import './chunk-R7OSDAFK.mjs';
17
- import './chunk-L5ORED7V.mjs';
18
- import './chunk-TPJU7NDJ.mjs';
@@ -1,469 +0,0 @@
1
- import { unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-V7YUSKKZ.mjs';
2
- import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive } from './chunk-S4LQDSUL.mjs';
3
- import { indexMetadataOf, readVectorFile, readGraphParts } from './chunk-CL4OM7GB.mjs';
4
- import './chunk-MXS5HWIH.mjs';
5
- import { decodeVectorIndexParts, bytesToVectors } from './chunk-YCJLVNWR.mjs';
6
- import './chunk-ULRPNYAG.mjs';
7
- import './chunk-37CD3OKS.mjs';
8
- import './chunk-DJHHF2L5.mjs';
9
- import { manifestKey, encodeSegmentManifest, readSegmentFile } from './chunk-QAMUC2BJ.mjs';
10
- import './chunk-MWJIAREY.mjs';
11
- import './chunk-GCM3JFVA.mjs';
12
- import './chunk-P36MUFEJ.mjs';
13
- import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, validateRequiredFieldsInSchema, validateEmbeddingConfig, decodeIndexConfiguration, insertDocumentVectors } from './chunk-R6BMMQXH.mjs';
14
- import { resolveIndexAnalysis, decodeEveryStoredDocument } from './chunk-EMXAYPTQ.mjs';
15
- import { validatePatternValueLimit, requireSurfaceFormsForPartialFields } from './chunk-DCK4P5BI.mjs';
16
- import { setNestedValue } from './chunk-X4SEXL4U.mjs';
17
- import { restoredConfigFields, restoredEmbedding } from './chunk-J2BPXTD7.mjs';
18
- import { getLanguage } from './chunk-PR4O7CAP.mjs';
19
- import { MAX_FILE_BYTES } from './chunk-RVM7AODE.mjs';
20
- import './chunk-AV4IJKD2.mjs';
21
- import { BATCH_CHUNK_SIZE } from './chunk-Q5DC6WAB.mjs';
22
- import './chunk-TGP5M5R7.mjs';
23
- import './chunk-Y2F66IQJ.mjs';
24
- import './chunk-ZJ4CSAB2.mjs';
25
- import './chunk-QZN3MIJ3.mjs';
26
- import './chunk-T6CNO72J.mjs';
27
- import './chunk-WQBCBKY3.mjs';
28
- import { parseVectorDimension } from './chunk-A3UB2MXL.mjs';
29
- import { raiseEarlierReleaseGraphSetting, validateSchema, validateVectorPromotion, validateVectorStorage, flattenSchema } from './chunk-CVQWQVFX.mjs';
30
- import './chunk-FGMGUDHX.mjs';
31
- import './chunk-KEHK2O6P.mjs';
32
- import './chunk-GES7U2PM.mjs';
33
- import './chunk-BVOJIYRV.mjs';
34
- import './chunk-U2BKYEC7.mjs';
35
- import './chunk-R6A773WK.mjs';
36
- import './chunk-YTJCGKM4.mjs';
37
- import './chunk-YRX6XMOQ.mjs';
38
- import { loadVectorCodeFor } from './chunk-R7OSDAFK.mjs';
39
- import './chunk-L5ORED7V.mjs';
40
- import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
41
- import { decode } from '@msgpack/msgpack';
42
-
43
- // src/persistence/archive/validate.ts
44
- function requireDimension(key, found, expected) {
45
- if (expected === found) return;
46
- throw new NarsilError(
47
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
48
- `The vector file "${key}" holds vectors of ${found} dimensions where the schema gives ${String(expected)}`,
49
- { key, found, expected }
50
- );
51
- }
52
- function requireSchemaOfManifest(key, schema, manifest) {
53
- for (const [field, type] of Object.entries(schema)) {
54
- const listed = manifest.schema[field];
55
- if (listed === void 0 || listed === type) continue;
56
- throw new NarsilError(
57
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
58
- `The segment "${key}" gives field "${field}" the type "${type}", while the manifest gives it "${listed}"`,
59
- { key, field }
60
- );
61
- }
62
- }
63
- async function validateStagedCheckpoint(store, manifest) {
64
- for (const partition of manifest.partitions) {
65
- for (const ref of partition.segments) {
66
- const contents = await readSegmentFile(store, ref);
67
- requireSchemaOfManifest(ref.key, contents.schema, manifest);
68
- decodeEveryStoredDocument(contents.arrays.documentTable);
69
- }
70
- }
71
- for (const field of manifest.vectors) {
72
- const fieldType = manifest.schema[field.fieldPath];
73
- const dimension = fieldType === void 0 ? null : parseVectorDimension(fieldType);
74
- for (const file of field.files) {
75
- const { payload } = await readVectorFile(store, file, field.fieldPath);
76
- requireDimension(file.key, payload.dimension, dimension);
77
- }
78
- await readGraphParts(store, field);
79
- }
80
- }
81
-
82
- // src/engine/snapshot/restore-setup.ts
83
- function prepareRestoredIndex(configuration, deps, reject) {
84
- const configFields = restoredConfigFields(configuration, reject);
85
- const embedding = restoredEmbedding(configuration.embedding, reject);
86
- const languageName = configuration.language ?? "";
87
- const language = getLanguage(languageName);
88
- const schema = validateSchema(configuration.schema);
89
- validatePatternValueLimit(configFields.patternValueLimit);
90
- requireSurfaceFormsForPartialFields(schema, configFields.surfaceForms);
91
- validateVectorPromotion(configFields.vectorPromotion);
92
- validateVectorStorage(configFields.vectorPromotion, deps.filesystemDurability);
93
- if (configFields.required !== void 0 && configFields.required.length > 0) {
94
- validateRequiredFieldsInSchema(configFields.required, schema);
95
- }
96
- const config = {
97
- schema,
98
- language: languageName,
99
- ...configFields,
100
- ...embedding !== void 0 ? {
101
- embedding: {
102
- fields: embedding.fields,
103
- ...embedding.adapter !== void 0 ? { adapter: embedding.adapter } : {}
104
- }
105
- } : {}
106
- };
107
- resolveIndexAnalysis(config);
108
- const adapterName = embedding?.adapter ?? null;
109
- let embeddingAdapter = null;
110
- if (embedding !== void 0) {
111
- const candidate = adapterName !== null ? deps.embeddingAdapters.get(adapterName) ?? null : deps.defaultEmbeddingAdapter;
112
- if (candidate) {
113
- validateEmbeddingConfig({ fields: embedding.fields, adapter: candidate }, schema, void 0);
114
- embeddingAdapter = candidate;
115
- }
116
- }
117
- const storedRevision = typeof configuration.analysisRevision === "string" ? configuration.analysisRevision : null;
118
- return { config, language, embeddingAdapter, adapterName, storedRevision };
119
- }
120
- async function createRestoredIndex(indexName, prepared, partitionCount, deps) {
121
- const loadingVectorCode = loadVectorCodeFor(prepared.config.schema);
122
- if (loadingVectorCode !== null) await loadingVectorCode;
123
- if (deps.indexRegistry.has(indexName)) {
124
- await deps.dropIndex(indexName);
125
- }
126
- deps.clearAnalysisStale(indexName);
127
- if (prepared.storedRevision !== prepared.language.revision) {
128
- deps.markAnalysisStale({
129
- indexName,
130
- language: prepared.language.name,
131
- storedRevision: prepared.storedRevision,
132
- currentRevision: prepared.language.revision
133
- });
134
- }
135
- deps.executor.createIndex(indexName, prepared.config, prepared.language);
136
- deps.indexRegistry.set(indexName, {
137
- config: prepared.config,
138
- language: prepared.language,
139
- embeddingAdapter: prepared.embeddingAdapter,
140
- embeddingAdapterName: prepared.adapterName,
141
- vectorFieldPaths: deps.getVectorFieldPaths(prepared.config.schema),
142
- indexUuid: null,
143
- heldPartitions: null,
144
- documentCount: 0,
145
- partitionCount,
146
- analysisRevision: prepared.storedRevision
147
- });
148
- await deps.indexState.registerOpen(indexName);
149
- const release = await deps.indexState.acquire(indexName, false);
150
- const manager = deps.requireManager(indexName);
151
- while (manager.partitionCount < partitionCount) manager.addPartition();
152
- return { manager, release };
153
- }
154
- async function abandonRestoredIndex(indexName, deps) {
155
- try {
156
- deps.executor.dropIndex(indexName);
157
- } catch (_) {
158
- }
159
- deps.indexRegistry.delete(indexName);
160
- deps.indexState.forget(indexName);
161
- if (deps.durability) {
162
- await deps.durability.manager.removeIndex(indexName).catch(() => void 0);
163
- }
164
- }
165
- function recordRestoredCounts(indexName, manager, deps) {
166
- const entry = deps.indexRegistry.get(indexName);
167
- if (entry !== void 0) {
168
- entry.documentCount = manager.countDocuments();
169
- entry.partitionCount = manager.partitionCount;
170
- }
171
- }
172
-
173
- // src/engine/snapshot/archive-restore.ts
174
- function rejectConfiguration(message) {
175
- throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Invalid index configuration: ${message}`);
176
- }
177
- function renamed(manifest, sourceName, indexName) {
178
- const rename = (key) => `${indexName}/${key.slice(sourceName.length + 1)}`;
179
- return {
180
- ...manifest,
181
- partitions: manifest.partitions.map((partition) => ({
182
- ...partition,
183
- segments: partition.segments.map((segment) => ({ ...segment, key: rename(segment.key) }))
184
- })),
185
- vectors: manifest.vectors.map((field) => ({
186
- ...field,
187
- files: field.files.map((file) => ({ ...file, key: rename(file.key) })),
188
- graphParts: field.graphParts.map((part) => ({ ...part, key: rename(part.key) }))
189
- }))
190
- };
191
- }
192
- function requireSameSchema(prepared, manifest) {
193
- const configured = flattenSchema(prepared.config.schema);
194
- const keys = /* @__PURE__ */ new Set([...Object.keys(configured), ...Object.keys(manifest.schema)]);
195
- for (const field of keys) {
196
- if (configured[field] === manifest.schema[field]) continue;
197
- throw new NarsilError(
198
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
199
- `The archive's configuration gives field "${field}" another type than its manifest does`,
200
- { field }
201
- );
202
- }
203
- }
204
- function partitionCountOf(manifest) {
205
- let count = 1;
206
- for (const partition of manifest.partitions) count = Math.max(count, partition.partitionId + 1);
207
- return count;
208
- }
209
- function missingPart(what) {
210
- throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The archive holds no ${what}`);
211
- }
212
- async function stageArchive(indexName, reader, staging, deps) {
213
- const staged = { prepared: null, manifest: null };
214
- await readSnapshotArchive(reader, {
215
- config: async (bytes) => {
216
- staged.prepared = prepareRestoredIndex(await decodeIndexConfiguration(bytes), deps, rejectConfiguration);
217
- },
218
- manifest: async (manifest2, sourceName) => {
219
- if (staged.prepared === null) missingPart("configuration");
220
- requireSameSchema(staged.prepared, manifest2);
221
- staged.manifest = renamed(manifest2, sourceName, indexName);
222
- },
223
- file: async (entryName, _ref, bytes) => {
224
- if (staged.manifest === null) missingPart("manifest");
225
- await staging.write(`${indexName}/${entryName}`, [bytes]);
226
- }
227
- });
228
- const { prepared, manifest } = staged;
229
- if (prepared === null) missingPart("configuration");
230
- if (manifest === null) missingPart("manifest");
231
- await validateStagedCheckpoint(staging, manifest);
232
- return { prepared, manifest };
233
- }
234
- async function replaceWithStaged(indexName, staged, staging, memory, deps) {
235
- const { prepared, manifest } = staged;
236
- const durable = deps.durability?.manager;
237
- const created = await createRestoredIndex(indexName, prepared, partitionCountOf(manifest), deps);
238
- try {
239
- if (staging !== null && durable !== void 0) await durable.persistMetadata(indexName);
240
- const store = staging === null ? memory : await staging.commit();
241
- await store.write(manifestKey(indexName), (await encodeSegmentManifest(manifest)).chunks);
242
- await loadSegmentedCheckpoint(
243
- store,
244
- indexName,
245
- manifest,
246
- { manager: created.manager, vectorIndexes: created.manager.getVectorIndexes() },
247
- true
248
- );
249
- recordRestoredCounts(indexName, created.manager, deps);
250
- if (durable?.adoptRestoredCheckpoint !== void 0 && staging !== null) {
251
- durable.adoptRestoredCheckpoint(indexName, manifest.checkpoint);
252
- await durable.persistMetadata(indexName);
253
- } else if (durable !== void 0) {
254
- await durable.persistMetadata(indexName);
255
- await durable.checkpoint(indexName);
256
- }
257
- await deps.announceIndexCreated(indexName, prepared.config);
258
- return created;
259
- } catch (error) {
260
- created.release();
261
- await abandonRestoredIndex(indexName, deps);
262
- throw error;
263
- }
264
- }
265
- async function restoreArchive(indexName, reader, deps) {
266
- const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
267
- const memory = memoryCheckpointStore();
268
- try {
269
- const staged = await stageArchive(indexName, reader, staging?.store ?? memory, deps);
270
- const created = await replaceWithStaged(indexName, staged, staging, memory, deps);
271
- created.release();
272
- } finally {
273
- await staging?.discard().catch(() => void 0);
274
- await reader.close();
275
- }
276
- }
277
- function invalidSnapshot(message, details) {
278
- throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, message, details);
279
- }
280
- async function decodeEarlierSnapshot(data) {
281
- const payloadBytes = await unpackIndexSnapshotEnvelope(data);
282
- let decoded;
283
- try {
284
- decoded = decode(payloadBytes);
285
- } catch (err) {
286
- invalidSnapshot(`Snapshot data is not a Narsil snapshot: ${err instanceof Error ? err.message : String(err)}`, {
287
- bytes: data.length
288
- });
289
- }
290
- if (typeof decoded !== "object" || decoded === null || Array.isArray(decoded)) {
291
- invalidSnapshot("Snapshot data does not hold a snapshot envelope", { bytes: data.length });
292
- }
293
- const envelope = decoded;
294
- if (envelope.version !== 1 && envelope.version !== 2) {
295
- invalidSnapshot(`Unsupported snapshot version: ${envelope.version}. Expected version 1 or 2`, {
296
- version: envelope.version
297
- });
298
- }
299
- if (!envelope.schema || typeof envelope.schema !== "object")
300
- invalidSnapshot("Invalid snapshot: missing or invalid schema");
301
- if (!envelope.language || typeof envelope.language !== "string") {
302
- invalidSnapshot("Invalid snapshot: missing or invalid language");
303
- }
304
- if (!Array.isArray(envelope.partitions)) invalidSnapshot("Invalid snapshot: missing partitions");
305
- return { ...envelope, vectorPromotion: raiseEarlierReleaseGraphSetting(envelope.vectorPromotion) };
306
- }
307
- function vectorsOfSnapshot(envelope) {
308
- const byDocument = /* @__PURE__ */ new Map();
309
- for (const [fieldPath, rawParts] of Object.entries(envelope.vectorIndexes ?? {})) {
310
- for (const part of decodeVectorIndexParts(rawParts)) {
311
- const values = bytesToVectors(part.vectors);
312
- for (let i = 0; i < part.docIds.length; i++) {
313
- let fields = byDocument.get(part.docIds[i]);
314
- if (fields === void 0) {
315
- fields = /* @__PURE__ */ new Map();
316
- byDocument.set(part.docIds[i], fields);
317
- }
318
- fields.set(fieldPath, values.slice(i * part.dimension, (i + 1) * part.dimension));
319
- }
320
- }
321
- }
322
- return byDocument;
323
- }
324
- function documentsOfPartition(envelope, payload) {
325
- return envelope.version === 2 ? documentsOfPayloadV2(payload).documents : deserializePayloadV1(payload).documents;
326
- }
327
- function withVectors(fields, vectors) {
328
- if (vectors === void 0 || vectors.size === 0) return fields;
329
- const nested = [...vectors.keys()].some((fieldPath) => fieldPath.includes("."));
330
- const document = nested ? structuredClone(fields) : { ...fields };
331
- for (const [fieldPath, vector] of vectors) setNestedValue(document, fieldPath, vector);
332
- return document;
333
- }
334
- function insertRestoredDocument(manager, docId, fields, vectors) {
335
- manager.insert(docId, fields, { skipClone: true });
336
- if (vectors === void 0) return;
337
- try {
338
- insertDocumentVectors(docId, vectors, manager.getVectorIndexes(), manager.partitionIdOf(docId));
339
- } catch (err) {
340
- manager.remove(docId);
341
- throw err;
342
- }
343
- }
344
- async function insertRestoredChunk(indexName, manager, chunk, vectors, deps) {
345
- if (deps.durability === null) {
346
- for (const [docId, fields] of chunk) insertRestoredDocument(manager, docId, fields, vectors.get(docId));
347
- return;
348
- }
349
- const mutations = chunk.map(([docId, fields]) => ({
350
- docId,
351
- document: withVectors(fields, vectors.get(docId)),
352
- apply: () => insertRestoredDocument(manager, docId, fields, vectors.get(docId))
353
- }));
354
- for (const outcome of await deps.durability.recordMutationBatch(indexName, mutations)) {
355
- if (!outcome.ok) throw outcome.error;
356
- }
357
- }
358
- async function insertSnapshotDocuments(indexName, envelope, manager, deps) {
359
- const vectors = vectorsOfSnapshot(envelope);
360
- for (const payload of envelope.partitions ?? []) {
361
- const documents = Object.entries(documentsOfPartition(envelope, payload)).map(
362
- ([docId, stored]) => [docId, stored.fields]
363
- );
364
- for (let start = 0; start < documents.length; start += BATCH_CHUNK_SIZE) {
365
- await insertRestoredChunk(indexName, manager, documents.slice(start, start + BATCH_CHUNK_SIZE), vectors, deps);
366
- }
367
- }
368
- for (const vectorIndex of manager.getVectorIndexes().values()) vectorIndex.scheduleBuild();
369
- }
370
- async function restoreEarlierSnapshot(indexName, data, deps) {
371
- const envelope = await decodeEarlierSnapshot(data);
372
- const prepared = prepareRestoredIndex(envelope, deps);
373
- const analysedNow = { ...prepared, storedRevision: prepared.language.revision };
374
- const partitionCount = envelope.partitions?.length ?? 0;
375
- const { manager, release } = await createRestoredIndex(indexName, analysedNow, partitionCount, deps);
376
- try {
377
- await insertSnapshotDocuments(indexName, envelope, manager, deps);
378
- recordRestoredCounts(indexName, manager, deps);
379
- if (deps.durability) {
380
- await deps.durability.manager.persistMetadata(indexName);
381
- await deps.durability.manager.checkpoint(indexName);
382
- }
383
- await deps.announceIndexCreated(indexName, prepared.config);
384
- } catch (err) {
385
- await abandonRestoredIndex(indexName, deps);
386
- throw err;
387
- } finally {
388
- release();
389
- }
390
- }
391
-
392
- // src/engine/snapshot/index.ts
393
- var NRSL_MAGIC_BYTES = 4;
394
- async function checkpointForArchive(deps) {
395
- const durable = deps.durability?.manager;
396
- if (durable?.openArchiveCheckpoint !== void 0) return durable.openArchiveCheckpoint(deps.indexName);
397
- return buildArchiveFromMemory({
398
- indexName: deps.indexName,
399
- metadata: indexMetadataOf(deps.indexName, deps.entry, deps.manager.partitionCount),
400
- manager: deps.manager,
401
- vectorIndexes: deps.manager.getVectorIndexes(),
402
- appliedSeqNo: (partitionId) => durable?.highestPersistedSeqNo(deps.indexName, partitionId) ?? 0,
403
- partitionId: null
404
- });
405
- }
406
- async function createSnapshot(deps) {
407
- const config = await encodeIndexConfiguration(deps.entry);
408
- const checkpoint = await checkpointForArchive(deps);
409
- return archiveStream({ ...checkpoint, config, sourceIndexName: deps.indexName });
410
- }
411
- function requireSnapshotSource(data) {
412
- if (data instanceof Uint8Array) return data;
413
- if (data !== null && typeof data === "object") {
414
- if (typeof data.getReader === "function") return data;
415
- if (typeof data[Symbol.asyncIterator] === "function") {
416
- return data;
417
- }
418
- }
419
- throw new NarsilError(
420
- ErrorCodes.DOC_VALIDATION_FAILED,
421
- "Snapshot data must be a Uint8Array, a ReadableStream of bytes, or an async iterable of bytes"
422
- );
423
- }
424
- async function readEarlierSnapshotFile(reader) {
425
- const length = HEADER_SIZE + readHeader(await reader.peek(HEADER_SIZE)).payloadLength;
426
- if (length > MAX_FILE_BYTES) {
427
- throw new NarsilError(
428
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
429
- `The snapshot declares ${length} bytes, above the file size limit of ${MAX_FILE_BYTES}`,
430
- { bytes: length }
431
- );
432
- }
433
- const bytes = await reader.readExactly(length) ?? new Uint8Array(0);
434
- if (bytes.byteLength < length) {
435
- throw new NarsilError(
436
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
437
- "The snapshot ends before the payload its header declares"
438
- );
439
- }
440
- const trailing = await reader.readExactly(1);
441
- if (trailing !== null && trailing.byteLength > 0) {
442
- throw new NarsilError(
443
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
444
- "The snapshot holds bytes after the payload its header declares"
445
- );
446
- }
447
- return bytes;
448
- }
449
- async function restoreFromSnapshot(indexName, data, deps) {
450
- const reader = createByteReader(requireSnapshotSource(data));
451
- const head = await reader.peek(NRSL_MAGIC_BYTES);
452
- if (head.byteLength >= NRSL_MAGIC_BYTES && hasNrslMagic(head)) {
453
- let bytes;
454
- try {
455
- bytes = await readEarlierSnapshotFile(reader);
456
- } finally {
457
- await reader.close();
458
- }
459
- await restoreEarlierSnapshot(indexName, bytes, deps);
460
- return;
461
- }
462
- if (head.byteLength === 0) {
463
- await reader.close();
464
- throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Snapshot data is empty");
465
- }
466
- await restoreArchive(indexName, reader, deps);
467
- }
468
-
469
- export { checkpointForArchive, createSnapshot, restoreFromSnapshot };