@delali/narsil 0.2.4-next.59 → 0.3.1-next.101

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. package/README.md +9 -8
  2. package/dist/adapters/filesystem.d.ts +1 -1
  3. package/dist/adapters/filesystem.mjs +1 -1
  4. package/dist/adapters/indexeddb.d.ts +1 -1
  5. package/dist/adapters/indexeddb.mjs +1 -1
  6. package/dist/adapters/memory.d.ts +1 -1
  7. package/dist/{adapters-79OIa4xm.d.ts → adapters-BQFQ4PRz.d.ts} +27 -1
  8. package/dist/browser/chunk-BDOSMEEP.mjs +493 -0
  9. package/dist/browser/chunk-EJDUFWSO.mjs +602 -0
  10. package/dist/browser/chunk-HOZW3PKP.mjs +9 -0
  11. package/dist/browser/chunk-JOVZ6HMT.mjs +4358 -0
  12. package/dist/browser/chunk-JPTQZWJQ.mjs +10 -0
  13. package/dist/browser/chunk-KPJMDPHG.mjs +24558 -0
  14. package/dist/browser/chunk-PGIM3HIM.mjs +2743 -0
  15. package/dist/browser/chunk-RHG4O77A.mjs +206 -0
  16. package/dist/browser/chunk-RIJSLY2I.mjs +360 -0
  17. package/dist/browser/durability-wiring-SRUYT3VV.mjs +3875 -0
  18. package/dist/browser/module-CBB2ETYP.mjs +3111 -0
  19. package/dist/browser/release-76MXRPGV.mjs +33 -0
  20. package/dist/browser/replace-RURXTGVG.mjs +57 -0
  21. package/dist/browser/restore-queue-EEZQO6RF.mjs +35 -0
  22. package/dist/browser/runtime-7ZAHCHWR.mjs +557 -0
  23. package/dist/browser/snapshot-OAAA5UJ7.mjs +1210 -0
  24. package/dist/browser/write-7D6VIK6O.mjs +208 -0
  25. package/dist/{capabilities-DSsOGz6F.d.ts → capabilities-DiGlB6Sr.d.ts} +68 -7
  26. package/dist/chunk-2RIRGOUL.mjs +113 -0
  27. package/dist/chunk-37CD3OKS.mjs +87 -0
  28. package/dist/chunk-44W3CBNG.mjs +7 -0
  29. package/dist/chunk-46D6JQIG.mjs +426 -0
  30. package/dist/{chunk-VMRR3DQU.mjs → chunk-5JNA5SKS.mjs} +546 -35
  31. package/dist/chunk-747OHOUA.mjs +11 -0
  32. package/dist/chunk-75XIEF6H.mjs +126 -0
  33. package/dist/chunk-7X5JST5L.mjs +114 -0
  34. package/dist/chunk-BOFBLJFZ.mjs +209 -0
  35. package/dist/{chunk-USKFU7J5.mjs → chunk-BWJVNZPR.mjs} +8 -1
  36. package/dist/chunk-C7SLA6CM.mjs +19 -0
  37. package/dist/{chunk-POWOTUKJ.mjs → chunk-CQUMNIIG.mjs} +3 -6
  38. package/dist/chunk-CVXLLEQ3.mjs +1011 -0
  39. package/dist/chunk-DJHHF2L5.mjs +17 -0
  40. package/dist/chunk-E74YLAIG.mjs +24 -0
  41. package/dist/{chunk-UXNWQZST.mjs → chunk-EMRMGYRE.mjs} +236 -18
  42. package/dist/chunk-EWZLIYHV.mjs +57 -0
  43. package/dist/chunk-GBEEG5AN.mjs +35 -0
  44. package/dist/chunk-GES7U2PM.mjs +68 -0
  45. package/dist/chunk-HJSN7UYG.mjs +10 -0
  46. package/dist/chunk-HLPRHOSO.mjs +17 -0
  47. package/dist/chunk-HMFNFVP7.mjs +14 -0
  48. package/dist/chunk-J5QP52HA.mjs +1751 -0
  49. package/dist/{chunk-YNVIUGF5.mjs → chunk-JZVARCRZ.mjs} +3 -28
  50. package/dist/chunk-KOKM4PQN.mjs +52 -0
  51. package/dist/{chunk-B2TFROFX.mjs → chunk-KWTW6VTG.mjs} +465 -250
  52. package/dist/chunk-KZ3NX2P7.mjs +21 -0
  53. package/dist/chunk-L5ORED7V.mjs +25 -0
  54. package/dist/{chunk-THYRHHO2.mjs → chunk-LGAMKHA2.mjs} +2009 -2351
  55. package/dist/chunk-LKOB5COA.mjs +117 -0
  56. package/dist/chunk-LPQOEU5M.mjs +96 -0
  57. package/dist/{chunk-3LHGNRYE.mjs → chunk-LXWDQLTW.mjs} +7 -2
  58. package/dist/{chunk-L6TNPLAR.mjs → chunk-OEYI3KRB.mjs} +270 -108
  59. package/dist/chunk-OM362BRO.mjs +948 -0
  60. package/dist/chunk-OZDFA4WC.mjs +67 -0
  61. package/dist/chunk-P36MUFEJ.mjs +10 -0
  62. package/dist/chunk-PC2G76VW.mjs +2540 -0
  63. package/dist/{chunk-GCM3JFVA.mjs → chunk-PNEYIHA4.mjs} +11 -2
  64. package/dist/chunk-PNFKUCG3.mjs +1130 -0
  65. package/dist/chunk-Q6XWVUZX.mjs +44 -0
  66. package/dist/chunk-R27EMH5M.mjs +617 -0
  67. package/dist/{chunk-YMZCMH5M.mjs → chunk-R6A773WK.mjs} +19 -24
  68. package/dist/chunk-RF7PLG43.mjs +106 -0
  69. package/dist/chunk-RFBLQLLZ.mjs +16 -0
  70. package/dist/{chunk-Y3L5MDRV.mjs → chunk-RFVUDM5E.mjs} +17 -2
  71. package/dist/chunk-S7FL6UAH.mjs +521 -0
  72. package/dist/chunk-S7GNDRPS.mjs +651 -0
  73. package/dist/chunk-SP3NM3ZG.mjs +3537 -0
  74. package/dist/chunk-U476IXKQ.mjs +10 -0
  75. package/dist/chunk-UA3AUGNV.mjs +462 -0
  76. package/dist/chunk-UTG5YNXV.mjs +507 -0
  77. package/dist/chunk-VI4R6W25.mjs +8484 -0
  78. package/dist/chunk-WQBCBKY3.mjs +8 -0
  79. package/dist/{chunk-24RXHMEL.mjs → chunk-WVCOCKBU.mjs} +3794 -7570
  80. package/dist/chunk-X5CV2E2B.mjs +1 -0
  81. package/dist/chunk-XADZPPZI.mjs +614 -0
  82. package/dist/chunk-YEEQHT4V.mjs +212 -0
  83. package/dist/chunk-YTJCGKM4.mjs +56 -0
  84. package/dist/chunk-YYQIODMF.mjs +275 -0
  85. package/dist/chunk-ZI34TVOA.mjs +144 -0
  86. package/dist/{chunk-OOSIDHA3.mjs → chunk-ZKWYWW4Y.mjs} +1 -1
  87. package/dist/chunk-ZM7KU4YW.mjs +35 -0
  88. package/dist/{client-BQK6Tf6A.d.ts → client-BOStAkc5.d.ts} +160 -37
  89. package/dist/client.d.ts +11 -10
  90. package/dist/client.mjs +209 -45
  91. package/dist/{config-DmXULPa-.d.ts → config-Cjvzbs_j.d.ts} +51 -3
  92. package/dist/config-DffQlmRq.d.ts +421 -0
  93. package/dist/direct-executor-ZD64PPEZ.mjs +24 -0
  94. package/dist/distribution/coordinator/etcd.d.ts +79 -18
  95. package/dist/distribution/coordinator/etcd.mjs +553 -109
  96. package/dist/distribution/coordinator/in-memory.d.ts +4 -3
  97. package/dist/distribution/coordinator/in-memory.mjs +22 -3
  98. package/dist/distribution/transport/grpc.d.ts +2 -2
  99. package/dist/distribution/transport/grpc.mjs +79 -14
  100. package/dist/distribution/transport/in-memory.d.ts +2 -2
  101. package/dist/distribution/transport/in-memory.mjs +12 -7
  102. package/dist/distribution/transport/tcp.d.ts +2 -2
  103. package/dist/distribution/transport/tcp.mjs +50 -28
  104. package/dist/distribution.d.ts +144 -23
  105. package/dist/distribution.mjs +14479 -8173
  106. package/dist/durability-wiring-FEUBR5BT.mjs +3330 -0
  107. package/dist/embeddings/openai.d.ts +2 -2
  108. package/dist/embeddings/openai.mjs +22 -133
  109. package/dist/{engine-CIOyFozK.d.ts → engine-rC0vjjY-.d.ts} +165 -141
  110. package/dist/{memory-yFaVn2xZ.d.ts → events-DqOIg5xa.d.ts} +645 -136
  111. package/dist/{types-DAK7mnUZ.d.ts → index-Bkdt8nDw.d.ts} +182 -45
  112. package/dist/index.browser.mjs +4 -32180
  113. package/dist/index.d.ts +91 -20
  114. package/dist/index.mjs +33 -16
  115. package/dist/invalidation/broadcast-channel.d.ts +1 -1
  116. package/dist/invalidation/filesystem.d.ts +1 -1
  117. package/dist/invalidation/filesystem.mjs +2 -1
  118. package/dist/invalidation/noop.d.ts +1 -1
  119. package/dist/module-3S65MVCM.mjs +2151 -0
  120. package/dist/partial-words-DTIHO8G4.d.ts +19 -0
  121. package/dist/partial-words.d.ts +22 -0
  122. package/dist/partial-words.mjs +17 -0
  123. package/dist/pattern-Cledccbl.d.ts +18 -0
  124. package/dist/pattern.d.ts +22 -0
  125. package/dist/pattern.mjs +12 -0
  126. package/dist/persistence/durability/checkpoint-worker.d.ts +119 -72
  127. package/dist/persistence/durability/checkpoint-worker.mjs +51 -18
  128. package/dist/react.d.ts +279 -183
  129. package/dist/react.mjs +201 -48
  130. package/dist/release-BH3DKW3T.mjs +33 -0
  131. package/dist/replace-CEO5KJJ3.mjs +57 -0
  132. package/dist/rerank-K7C4WZFG.mjs +60 -0
  133. package/dist/rerankers/http.d.ts +50 -0
  134. package/dist/rerankers/http.mjs +104 -0
  135. package/dist/restore-queue-KQHA7ZV2.mjs +35 -0
  136. package/dist/runtime-6Y4ZYXOG.mjs +177 -0
  137. package/dist/{schema-d5Afu03A.d.ts → schema-4W_7tk1f.d.ts} +81 -11
  138. package/dist/server.d.ts +13 -12
  139. package/dist/server.mjs +611 -80
  140. package/dist/snapshot-QGZPODM5.mjs +799 -0
  141. package/dist/synonyms-CpOAHPa-.d.ts +52 -0
  142. package/dist/synonyms.d.ts +22 -0
  143. package/dist/synonyms.mjs +12 -0
  144. package/dist/{tasks-CH2Dod8w.d.ts → tasks-CwPTz-tP.d.ts} +10 -3
  145. package/dist/thread-N3IUGUS3.mjs +697 -0
  146. package/dist/{types-BQDlIvfS.d.ts → types-2qoitTXg.d.ts} +1 -1
  147. package/dist/{types-D5OwhtsQ.d.ts → types-Cz5okLml.d.ts} +2 -2
  148. package/dist/vector/search-worker.d.ts +46 -4
  149. package/dist/vector/search-worker.mjs +21 -11
  150. package/dist/workers/entry.mjs +9 -6
  151. package/dist/write-U5RI7MAX.mjs +241 -0
  152. package/package.json +55 -9
  153. package/dist/chunk-3L4U6EJQ.mjs +0 -49
  154. package/dist/chunk-IFBKLBGB.mjs +0 -7
  155. package/dist/chunk-ORBLD3EL.mjs +0 -6
  156. package/dist/chunk-U7KYW4BA.mjs +0 -135
  157. package/dist/chunk-ZJ4CSAB2.mjs +0 -13
  158. package/dist/chunk-ZNZBXMP5.mjs +0 -2117
  159. package/dist/chunk-ZZT3EAUC.mjs +0 -11168
  160. package/dist/config-BhldZgaK.d.ts +0 -194
  161. package/dist/direct-executor-ALAVE7FC.mjs +0 -12
  162. package/dist/serialization/crc32-worker.d.ts +0 -20
  163. package/dist/serialization/crc32-worker.mjs +0 -62
  164. package/dist/simd-DW5AtJ6B.d.ts +0 -41
  165. package/dist/thread-DYHEQRVY.mjs +0 -402
@@ -0,0 +1,521 @@
1
+ import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR } from './chunk-P36MUFEJ.mjs';
2
+ import { readSharedConfigFields, restoredEmbedding, restoredPartitionConfig } from './chunk-EMRMGYRE.mjs';
3
+ import { generateId, getLanguage } from './chunk-ZM7KU4YW.mjs';
4
+ import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-LXWDQLTW.mjs';
5
+ import { decodeSynonymRules } from './chunk-7X5JST5L.mjs';
6
+ import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
7
+ import { NarsilError, ErrorCodes } from './chunk-BWJVNZPR.mjs';
8
+ import { encode, decode } from '@msgpack/msgpack';
9
+
10
+ // src/narsil/write-blocks.ts
11
+ var writeBlockLookups = /* @__PURE__ */ new WeakMap();
12
+ function bindWriteBlockLookup(owner, lookup) {
13
+ writeBlockLookups.set(owner, lookup);
14
+ }
15
+ function writeBlockLookupOf(owner) {
16
+ return writeBlockLookups.get(owner);
17
+ }
18
+ function bindEngineWriteBlock(engine, owner) {
19
+ const lookup = writeBlockLookups.get(owner);
20
+ if (lookup !== void 0) writeBlockLookups.set(engine, lookup);
21
+ }
22
+
23
+ // src/distribution/cluster/constants.ts
24
+ var LEADERSHIP_IMBALANCE_THRESHOLD = 2;
25
+ var ESTIMATED_PARTITION_BYTES = 50 * 1024 * 1024;
26
+ var REBALANCE_THRESHOLD = 0.1;
27
+ var BOOTSTRAP_CAS_ATTEMPTS = 5;
28
+ var ALLOCATION_CAS_ATTEMPTS = 5;
29
+ var ALLOCATION_RETRY_DELAY_MS = 1e3;
30
+ var TEARDOWN_CAS_ATTEMPTS = 5;
31
+ var TEARDOWN_RETRY_DELAY_MS = 500;
32
+ var TEARDOWN_RETRY_ROUNDS = 3;
33
+ var RECOVERY_CAS_ATTEMPTS = 5;
34
+ var REPLICATION_FACTOR_CAS_ATTEMPTS = 5;
35
+ var HIDDEN_GENERATION_PREFIX = "narsil-generation-";
36
+ var RESHARD_CUTOVER_LAG_ENTRIES = 1e3;
37
+ var RESHARD_CUTOVER_TIMEOUT_MS = 1e4;
38
+ var MAX_RESHARD_CUTOVER_ATTEMPTS = 10;
39
+ var RESHARD_POLL_INTERVAL_MS = 50;
40
+ var RESHARD_CONTROLLER_POLL_INTERVAL_MS = 250;
41
+ var RESHARD_RECORD_CHECK_INTERVAL_MS = 1e3;
42
+ var RESHARD_RETRY_DELAY_MS = 200;
43
+ var RESHARD_FORWARD_BATCH_DOCUMENTS = 256;
44
+ var RESHARD_RECORD_CAS_ATTEMPTS = 10;
45
+ var PARTITION_STORES_TIMEOUT_MS = 5e3;
46
+ var DEFAULT_CONTROLLER_CONFIG = {
47
+ leaseTtlMs: 15e3,
48
+ standbyRetryMs: 5e3,
49
+ nodeLeftDelayMs: 6e4
50
+ };
51
+ var DEFAULT_NODE_LIFECYCLE_CONFIG = {
52
+ bootstrapRetryBaseMs: 1e3,
53
+ bootstrapRetryMaxMs: 5e3,
54
+ bootstrapMaxRetries: 10,
55
+ allocationDebounceMs: 250,
56
+ nodeHeartbeatIntervalMs: 1e4
57
+ };
58
+ var MAX_SYNONYM_GENERATION_LENGTH = 128;
59
+
60
+ // src/narsil/index-reads.ts
61
+ var boundReads = /* @__PURE__ */ new WeakMap();
62
+ function bindIndexReads(engine, reads) {
63
+ boundReads.set(engine, reads);
64
+ }
65
+ function indexReadsOf(engine) {
66
+ const bound = boundReads.get(engine);
67
+ if (bound !== void 0) return bound;
68
+ return {
69
+ listIndexes: async () => engine.listIndexes(),
70
+ getStats: async (indexName) => engine.getStats(indexName),
71
+ getPartitionStats: async (indexName) => engine.getPartitionStats(indexName)
72
+ };
73
+ }
74
+
75
+ // src/distribution/payload-guards.ts
76
+ function isRecord(value) {
77
+ return value !== null && typeof value === "object" && !Array.isArray(value);
78
+ }
79
+ function isValidInteger(value) {
80
+ return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value);
81
+ }
82
+
83
+ // src/distribution/cluster/synonym-set.ts
84
+ var SYNONYM_CHUNK_BYTES = 1048576;
85
+ var SYNONYM_SET_PREFIX = "_narsil/synonyms/";
86
+ var MAX_UINT32 = 4294967295;
87
+ var DIGEST_PATTERN = /^[0-9a-f]{64}$/;
88
+ var FIRST_PRINTABLE_CODE = 32;
89
+ function isKeySegment(value) {
90
+ for (let i = 0; i < value.length; i++) {
91
+ const code = value.charCodeAt(i);
92
+ if (code < FIRST_PRINTABLE_CODE || value[i] === "/" || value[i] === "\\") return false;
93
+ }
94
+ return true;
95
+ }
96
+ function synonymChunkKey(indexUuid, generation, chunk) {
97
+ return `${SYNONYM_SET_PREFIX}${indexUuid}/${generation}/${chunk}`;
98
+ }
99
+ function isUint32(value) {
100
+ return typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= MAX_UINT32;
101
+ }
102
+ async function digestOf(bytes) {
103
+ const view = bytes.buffer instanceof ArrayBuffer ? new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength) : Uint8Array.from(bytes);
104
+ const hash = new Uint8Array(await crypto.subtle.digest("SHA-256", view));
105
+ return Array.from(hash, (byte) => byte.toString(16).padStart(2, "0")).join("");
106
+ }
107
+ function synonymSetRefOf(raw) {
108
+ if (!isRecord(raw)) return "synonyms must be an object";
109
+ const { generation, digest, chunkCount, byteLength } = raw;
110
+ if (typeof generation !== "string" || generation.length === 0 || generation.length > MAX_SYNONYM_GENERATION_LENGTH || !isKeySegment(generation)) {
111
+ return "synonyms.generation must hold 1 to 128 characters, with no control character, slash, or backslash";
112
+ }
113
+ if (typeof digest !== "string" || !DIGEST_PATTERN.test(digest)) {
114
+ return "synonyms.digest must be a lowercase hexadecimal SHA-256";
115
+ }
116
+ if (!isUint32(byteLength) || byteLength === 0) {
117
+ return "synonyms.byteLength must be a positive whole number";
118
+ }
119
+ if (!isUint32(chunkCount) || chunkCount !== Math.ceil(byteLength / SYNONYM_CHUNK_BYTES)) {
120
+ return "synonyms.chunkCount must match synonyms.byteLength";
121
+ }
122
+ return { generation, digest, chunkCount, byteLength };
123
+ }
124
+ function decodeSynonymSetRef(raw, reject) {
125
+ const ref = synonymSetRefOf(raw);
126
+ return typeof ref === "string" ? reject(ref) : ref;
127
+ }
128
+ function synonymSetRefOrNull(raw) {
129
+ const ref = synonymSetRefOf(raw);
130
+ return typeof ref === "string" ? null : ref;
131
+ }
132
+ async function deleteSynonymSet(coordinator, indexUuid, ref) {
133
+ for (let chunk = 0; chunk < ref.chunkCount; chunk++) {
134
+ await coordinator.delete(synonymChunkKey(indexUuid, ref.generation, chunk));
135
+ }
136
+ }
137
+ async function storeSynonymSet(coordinator, indexUuid, encoded) {
138
+ const ref = {
139
+ generation: generateId(),
140
+ digest: await digestOf(encoded),
141
+ chunkCount: Math.ceil(encoded.byteLength / SYNONYM_CHUNK_BYTES),
142
+ byteLength: encoded.byteLength
143
+ };
144
+ try {
145
+ for (let chunk = 0; chunk < ref.chunkCount; chunk++) {
146
+ const start = chunk * SYNONYM_CHUNK_BYTES;
147
+ const bytes = encoded.subarray(start, Math.min(start + SYNONYM_CHUNK_BYTES, encoded.byteLength));
148
+ if (!await coordinator.compareAndSet(synonymChunkKey(indexUuid, ref.generation, chunk), null, bytes)) {
149
+ throw new NarsilError(
150
+ ErrorCodes.CONTROLLER_METADATA_INVALID,
151
+ `Synonym set chunk ${chunk} of a fresh copy already exists in the coordinator`,
152
+ { indexUuid, generation: ref.generation, chunk }
153
+ );
154
+ }
155
+ }
156
+ } catch (err) {
157
+ await deleteSynonymSet(coordinator, indexUuid, ref).catch(() => void 0);
158
+ throw err;
159
+ }
160
+ return ref;
161
+ }
162
+ async function readSynonymSet(coordinator, indexUuid, ref, reject) {
163
+ const encoded = new Uint8Array(ref.byteLength);
164
+ let filled = 0;
165
+ for (let chunk = 0; chunk < ref.chunkCount; chunk++) {
166
+ const bytes = await coordinator.get(synonymChunkKey(indexUuid, ref.generation, chunk));
167
+ if (bytes === null) return null;
168
+ if (bytes.byteLength > encoded.byteLength - filled) return reject("the stored synonym set exceeds its byteLength");
169
+ encoded.set(bytes, filled);
170
+ filled += bytes.byteLength;
171
+ }
172
+ if (filled !== ref.byteLength) return reject("the stored synonym set falls short of its byteLength");
173
+ if (await digestOf(encoded) !== ref.digest) return reject("the stored synonym set does not match its digest");
174
+ return encoded;
175
+ }
176
+ function storedSynonymSetRules(encoded, reject) {
177
+ return decodeSynonymRules(encoded, (message) => reject(message));
178
+ }
179
+
180
+ // src/distribution/cluster/index-settings.ts
181
+ function partitionLimitsOf(partitions) {
182
+ if (partitions === void 0) {
183
+ return void 0;
184
+ }
185
+ const { maxDocsPerPartition, watermark } = partitions;
186
+ if (maxDocsPerPartition === void 0 && watermark === void 0) {
187
+ return void 0;
188
+ }
189
+ return {
190
+ ...maxDocsPerPartition !== void 0 ? { maxDocsPerPartition } : {},
191
+ ...watermark !== void 0 ? { watermark } : {}
192
+ };
193
+ }
194
+ function codeInSettings(option, registration) {
195
+ return new NarsilError(
196
+ ErrorCodes.CONFIG_INVALID,
197
+ `A node cannot serialise code into the cluster's index metadata, so register the index ${option} with ${registration} and pass its name`,
198
+ { option }
199
+ );
200
+ }
201
+ function settingsFieldsOf(source) {
202
+ const { tokenizer, stopWords, bm25, embedding } = source;
203
+ const partitionConfig = partitionLimitsOf(source.partitions);
204
+ if (tokenizer !== void 0 && typeof tokenizer !== "string") {
205
+ throw codeInSettings("tokenizer", "registerTokenizer");
206
+ }
207
+ if (typeof stopWords === "function") {
208
+ throw codeInSettings("stop words", "registerStopWords");
209
+ }
210
+ if (embedding?.adapter !== void 0 && typeof embedding.adapter !== "string") {
211
+ throw codeInSettings("embedding adapter", "registerEmbeddingAdapter");
212
+ }
213
+ return {
214
+ ...tokenizer !== void 0 ? { tokenizer } : {},
215
+ ...typeof stopWords === "string" ? { stopWords } : {},
216
+ ...stopWords instanceof Set ? { stopWordList: [...stopWords].sort(compareCodePoints) } : {},
217
+ ...bm25 !== void 0 ? {
218
+ bm25: {
219
+ ...bm25.k1 !== void 0 ? { k1: bm25.k1 } : {},
220
+ ...bm25.b !== void 0 ? { b: bm25.b } : {}
221
+ }
222
+ } : {},
223
+ ...source.surfaceForms !== void 0 ? { surfaceForms: source.surfaceForms } : {},
224
+ ...source.defaultScoring !== void 0 ? { defaultScoring: source.defaultScoring } : {},
225
+ ...source.trackPositions !== void 0 ? { trackPositions: source.trackPositions } : {},
226
+ ...source.strict !== void 0 ? { strict: source.strict } : {},
227
+ ...source.required !== void 0 ? { required: source.required } : {},
228
+ ...source.vectorPromotion !== void 0 ? { vectorPromotion: source.vectorPromotion } : {},
229
+ ...source.patternValueLimit !== void 0 ? { patternValueLimit: source.patternValueLimit } : {},
230
+ ...partitionConfig !== void 0 ? { partitionConfig } : {},
231
+ ...embedding !== void 0 ? {
232
+ embedding: {
233
+ fields: embedding.fields,
234
+ ...typeof embedding.adapter === "string" ? { adapter: embedding.adapter } : {}
235
+ }
236
+ } : {}
237
+ };
238
+ }
239
+ function indexSettingsOf(config) {
240
+ return {
241
+ language: getLanguage(config.language ?? "english").name,
242
+ ...settingsFieldsOf(config)
243
+ };
244
+ }
245
+ function isRecord2(value) {
246
+ return value !== null && typeof value === "object" && !Array.isArray(value);
247
+ }
248
+ function settingsRejection(indexName) {
249
+ return (message) => {
250
+ throw new NarsilError(
251
+ ErrorCodes.CONTROLLER_METADATA_INVALID,
252
+ `Index metadata for '${indexName}' has invalid settings: ${message}`,
253
+ { indexName }
254
+ );
255
+ };
256
+ }
257
+ function decodeIndexSettings(raw, indexName) {
258
+ if (raw === void 0 || raw === null) {
259
+ return void 0;
260
+ }
261
+ const reject = settingsRejection(indexName);
262
+ if (!isRecord2(raw)) {
263
+ return reject("settings must be an object");
264
+ }
265
+ if (typeof raw.language !== "string" || raw.language.length === 0) {
266
+ return reject("language must be a name");
267
+ }
268
+ const embedding = restoredEmbedding(raw.embedding, reject);
269
+ const partitions = restoredPartitionConfig(raw.partitionConfig, reject);
270
+ const synonyms = raw.synonyms === void 0 ? void 0 : decodeSynonymSetRef(raw.synonyms, reject);
271
+ return {
272
+ language: raw.language,
273
+ ...settingsFieldsOf({
274
+ ...readSharedConfigFields(raw, reject),
275
+ ...partitions !== void 0 ? { partitions } : {},
276
+ ...embedding !== void 0 ? { embedding } : {}
277
+ }),
278
+ ...synonyms !== void 0 ? { synonyms } : {}
279
+ };
280
+ }
281
+ function withPartitionCount(config, partitionCount) {
282
+ return { ...config, partitions: { ...config.partitions, maxPartitions: partitionCount } };
283
+ }
284
+ function indexConfigFromSettings(schema, settings, indexName) {
285
+ if (settings === void 0) {
286
+ return { schema };
287
+ }
288
+ const { language, embedding, partitionConfig, synonyms: _synonyms, ...fields } = settings;
289
+ return {
290
+ schema,
291
+ language,
292
+ ...readSharedConfigFields(fields, settingsRejection(indexName)),
293
+ ...partitionConfig !== void 0 ? { partitions: { ...partitionConfig } } : {},
294
+ ...embedding !== void 0 ? { embedding } : {}
295
+ };
296
+ }
297
+
298
+ // src/distribution/cluster/index-metadata.ts
299
+ var INDEX_CONFIG_PREFIX = "_narsil/index/";
300
+ var INDEX_CONFIG_SUFFIX = "/config";
301
+ function truncateForDisplay(value) {
302
+ const str = String(value);
303
+ if (str.length > 100) {
304
+ return `${str.slice(0, 100)}...`;
305
+ }
306
+ return str;
307
+ }
308
+ function validateIndexName(indexName) {
309
+ if (indexName.length === 0 || indexName.length > MAX_INDEX_NAME_LENGTH) {
310
+ throw new NarsilError(
311
+ ErrorCodes.CONTROLLER_METADATA_INVALID,
312
+ `Index name must be between 1 and ${MAX_INDEX_NAME_LENGTH} characters`,
313
+ { indexName: truncateForDisplay(indexName) }
314
+ );
315
+ }
316
+ if (indexName.includes("\0") || indexName.includes("/") || indexName.includes("\\") || indexName.includes("..")) {
317
+ throw new NarsilError(
318
+ ErrorCodes.CONTROLLER_METADATA_INVALID,
319
+ "Index name contains forbidden characters (/, \\, .., or null bytes)",
320
+ { indexName: truncateForDisplay(indexName) }
321
+ );
322
+ }
323
+ if (!INDEX_NAME_PATTERN.test(indexName)) {
324
+ throw new NarsilError(
325
+ ErrorCodes.CONTROLLER_METADATA_INVALID,
326
+ "Index name must start with an alphanumeric character and contain only alphanumeric characters, hyphens, underscores, or dots",
327
+ { indexName: truncateForDisplay(indexName) }
328
+ );
329
+ }
330
+ }
331
+ function isValidInteger2(value) {
332
+ return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value);
333
+ }
334
+ function indexConfigKey(indexName) {
335
+ return `${INDEX_CONFIG_PREFIX}${indexName}${INDEX_CONFIG_SUFFIX}`;
336
+ }
337
+ function validateDecodedMetadata(decoded, indexName) {
338
+ if (!isRecord2(decoded)) {
339
+ throw new NarsilError(
340
+ ErrorCodes.CONTROLLER_METADATA_INVALID,
341
+ `Index metadata for '${indexName}' is not an object`,
342
+ { indexName }
343
+ );
344
+ }
345
+ if (typeof decoded.indexUuid !== "string" || decoded.indexUuid.length === 0 || decoded.indexUuid.length > 128) {
346
+ throw new NarsilError(
347
+ ErrorCodes.CONTROLLER_METADATA_INVALID,
348
+ `Index metadata for '${indexName}' has invalid indexUuid`,
349
+ { indexName, received: truncateForDisplay(decoded.indexUuid) }
350
+ );
351
+ }
352
+ if (typeof decoded.indexName !== "string") {
353
+ throw new NarsilError(
354
+ ErrorCodes.CONTROLLER_METADATA_INVALID,
355
+ `Index metadata for '${indexName}' has invalid indexName`,
356
+ { indexName, received: truncateForDisplay(decoded.indexName) }
357
+ );
358
+ }
359
+ if (!isValidInteger2(decoded.partitionCount) || decoded.partitionCount <= 0 || decoded.partitionCount > MAX_PARTITION_COUNT) {
360
+ throw new NarsilError(
361
+ ErrorCodes.CONTROLLER_METADATA_INVALID,
362
+ `Index metadata for '${indexName}' has invalid partitionCount (must be an integer between 1 and ${MAX_PARTITION_COUNT})`,
363
+ { indexName, partitionCount: truncateForDisplay(decoded.partitionCount) }
364
+ );
365
+ }
366
+ if (!isValidInteger2(decoded.replicationFactor) || decoded.replicationFactor < 0 || decoded.replicationFactor > MAX_REPLICATION_FACTOR) {
367
+ throw new NarsilError(
368
+ ErrorCodes.CONTROLLER_METADATA_INVALID,
369
+ `Index metadata for '${indexName}' has invalid replicationFactor (must be an integer between 0 and ${MAX_REPLICATION_FACTOR})`,
370
+ { indexName, replicationFactor: truncateForDisplay(decoded.replicationFactor) }
371
+ );
372
+ }
373
+ if (!isRecord2(decoded.constraints)) {
374
+ throw new NarsilError(
375
+ ErrorCodes.CONTROLLER_METADATA_INVALID,
376
+ `Index metadata for '${indexName}' has invalid constraints`,
377
+ { indexName }
378
+ );
379
+ }
380
+ const constraints = decoded.constraints;
381
+ if (constraints.maxShardsPerNode !== void 0 && constraints.maxShardsPerNode !== null && typeof constraints.maxShardsPerNode !== "number") {
382
+ throw new NarsilError(
383
+ ErrorCodes.CONTROLLER_METADATA_INVALID,
384
+ `Index metadata for '${indexName}' has invalid maxShardsPerNode (must be a number or null)`,
385
+ { indexName, maxShardsPerNode: truncateForDisplay(constraints.maxShardsPerNode) }
386
+ );
387
+ }
388
+ return {
389
+ indexUuid: decoded.indexUuid,
390
+ indexName: decoded.indexName,
391
+ partitionCount: decoded.partitionCount,
392
+ replicationFactor: decoded.replicationFactor,
393
+ constraints: {
394
+ zoneAwareness: constraints.zoneAwareness === true,
395
+ zoneAttribute: typeof constraints.zoneAttribute === "string" ? constraints.zoneAttribute : "zone",
396
+ maxShardsPerNode: typeof constraints.maxShardsPerNode === "number" ? constraints.maxShardsPerNode : null
397
+ }
398
+ };
399
+ }
400
+ async function putIndexMetadata(coordinator, metadata) {
401
+ validateIndexName(metadata.indexName);
402
+ const key = indexConfigKey(metadata.indexName);
403
+ const encoded = encode(metadata, { ignoreUndefined: true });
404
+ const bytes = new Uint8Array(encoded);
405
+ if (await coordinator.compareAndSet(key, null, bytes)) {
406
+ return true;
407
+ }
408
+ const current = await coordinator.get(key);
409
+ if (current === null || current.byteLength > 0) {
410
+ return false;
411
+ }
412
+ return coordinator.compareAndSet(key, current, bytes);
413
+ }
414
+ async function putGenerationMetadata(coordinator, sourceName, generation) {
415
+ const sourceBytes = await coordinator.get(indexConfigKey(sourceName));
416
+ if (sourceBytes === null || sourceBytes.byteLength === 0) return false;
417
+ const decoded = decode(sourceBytes);
418
+ validateDecodedMetadata(decoded, sourceName);
419
+ if (!isRecord2(decoded)) return false;
420
+ const key = indexConfigKey(generation.indexName);
421
+ const current = await coordinator.get(key);
422
+ if (current !== null && current.byteLength > 0) return true;
423
+ const bytes = new Uint8Array(encode({ ...decoded, ...generation }, { ignoreUndefined: true }));
424
+ return coordinator.compareAndSet(key, current, bytes);
425
+ }
426
+ async function readStoredMetadata(coordinator, indexName) {
427
+ validateIndexName(indexName);
428
+ const raw = await coordinator.get(indexConfigKey(indexName));
429
+ if (raw === null || raw.byteLength === 0) {
430
+ return null;
431
+ }
432
+ const decoded = decode(raw);
433
+ const record = validateDecodedMetadata(decoded, indexName);
434
+ return { bytes: raw, record, rawSettings: isRecord2(decoded) ? decoded.settings : void 0 };
435
+ }
436
+ async function readIndexMetadataSnapshot(coordinator, indexName) {
437
+ const stored = await readStoredMetadata(coordinator, indexName);
438
+ if (stored === null) return null;
439
+ return { bytes: stored.bytes, record: stored.record, settings: decodeIndexSettings(stored.rawSettings, indexName) };
440
+ }
441
+ function metadataWithSynonymSet(snapshot, ref) {
442
+ const decoded = decode(snapshot.bytes);
443
+ const fields = isRecord2(decoded) ? decoded : {};
444
+ const { synonyms: _replaced, ...settings } = isRecord2(fields.settings) ? fields.settings : { language: snapshot.settings?.language ?? getLanguage("english").name };
445
+ const nextSettings = ref === null ? settings : { ...settings, synonyms: ref };
446
+ return new Uint8Array(encode({ ...fields, settings: nextSettings }, { ignoreUndefined: true }));
447
+ }
448
+ async function putIndexReplicationFactor(coordinator, indexName, replicationFactor) {
449
+ validateIndexName(indexName);
450
+ const key = indexConfigKey(indexName);
451
+ for (let attempt = 0; attempt < REPLICATION_FACTOR_CAS_ATTEMPTS; attempt++) {
452
+ const raw = await coordinator.get(key);
453
+ if (raw === null || raw.byteLength === 0) {
454
+ return null;
455
+ }
456
+ const decoded = decode(raw);
457
+ const record = validateDecodedMetadata(decoded, indexName);
458
+ if (record.replicationFactor === replicationFactor) {
459
+ return record;
460
+ }
461
+ const updated = encode(isRecord2(decoded) ? { ...decoded, replicationFactor } : decoded, { ignoreUndefined: true });
462
+ if (await coordinator.compareAndSet(key, raw, new Uint8Array(updated))) {
463
+ return { ...record, replicationFactor };
464
+ }
465
+ }
466
+ throw new NarsilError(
467
+ ErrorCodes.ALLOCATION_FAILED,
468
+ `The replication factor of index '${indexName}' lost ${REPLICATION_FACTOR_CAS_ATTEMPTS} compare-and-set attempts`,
469
+ { indexName, attempts: REPLICATION_FACTOR_CAS_ATTEMPTS }
470
+ );
471
+ }
472
+ async function getIndexMetadata(coordinator, indexName) {
473
+ return (await readStoredMetadata(coordinator, indexName))?.record ?? null;
474
+ }
475
+ async function readIndexState(coordinator, indexName) {
476
+ const reject = settingsRejection(indexName);
477
+ let stored = await readStoredMetadata(coordinator, indexName);
478
+ while (stored !== null) {
479
+ const { record } = stored;
480
+ const settings = decodeIndexSettings(stored.rawSettings, indexName);
481
+ const ref = settings?.synonyms;
482
+ if (ref === void 0) return { record, settings, synonyms: [] };
483
+ const encoded = await readSynonymSet(coordinator, record.indexUuid, ref, reject);
484
+ if (encoded !== null) return { record, settings, synonyms: storedSynonymSetRules(encoded, reject) };
485
+ stored = await readStoredMetadata(coordinator, indexName);
486
+ if (stored !== null && referencesSameSet(stored, record.indexUuid, ref, indexName)) {
487
+ return reject(`a chunk of synonym set ${ref.generation} is missing`);
488
+ }
489
+ }
490
+ return null;
491
+ }
492
+ function referencesSameSet(stored, indexUuid, ref, indexName) {
493
+ if (stored.record.indexUuid !== indexUuid) return false;
494
+ return decodeIndexSettings(stored.rawSettings, indexName)?.synonyms?.generation === ref.generation;
495
+ }
496
+ async function getClusterIndexConfig(coordinator, indexName, schema) {
497
+ const state = await readIndexState(coordinator, indexName);
498
+ const config = indexConfigFromSettings(schema, state?.settings, indexName);
499
+ return state === null || state.synonyms.length === 0 ? config : { ...config, synonyms: state.synonyms };
500
+ }
501
+ function decodedMetadataOf(bytes) {
502
+ let decoded;
503
+ try {
504
+ decoded = decode(bytes);
505
+ } catch {
506
+ return null;
507
+ }
508
+ return isRecord2(decoded) ? decoded : null;
509
+ }
510
+ function indexUuidOfStoredMetadata(bytes) {
511
+ const decoded = decodedMetadataOf(bytes);
512
+ return typeof decoded?.indexUuid === "string" ? decoded.indexUuid : null;
513
+ }
514
+ function storedSynonymSetOf(bytes) {
515
+ const decoded = decodedMetadataOf(bytes);
516
+ if (decoded === null || typeof decoded.indexUuid !== "string" || !isRecord2(decoded.settings)) return null;
517
+ const ref = synonymSetRefOrNull(decoded.settings.synonyms);
518
+ return ref === null ? null : { indexUuid: decoded.indexUuid, ref };
519
+ }
520
+
521
+ export { ALLOCATION_CAS_ATTEMPTS, ALLOCATION_RETRY_DELAY_MS, BOOTSTRAP_CAS_ATTEMPTS, DEFAULT_CONTROLLER_CONFIG, DEFAULT_NODE_LIFECYCLE_CONFIG, ESTIMATED_PARTITION_BYTES, HIDDEN_GENERATION_PREFIX, LEADERSHIP_IMBALANCE_THRESHOLD, MAX_RESHARD_CUTOVER_ATTEMPTS, PARTITION_STORES_TIMEOUT_MS, REBALANCE_THRESHOLD, RECOVERY_CAS_ATTEMPTS, RESHARD_CONTROLLER_POLL_INTERVAL_MS, RESHARD_CUTOVER_LAG_ENTRIES, RESHARD_CUTOVER_TIMEOUT_MS, RESHARD_FORWARD_BATCH_DOCUMENTS, RESHARD_POLL_INTERVAL_MS, RESHARD_RECORD_CAS_ATTEMPTS, RESHARD_RECORD_CHECK_INTERVAL_MS, RESHARD_RETRY_DELAY_MS, TEARDOWN_CAS_ATTEMPTS, TEARDOWN_RETRY_DELAY_MS, TEARDOWN_RETRY_ROUNDS, bindEngineWriteBlock, bindIndexReads, bindWriteBlockLookup, deleteSynonymSet, getClusterIndexConfig, getIndexMetadata, indexConfigFromSettings, indexConfigKey, indexReadsOf, indexSettingsOf, indexUuidOfStoredMetadata, isRecord, isValidInteger, metadataWithSynonymSet, putGenerationMetadata, putIndexMetadata, putIndexReplicationFactor, readIndexMetadataSnapshot, readIndexState, storeSynonymSet, storedSynonymSetOf, validateIndexName, withPartitionCount, writeBlockLookupOf };