@delali/narsil 0.3.0 → 0.3.1-next.102

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. package/README.md +9 -8
  2. package/dist/adapters/filesystem.d.ts +1 -1
  3. package/dist/adapters/filesystem.mjs +1 -1
  4. package/dist/adapters/indexeddb.d.ts +1 -1
  5. package/dist/adapters/indexeddb.mjs +1 -1
  6. package/dist/adapters/memory.d.ts +1 -1
  7. package/dist/{adapters-79OIa4xm.d.ts → adapters-BQFQ4PRz.d.ts} +27 -1
  8. package/dist/browser/chunk-BDOSMEEP.mjs +493 -0
  9. package/dist/browser/chunk-EJDUFWSO.mjs +602 -0
  10. package/dist/browser/chunk-HOZW3PKP.mjs +9 -0
  11. package/dist/browser/chunk-JOVZ6HMT.mjs +4358 -0
  12. package/dist/browser/chunk-JPTQZWJQ.mjs +10 -0
  13. package/dist/browser/chunk-KPJMDPHG.mjs +24558 -0
  14. package/dist/browser/chunk-PGIM3HIM.mjs +2743 -0
  15. package/dist/browser/chunk-RHG4O77A.mjs +206 -0
  16. package/dist/browser/chunk-RIJSLY2I.mjs +360 -0
  17. package/dist/browser/durability-wiring-SRUYT3VV.mjs +3875 -0
  18. package/dist/browser/module-CBB2ETYP.mjs +3111 -0
  19. package/dist/browser/release-76MXRPGV.mjs +33 -0
  20. package/dist/browser/replace-RURXTGVG.mjs +57 -0
  21. package/dist/browser/restore-queue-EEZQO6RF.mjs +35 -0
  22. package/dist/browser/runtime-7ZAHCHWR.mjs +557 -0
  23. package/dist/browser/snapshot-OAAA5UJ7.mjs +1210 -0
  24. package/dist/browser/write-7D6VIK6O.mjs +208 -0
  25. package/dist/{capabilities-DSsOGz6F.d.ts → capabilities-ByQRMOx-.d.ts} +68 -7
  26. package/dist/chunk-2RIRGOUL.mjs +113 -0
  27. package/dist/chunk-37CD3OKS.mjs +87 -0
  28. package/dist/chunk-44W3CBNG.mjs +7 -0
  29. package/dist/chunk-46D6JQIG.mjs +426 -0
  30. package/dist/{chunk-VMRR3DQU.mjs → chunk-5JNA5SKS.mjs} +546 -35
  31. package/dist/chunk-747OHOUA.mjs +11 -0
  32. package/dist/chunk-75XIEF6H.mjs +126 -0
  33. package/dist/chunk-7X5JST5L.mjs +114 -0
  34. package/dist/chunk-BOFBLJFZ.mjs +209 -0
  35. package/dist/{chunk-USKFU7J5.mjs → chunk-BWJVNZPR.mjs} +8 -1
  36. package/dist/chunk-C7SLA6CM.mjs +19 -0
  37. package/dist/{chunk-POWOTUKJ.mjs → chunk-CQUMNIIG.mjs} +3 -6
  38. package/dist/chunk-CVXLLEQ3.mjs +1011 -0
  39. package/dist/chunk-DJHHF2L5.mjs +17 -0
  40. package/dist/chunk-E74YLAIG.mjs +24 -0
  41. package/dist/{chunk-UXNWQZST.mjs → chunk-EMRMGYRE.mjs} +236 -18
  42. package/dist/chunk-EWZLIYHV.mjs +57 -0
  43. package/dist/chunk-GBEEG5AN.mjs +35 -0
  44. package/dist/chunk-GES7U2PM.mjs +68 -0
  45. package/dist/chunk-HJSN7UYG.mjs +10 -0
  46. package/dist/chunk-HLPRHOSO.mjs +17 -0
  47. package/dist/chunk-HMFNFVP7.mjs +14 -0
  48. package/dist/chunk-J5QP52HA.mjs +1751 -0
  49. package/dist/{chunk-YNVIUGF5.mjs → chunk-JZVARCRZ.mjs} +3 -28
  50. package/dist/chunk-KOKM4PQN.mjs +52 -0
  51. package/dist/{chunk-B2TFROFX.mjs → chunk-KWTW6VTG.mjs} +465 -250
  52. package/dist/chunk-KZ3NX2P7.mjs +21 -0
  53. package/dist/chunk-L5ORED7V.mjs +25 -0
  54. package/dist/{chunk-THYRHHO2.mjs → chunk-LGAMKHA2.mjs} +2009 -2351
  55. package/dist/chunk-LKOB5COA.mjs +117 -0
  56. package/dist/chunk-LPQOEU5M.mjs +96 -0
  57. package/dist/{chunk-3LHGNRYE.mjs → chunk-LXWDQLTW.mjs} +7 -2
  58. package/dist/{chunk-L6TNPLAR.mjs → chunk-OEYI3KRB.mjs} +270 -108
  59. package/dist/chunk-OM362BRO.mjs +948 -0
  60. package/dist/chunk-OZDFA4WC.mjs +67 -0
  61. package/dist/chunk-P36MUFEJ.mjs +10 -0
  62. package/dist/chunk-PC2G76VW.mjs +2540 -0
  63. package/dist/{chunk-GCM3JFVA.mjs → chunk-PNEYIHA4.mjs} +11 -2
  64. package/dist/chunk-PNFKUCG3.mjs +1130 -0
  65. package/dist/chunk-Q6XWVUZX.mjs +44 -0
  66. package/dist/chunk-R27EMH5M.mjs +617 -0
  67. package/dist/{chunk-YMZCMH5M.mjs → chunk-R6A773WK.mjs} +19 -24
  68. package/dist/chunk-RF7PLG43.mjs +106 -0
  69. package/dist/chunk-RFBLQLLZ.mjs +16 -0
  70. package/dist/{chunk-Y3L5MDRV.mjs → chunk-RFVUDM5E.mjs} +17 -2
  71. package/dist/chunk-S7FL6UAH.mjs +521 -0
  72. package/dist/chunk-S7GNDRPS.mjs +651 -0
  73. package/dist/chunk-SP3NM3ZG.mjs +3537 -0
  74. package/dist/chunk-U476IXKQ.mjs +10 -0
  75. package/dist/chunk-UA3AUGNV.mjs +462 -0
  76. package/dist/chunk-UTG5YNXV.mjs +507 -0
  77. package/dist/chunk-VI4R6W25.mjs +8484 -0
  78. package/dist/chunk-WQBCBKY3.mjs +8 -0
  79. package/dist/{chunk-24RXHMEL.mjs → chunk-WVCOCKBU.mjs} +3794 -7570
  80. package/dist/chunk-X5CV2E2B.mjs +1 -0
  81. package/dist/chunk-XADZPPZI.mjs +614 -0
  82. package/dist/chunk-YEEQHT4V.mjs +212 -0
  83. package/dist/chunk-YTJCGKM4.mjs +56 -0
  84. package/dist/chunk-YYQIODMF.mjs +275 -0
  85. package/dist/chunk-ZI34TVOA.mjs +144 -0
  86. package/dist/{chunk-OOSIDHA3.mjs → chunk-ZKWYWW4Y.mjs} +1 -1
  87. package/dist/chunk-ZM7KU4YW.mjs +35 -0
  88. package/dist/{client-BQK6Tf6A.d.ts → client-BOStAkc5.d.ts} +160 -37
  89. package/dist/client.d.ts +11 -10
  90. package/dist/client.mjs +209 -45
  91. package/dist/{config-DmXULPa-.d.ts → config-Cjvzbs_j.d.ts} +51 -3
  92. package/dist/config-H6SGzbPH.d.ts +421 -0
  93. package/dist/direct-executor-ZD64PPEZ.mjs +24 -0
  94. package/dist/distribution/coordinator/etcd.d.ts +79 -18
  95. package/dist/distribution/coordinator/etcd.mjs +553 -109
  96. package/dist/distribution/coordinator/in-memory.d.ts +4 -3
  97. package/dist/distribution/coordinator/in-memory.mjs +22 -3
  98. package/dist/distribution/transport/grpc.d.ts +2 -2
  99. package/dist/distribution/transport/grpc.mjs +79 -14
  100. package/dist/distribution/transport/in-memory.d.ts +2 -2
  101. package/dist/distribution/transport/in-memory.mjs +12 -7
  102. package/dist/distribution/transport/tcp.d.ts +2 -2
  103. package/dist/distribution/transport/tcp.mjs +50 -28
  104. package/dist/distribution.d.ts +144 -23
  105. package/dist/distribution.mjs +14479 -8173
  106. package/dist/durability-wiring-FEUBR5BT.mjs +3330 -0
  107. package/dist/embeddings/openai.d.ts +2 -2
  108. package/dist/embeddings/openai.mjs +22 -133
  109. package/dist/{engine-CIOyFozK.d.ts → engine-rC0vjjY-.d.ts} +165 -141
  110. package/dist/{memory-yFaVn2xZ.d.ts → events-DqOIg5xa.d.ts} +645 -136
  111. package/dist/{types-DAK7mnUZ.d.ts → index-Bkdt8nDw.d.ts} +182 -45
  112. package/dist/index.browser.mjs +4 -32180
  113. package/dist/index.d.ts +91 -20
  114. package/dist/index.mjs +33 -16
  115. package/dist/invalidation/broadcast-channel.d.ts +1 -1
  116. package/dist/invalidation/filesystem.d.ts +1 -1
  117. package/dist/invalidation/filesystem.mjs +2 -1
  118. package/dist/invalidation/noop.d.ts +1 -1
  119. package/dist/module-3S65MVCM.mjs +2151 -0
  120. package/dist/partial-words-DTIHO8G4.d.ts +19 -0
  121. package/dist/partial-words.d.ts +22 -0
  122. package/dist/partial-words.mjs +17 -0
  123. package/dist/pattern-Cledccbl.d.ts +18 -0
  124. package/dist/pattern.d.ts +22 -0
  125. package/dist/pattern.mjs +12 -0
  126. package/dist/persistence/durability/checkpoint-worker.d.ts +119 -72
  127. package/dist/persistence/durability/checkpoint-worker.mjs +51 -18
  128. package/dist/react.d.ts +279 -183
  129. package/dist/react.mjs +201 -48
  130. package/dist/release-BH3DKW3T.mjs +33 -0
  131. package/dist/replace-CEO5KJJ3.mjs +57 -0
  132. package/dist/rerank-K7C4WZFG.mjs +60 -0
  133. package/dist/rerankers/http.d.ts +50 -0
  134. package/dist/rerankers/http.mjs +104 -0
  135. package/dist/restore-queue-KQHA7ZV2.mjs +35 -0
  136. package/dist/runtime-6Y4ZYXOG.mjs +177 -0
  137. package/dist/{schema-d5Afu03A.d.ts → schema-4W_7tk1f.d.ts} +81 -11
  138. package/dist/server.d.ts +13 -12
  139. package/dist/server.mjs +611 -80
  140. package/dist/snapshot-QGZPODM5.mjs +799 -0
  141. package/dist/synonyms-CpOAHPa-.d.ts +52 -0
  142. package/dist/synonyms.d.ts +22 -0
  143. package/dist/synonyms.mjs +12 -0
  144. package/dist/{tasks-CH2Dod8w.d.ts → tasks-CwPTz-tP.d.ts} +10 -3
  145. package/dist/thread-N3IUGUS3.mjs +697 -0
  146. package/dist/{types-BQDlIvfS.d.ts → types-2qoitTXg.d.ts} +1 -1
  147. package/dist/{types-D5OwhtsQ.d.ts → types-Cz5okLml.d.ts} +2 -2
  148. package/dist/vector/search-worker.d.ts +46 -4
  149. package/dist/vector/search-worker.mjs +21 -11
  150. package/dist/workers/entry.mjs +9 -6
  151. package/dist/write-U5RI7MAX.mjs +241 -0
  152. package/package.json +55 -9
  153. package/dist/chunk-3L4U6EJQ.mjs +0 -49
  154. package/dist/chunk-IFBKLBGB.mjs +0 -7
  155. package/dist/chunk-ORBLD3EL.mjs +0 -6
  156. package/dist/chunk-U7KYW4BA.mjs +0 -135
  157. package/dist/chunk-ZJ4CSAB2.mjs +0 -13
  158. package/dist/chunk-ZNZBXMP5.mjs +0 -2117
  159. package/dist/chunk-ZZT3EAUC.mjs +0 -11168
  160. package/dist/config-BhldZgaK.d.ts +0 -194
  161. package/dist/direct-executor-ALAVE7FC.mjs +0 -12
  162. package/dist/serialization/crc32-worker.d.ts +0 -20
  163. package/dist/serialization/crc32-worker.mjs +0 -62
  164. package/dist/simd-DW5AtJ6B.d.ts +0 -41
  165. package/dist/thread-DYHEQRVY.mjs +0 -402
@@ -1,4 +1,4 @@
1
- import { E as EmbeddingAdapter } from '../adapters-79OIa4xm.js';
1
+ import { E as EmbeddingAdapter } from '../adapters-BQFQ4PRz.js';
2
2
 
3
3
  /**
4
4
  * How {@link createOpenAIEmbedding} reaches an embeddings endpoint.
@@ -29,7 +29,7 @@ interface OpenAIEmbeddingConfig {
29
29
  *
30
30
  * @param config - Endpoint, credentials, model, and vector length.
31
31
  * @returns An adapter you pass as `embedding`, or register by name with
32
- * {@link Narsil.registerEmbeddingAdapter}.
32
+ * {@link AdapterRegistrationOperations.registerEmbeddingAdapter}.
33
33
  * @throws A `NarsilError` with `EMBEDDING_CONFIG_INVALID` when the base URL,
34
34
  * the model, or the dimension count is missing or out of range.
35
35
  *
@@ -1,64 +1,11 @@
1
- import { NarsilError, ErrorCodes } from '../chunk-USKFU7J5.mjs';
1
+ import { resolveApiKey, buildSignal, postJsonWithRetry } from '../chunk-75XIEF6H.mjs';
2
+ import { NarsilError, ErrorCodes } from '../chunk-BWJVNZPR.mjs';
2
3
 
3
4
  // src/embeddings/constants.ts
4
- var RETRYABLE_STATUS_CODES = /* @__PURE__ */ new Set([429, 500, 502, 503, 504]);
5
- var MAX_BACKOFF_MS = 3e4;
6
- var BASE_BACKOFF_MS = 1e3;
7
- var MAX_JITTER_MS = 1e3;
8
5
  var DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
9
6
  var MAX_INPUTS_PER_REQUEST = 2048;
10
7
 
11
8
  // src/embeddings/openai.ts
12
- function isRetryableStatus(status) {
13
- return RETRYABLE_STATUS_CODES.has(status);
14
- }
15
- function computeBackoffMs(attempt, retryAfterSeconds) {
16
- if (retryAfterSeconds !== null && retryAfterSeconds > 0) {
17
- return Math.min(retryAfterSeconds * 1e3, MAX_BACKOFF_MS);
18
- }
19
- const exponential = Math.min(BASE_BACKOFF_MS * 2 ** attempt, MAX_BACKOFF_MS);
20
- const jitter = Math.random() * MAX_JITTER_MS;
21
- return exponential + jitter;
22
- }
23
- function parseRetryAfterHeader(headers) {
24
- const raw = headers.get("retry-after");
25
- if (raw === null) return null;
26
- const seconds = Number(raw);
27
- if (Number.isFinite(seconds) && seconds > 0) return seconds;
28
- return null;
29
- }
30
- async function resolveApiKey(apiKey) {
31
- const resolved = typeof apiKey === "function" ? await apiKey() : apiKey;
32
- if (typeof resolved !== "string" || resolved.trim().length === 0) {
33
- throw new NarsilError(
34
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
35
- "OpenAI embedding adapter apiKey resolved to an empty or non-string value"
36
- );
37
- }
38
- return resolved;
39
- }
40
- function buildSignal(callerSignal, timeoutMs) {
41
- const timeoutSignal = AbortSignal.timeout(timeoutMs);
42
- if (callerSignal === void 0) return timeoutSignal;
43
- return AbortSignal.any([callerSignal, timeoutSignal]);
44
- }
45
- async function sleep(ms, signal) {
46
- return new Promise((resolve, reject) => {
47
- if (signal?.aborted) {
48
- reject(signal.reason);
49
- return;
50
- }
51
- const onAbort = () => {
52
- clearTimeout(timer);
53
- reject(signal?.reason);
54
- };
55
- const timer = setTimeout(() => {
56
- signal?.removeEventListener("abort", onAbort);
57
- resolve();
58
- }, ms);
59
- signal?.addEventListener("abort", onAbort, { once: true });
60
- });
61
- }
62
9
  function parseEmbeddingItems(data) {
63
10
  if (!Array.isArray(data)) {
64
11
  throw new NarsilError(ErrorCodes.EMBEDDING_FAILED, 'OpenAI response missing "data" array');
@@ -85,80 +32,6 @@ function parseEmbeddingItems(data) {
85
32
  }
86
33
  return items;
87
34
  }
88
- async function parseErrorBody(response) {
89
- try {
90
- const body = await response.json();
91
- const errorType = body?.error?.type ?? "unknown";
92
- const errorMessage = body?.error?.message ?? `HTTP ${response.status}`;
93
- return { type: errorType, message: errorMessage };
94
- } catch {
95
- return { type: "unknown", message: `HTTP ${response.status}` };
96
- }
97
- }
98
- async function executeWithRetry(url, requestBody, apiKey, maxRetries, signal) {
99
- let lastError = null;
100
- for (let attempt = 0; attempt <= maxRetries; attempt++) {
101
- let response;
102
- try {
103
- response = await fetch(url, {
104
- method: "POST",
105
- headers: {
106
- Authorization: `Bearer ${apiKey}`,
107
- "Content-Type": "application/json"
108
- },
109
- body: requestBody,
110
- signal
111
- });
112
- } catch (err) {
113
- if (err instanceof DOMException || err instanceof Error && err.name === "AbortError") {
114
- throw err;
115
- }
116
- lastError = err instanceof Error ? err : new Error(String(err));
117
- if (attempt < maxRetries) {
118
- await sleep(computeBackoffMs(attempt, null), signal);
119
- continue;
120
- }
121
- throw new NarsilError(
122
- ErrorCodes.EMBEDDING_FAILED,
123
- `OpenAI request failed after ${maxRetries + 1} attempts: network error`,
124
- {
125
- cause: lastError.message
126
- }
127
- );
128
- }
129
- if (response.ok) {
130
- let parsed;
131
- try {
132
- parsed = await response.json();
133
- } catch {
134
- lastError = new Error("invalid JSON in a 2xx response body");
135
- if (attempt < maxRetries) {
136
- await sleep(computeBackoffMs(attempt, null), signal);
137
- continue;
138
- }
139
- throw new NarsilError(ErrorCodes.EMBEDDING_FAILED, "OpenAI returned invalid JSON in response body");
140
- }
141
- if (parsed === null || typeof parsed !== "object") {
142
- throw new NarsilError(ErrorCodes.EMBEDDING_FAILED, "OpenAI returned non-object JSON response");
143
- }
144
- return parsed;
145
- }
146
- const retryAfter = parseRetryAfterHeader(response.headers);
147
- const errorInfo = await parseErrorBody(response);
148
- if (!isRetryableStatus(response.status) || attempt >= maxRetries) {
149
- throw new NarsilError(
150
- ErrorCodes.EMBEDDING_FAILED,
151
- `OpenAI API error (${response.status}): [${errorInfo.type}] ${errorInfo.message}`,
152
- { status: response.status, errorType: errorInfo.type }
153
- );
154
- }
155
- lastError = new Error(`HTTP ${response.status}: ${errorInfo.type}`);
156
- await sleep(computeBackoffMs(attempt, retryAfter), signal);
157
- }
158
- throw new NarsilError(ErrorCodes.EMBEDDING_FAILED, "OpenAI request failed: exhausted all retries", {
159
- cause: lastError?.message
160
- });
161
- }
162
35
  function createOpenAIEmbedding(config) {
163
36
  if (!config.baseUrl || config.baseUrl.trim().length === 0) {
164
37
  throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, "OpenAI embedding adapter requires a non-empty baseUrl");
@@ -203,10 +76,18 @@ function createOpenAIEmbedding(config) {
203
76
  return dimensionCount;
204
77
  },
205
78
  async embed(input, _purpose, signal) {
206
- const apiKey = await resolveApiKey(config.apiKey);
79
+ const apiKey = await resolveApiKey(config.apiKey, "OpenAI embedding adapter", ErrorCodes.EMBEDDING_CONFIG_INVALID);
207
80
  const body = JSON.stringify({ input: [input], model, dimensions: dimensionCount });
208
81
  const combinedSignal = buildSignal(signal, timeoutMs);
209
- const result = await executeWithRetry(url, body, apiKey, maxRetries, combinedSignal);
82
+ const result = await postJsonWithRetry({
83
+ url,
84
+ body,
85
+ apiKey,
86
+ maxRetries,
87
+ signal: combinedSignal,
88
+ service: "OpenAI",
89
+ failureCode: ErrorCodes.EMBEDDING_FAILED
90
+ });
210
91
  const items = parseEmbeddingItems(result.data);
211
92
  if (items.length === 0) {
212
93
  throw new NarsilError(
@@ -218,11 +99,19 @@ function createOpenAIEmbedding(config) {
218
99
  },
219
100
  async embedBatch(inputs, _purpose, signal) {
220
101
  if (inputs.length === 0) return [];
221
- const apiKey = await resolveApiKey(config.apiKey);
102
+ const apiKey = await resolveApiKey(config.apiKey, "OpenAI embedding adapter", ErrorCodes.EMBEDDING_CONFIG_INVALID);
222
103
  const combinedSignal = buildSignal(signal, timeoutMs);
223
104
  const executeChunk = async (chunk) => {
224
105
  const body = JSON.stringify({ input: chunk, model, dimensions: dimensionCount });
225
- const result = await executeWithRetry(url, body, apiKey, maxRetries, combinedSignal);
106
+ const result = await postJsonWithRetry({
107
+ url,
108
+ body,
109
+ apiKey,
110
+ maxRetries,
111
+ signal: combinedSignal,
112
+ service: "OpenAI",
113
+ failureCode: ErrorCodes.EMBEDDING_FAILED
114
+ });
226
115
  const items = parseEmbeddingItems(result.data);
227
116
  if (items.length !== chunk.length) {
228
117
  throw new NarsilError(
@@ -1,122 +1,46 @@
1
- import { E as EmbeddingAdapter } from './adapters-79OIa4xm.js';
2
- import { B as BatchResult, I as IndexInfo, f as IndexStats, P as PartitionStatsResult, L as ListParams, g as ListResult, Q as QueryParams, e as QueryResult, h as PreflightResult, j as SuggestParams, k as SuggestResult, M as MemoryStats, V as VectorMaintenanceResult } from './memory-yFaVn2xZ.js';
3
- import { A as AnyDocument, a as InsertOptions, W as WriteOptions, I as IndexConfig, P as PartitionConfig } from './schema-d5Afu03A.js';
1
+ import { E as EmbeddingAdapter, R as RerankAdapter } from './adapters-BQFQ4PRz.js';
2
+ import { B as BatchResult, I as IndexInfo, k as IndexStats, l as PartitionStatsResult, L as ListParams, g as ListResult, Q as QueryParams, e as QueryResult, P as PreflightResult, h as SuggestParams, j as SuggestResult, D as DocumentImportResult, M as MemoryStats, V as VectorMaintenanceResult, m as NarsilEventMap } from './events-DqOIg5xa.js';
3
+ import { A as AnyDocument, a as InsertOptions, W as WriteOptions, I as IndexConfig, P as PartitionConfig } from './schema-4W_7tk1f.js';
4
+ import { a as SynonymRule } from './synonyms-CpOAHPa-.js';
4
5
 
5
6
  /**
6
- * Every event the engine emits, with the payload each one carries.
7
+ * Registers the adapters that indexes and queries reach by name, which is
8
+ * what lets a binding outlive a restart and reach a worker thread or another
9
+ * node.
7
10
  *
8
- * {@link Narsil.on} takes a key from this map and hands your listener the
9
- * matching payload. These events report work the engine does on its own, away
10
- * from the call that triggered it, so a failure here never rejects a promise
11
- * you are holding.
11
+ * @public
12
+ */
13
+ interface AdapterRegistrationOperations {
14
+ /**
15
+ * Registers an embedding adapter under a name that an index config can
16
+ * give and that durability metadata keeps. Registering a name again binds
17
+ * every index that gives that name to the new adapter.
18
+ *
19
+ * @param name - The name that an index config gives in `embedding.adapter`.
20
+ * @param adapter - The adapter that embeds documents and queries.
21
+ */
22
+ registerEmbeddingAdapter(name: string, adapter: EmbeddingAdapter): void;
23
+ /**
24
+ * Registers a rerank adapter under a name that a query gives in
25
+ * `rerank.adapter`. Registering a name again sends every later query that
26
+ * gives that name to the new adapter. The engine throws
27
+ * `RERANK_CONFIG_INVALID` for an empty name and for an adapter without a
28
+ * `rerank` function.
29
+ *
30
+ * @param name - The name that a query gives in `rerank.adapter`.
31
+ * @param adapter - The adapter that scores the top hits of a query.
32
+ */
33
+ registerRerankAdapter(name: string, adapter: RerankAdapter): void;
34
+ }
35
+
36
+ /**
37
+ * The bytes that {@link Narsil.restore} and {@link Narsil.importDocumentExport}
38
+ * accept, as one array, a stream of chunks, or an async iterable of chunks,
39
+ * such as a Node.js file stream.
12
40
  *
13
41
  * @public
14
42
  */
15
- type NarsilEventMap = {
16
- /**
17
- * A worker thread died. The pool drops it and fails the requests that it was
18
- * serving with `WORKER_CRASHED`, and the engine answers each of those queries
19
- * again on the main thread, so the caller receives a result and no error.
20
- * The remaining workers keep answering, because each holds a full worker
21
- * copy of every promoted index. After a delay the
22
- * engine spawns a replacement, loads every copy onto it, and puts it back
23
- * into rotation. Once no worker is left, queries fall back to the main
24
- * thread, which holds every document, and the next request after the delay
25
- * starts a new pool. The delay starts at one second and doubles up to a
26
- * minute while replacements keep failing.
27
- */
28
- workerCrash: {
29
- /** This worker died. */
30
- workerId: number;
31
- /** That worker was holding these indexes. */
32
- indexNames: string[];
33
- /** This ended the worker. */
34
- error: Error;
35
- };
36
- /** An index gained worker copies, whether it reached the copy threshold or loaded its copies again after an idle spell. */
37
- workerPromote: {
38
- /** The pool holds this many workers. */
39
- workerCount: number;
40
- /** The index gained its copies for this reason, such as the copy threshold it reached. */
41
- reason: string;
42
- };
43
- /** An index could not gain worker copies. It keeps answering on the main thread. */
44
- workerPromoteFailure: {
45
- /** The engine tried to load the copies for this reason. */
46
- reason: string;
47
- /** This stopped it. */
48
- error: Error;
49
- /** This turns true when the engine will try again on a later threshold. */
50
- retryable: boolean;
51
- };
52
- /** An index finished spreading its documents across a new partition count. */
53
- partitionRebalance: {
54
- /** This index was rebalanced. */
55
- indexName: string;
56
- /** The index held this many partitions before. */
57
- oldCount: number;
58
- /** The index holds this many partitions now. */
59
- newCount: number;
60
- };
61
- /** A partition passed its watermark, so the index is close to needing another one. */
62
- partitionWatermark: {
63
- /** This index crossed the mark. */
64
- indexName: string;
65
- /** The index holds this many documents. */
66
- documentCount: number;
67
- /** The index can hold this many across its current partitions. */
68
- capacity: number;
69
- /** The index holds this many partitions. */
70
- partitionCount: number;
71
- };
72
- /**
73
- * The process uses nine tenths of its heap, measured during a write or a
74
- * load, so the next large index may end the process with an out-of-memory
75
- * error. Where `--max-old-space-size` or `--max-old-space-size-percentage`
76
- * sets a limit, on the command line or in `NODE_OPTIONS`, the engine measures
77
- * the used bytes against that figure and `heapLimit` reports it. Where
78
- * neither flag sets one, the engine compares the used bytes with the headroom
79
- * that V8 reports, which runs about 192 MB above the true ceiling, so a
80
- * default heap below about 2 GB may end before this event fires. The engine
81
- * emits the event once per crossing and arms it again once the headroom
82
- * recovers to two tenths. Raise the limit with
83
- * `--max-old-space-size-percentage` or `--max-old-space-size`, or close an
84
- * idle index; see {@link ProcessMemoryReport.heapLimit}.
85
- */
86
- heapPressure: {
87
- /** The heap reaches that point during a write to this index or a load of it. */
88
- indexName: string;
89
- /** The process uses this much heap, in bytes. */
90
- heapUsed: number;
91
- /** The heap may grow to this many bytes. */
92
- heapLimit: number;
93
- /** The engine estimates that the index holds this many bytes. */
94
- estimatedMemoryBytes: number;
95
- };
96
- /** A rebuild is bringing an index's terms up to its language module's current analysis. */
97
- analysisRebuild: {
98
- /** This index is being rebuilt. */
99
- indexName: string;
100
- /** The rebuild stands here. */
101
- status: 'started' | 'completed' | 'failed';
102
- /** The rebuild has covered this many partitions so far. */
103
- partitionsRebuilt: number;
104
- /** The rebuild covers this many partitions in total. */
105
- partitionCount: number;
106
- /** This stopped the rebuild, and only a `failed` status carries it. */
107
- error?: Error;
108
- };
109
- /** The write-ahead log or a checkpoint failed, so writes since the last checkpoint are at risk. */
110
- durabilityError: {
111
- /** The durability layer threw this. */
112
- error: Error;
113
- };
114
- /** The invalidation channel failed, so this instance may be reading partitions another instance has changed. */
115
- invalidationError: {
116
- /** The adapter threw this. */
117
- error: Error;
118
- };
119
- };
43
+ type ByteSource = Uint8Array | ReadableStream<Uint8Array> | AsyncIterable<Uint8Array>;
120
44
 
121
45
  /**
122
46
  * Opens and closes durable indexes without deleting them.
@@ -140,6 +64,42 @@ interface IndexLifecycleOperations {
140
64
  close(indexName: string): Promise<void>;
141
65
  }
142
66
 
67
+ /**
68
+ * Reads and replaces the synonym set of an index, which
69
+ * {@link IndexConfig.synonyms} sets when the index is created.
70
+ *
71
+ * @public
72
+ */
73
+ interface SynonymOperations {
74
+ /**
75
+ * Replaces the synonym set of an index, so that every search that starts
76
+ * after the promise resolves expands its words with the new set.
77
+ *
78
+ * The engine applies the set without rebuilding the index, because a
79
+ * synonym set changes no stored token. It saves the set with the index, so
80
+ * the set stays in place after a restart, and every snapshot of the index
81
+ * includes it. Where the index has copies on worker threads, the promise
82
+ * resolves only once every copy uses the new set. Pass an empty list to
83
+ * remove the set.
84
+ *
85
+ * @param indexName - The index whose set the engine replaces.
86
+ * @param synonyms - The new set, in the shape that {@link SynonymRule}
87
+ * describes.
88
+ * @throws A `NarsilError` with `CONFIG_INVALID` when a rule breaks that
89
+ * shape, when a word analyses to no token or to more than one, or when the
90
+ * set passes 100,000 rules, 500,000 words, or 4,194,304 bytes of words in
91
+ * UTF-8. The index then keeps the set that it held before the call.
92
+ */
93
+ updateSynonyms(indexName: string, synonyms: SynonymRule[]): Promise<void>;
94
+ /**
95
+ * Returns the synonym set of an index with every word as the caller wrote
96
+ * it, or an empty list when the index holds no set.
97
+ *
98
+ * @param indexName - The index whose set the engine returns.
99
+ */
100
+ getSynonyms(indexName: string): Promise<SynonymRule[]>;
101
+ }
102
+
143
103
  /**
144
104
  * Adds, replaces, and removes documents, and waits for the worker copies to
145
105
  * catch up with those writes.
@@ -249,13 +209,14 @@ interface DocumentWriteOperations {
249
209
  *
250
210
  * One instance holds every index you create, so an application usually keeps a
251
211
  * single engine for its lifetime and reaches each index by name. Build one
252
- * with {@link createNarsil}. The write methods come from
253
- * {@link DocumentWriteOperations} and the open and close methods from
254
- * {@link IndexLifecycleOperations}.
212
+ * with {@link createNarsil}. Its write, open and close, synonym, and adapter
213
+ * registration methods come from {@link DocumentWriteOperations},
214
+ * {@link IndexLifecycleOperations}, {@link SynonymOperations}, and
215
+ * {@link AdapterRegistrationOperations}.
255
216
  *
256
217
  * @public
257
218
  */
258
- interface Narsil extends IndexLifecycleOperations, DocumentWriteOperations {
219
+ interface Narsil extends IndexLifecycleOperations, DocumentWriteOperations, SynonymOperations, AdapterRegistrationOperations {
259
220
  /**
260
221
  * Creates an index you can insert documents into and query.
261
222
  *
@@ -268,9 +229,6 @@ interface Narsil extends IndexLifecycleOperations, DocumentWriteOperations {
268
229
  * and scoring settings.
269
230
  */
270
231
  createIndex(name: string, config: IndexConfig): Promise<void>;
271
- /** Registers an adapter under a name that index configs can reference and
272
- * durability metadata can persist; re-registering rebinds referencing indexes. */
273
- registerEmbeddingAdapter(name: string, adapter: EmbeddingAdapter): void;
274
232
  /**
275
233
  * Removes an index and everything it holds, including whatever persistence
276
234
  * wrote for it. The documents do not come back.
@@ -423,28 +381,94 @@ interface Narsil extends IndexLifecycleOperations, DocumentWriteOperations {
423
381
  */
424
382
  rebuildAnalysis(indexName: string): Promise<void>;
425
383
  /**
426
- * Serialises a whole index into the portable `.nrsl` format, which any
427
- * Narsil implementation reads.
384
+ * Returns a stream of the index's snapshot archive. The archive is a pax
385
+ * file containing the index's configuration and the files of one
386
+ * checkpoint, and any implementation of the Narsil format can restore it.
387
+ *
388
+ * With write-ahead log durability, the engine takes a checkpoint before it
389
+ * starts the stream, so the archive contains every write that the engine
390
+ * acknowledged before the call. The engine keeps that checkpoint's files on
391
+ * disk until the stream ends or you cancel it. Without write-ahead log
392
+ * durability, the engine builds the archive's files from the index in
393
+ * memory.
394
+ *
395
+ * @param indexName - The index to stream.
396
+ * @returns A stream of the archive's bytes, which you can write to a file or
397
+ * send over a network.
398
+ */
399
+ snapshot(indexName: string): Promise<ReadableStream<Uint8Array>>;
400
+ /**
401
+ * Creates an index from a snapshot under the name that you pass, replacing
402
+ * any index of that name.
403
+ *
404
+ * The engine accepts the archive that {@link Narsil.snapshot} streams and
405
+ * the `.nrsl` file of release 0.3. It checks each archive file against its
406
+ * manifest and loads the restored index beside the existing one, which
407
+ * answers requests until the swap. A failed restore leaves the existing index
408
+ * serving, and after a crash the engine reopens either the old or the new one.
409
+ *
410
+ * @param indexName - The name that the restored index takes.
411
+ * @param data - The snapshot's bytes, as an array, a stream, or async chunks.
412
+ * @throws A `NarsilError` with `PERSISTENCE_LOAD_FAILED` when the archive is
413
+ * malformed or a file's length differs from the manifest. The engine raises
414
+ * `PERSISTENCE_CRC_MISMATCH` when a file's checksum differs from its
415
+ * contents, and `ENVELOPE_VERSION_MISMATCH` when a file has a format version
416
+ * other than the ones that this release implements. It raises
417
+ * `CONFIG_INVALID` when the archive's configuration is invalid,
418
+ * `DOC_VALIDATION_FAILED` when the data is empty or the release 0.3 file is
419
+ * invalid, `RESTORE_MEMORY_EXCEEDED` when the restored index needs more
420
+ * memory than the process has free, and `WORKER_TIMEOUT` when a
421
+ * worker still holds the old index 30 seconds after the engine asks for it.
422
+ */
423
+ restore(indexName: string, data: ByteSource): Promise<void>;
424
+ /**
425
+ * Returns a stream of the index's document export. The export is JSON Lines
426
+ * text whose format stays the same in every release, so you can use it to
427
+ * move an index across releases or between implementations.
428
+ *
429
+ * The first line is a header containing the format name, the format
430
+ * version, and the index configuration. Each later line contains one
431
+ * document, with its ID under `id` and the value of each vector field as an
432
+ * array of numbers. With write-ahead log durability, the engine exports every
433
+ * document from one checkpoint; otherwise, it exports them from the index in
434
+ * memory at one moment.
435
+ *
436
+ * JSON has no type for a date, for raw bytes, or for an infinite number or
437
+ * `NaN`, so the engine writes each such value as a tag. Each tag is an
438
+ * object with one key that begins with `$`, such as
439
+ * `{"$date":[1700000000,123000000]}`. The engine also wraps a stored object
440
+ * whose only key begins with `$` in an `$object` tag, because the import
441
+ * decodes every other object of that shape as a tag.
442
+ *
443
+ * @param indexName - The index to export.
444
+ * @returns A stream of the export's UTF-8 bytes.
445
+ */
446
+ exportDocuments(indexName: string): Promise<ReadableStream<Uint8Array>>;
447
+ /**
448
+ * Imports a document export into the named index. Where the name is free,
449
+ * the engine creates the index from the export's configuration.
450
+ *
451
+ * The engine writes the documents in batches and replaces any document
452
+ * whose ID the index already contains. Because it replaces existing
453
+ * documents, you can run an interrupted import again to make the index equal
454
+ * to the export. When a line is invalid or the engine rejects its document,
455
+ * the engine records the refusal and continues with the next line.
456
+ *
457
+ * The engine converts each tag that {@link Narsil.exportDocuments} writes
458
+ * into the value that the tag stands for. When an object has one key that
459
+ * begins with `$` and that key names no tag, or when a tag holds an invalid
460
+ * value, the engine refuses the line.
428
461
  *
429
- * @param indexName - The index to serialise.
430
- * @returns The file's bytes, ready to store or send.
431
- */
432
- snapshot(indexName: string): Promise<Uint8Array>;
433
- /**
434
- * Loads an index from snapshot bytes, replacing whatever the name held.
435
- *
436
- * The engine reads the `.nrsl` envelope that {@link Narsil.snapshot} writes.
437
- *
438
- * @param indexName - The name the restored index takes.
439
- * @param data - Bytes that {@link Narsil.snapshot} produced.
440
- * @throws A `NarsilError` with `ENVELOPE_INVALID_MAGIC` when the bytes are
441
- * shorter than an envelope header or open with anything other than the
442
- * `NRSL` magic bytes, `ENVELOPE_VERSION_MISMATCH` when the envelope carries a
443
- * format version other than the index snapshot format,
444
- * `PERSISTENCE_CRC_MISMATCH` when the payload fails its checksum, and
445
- * `DOC_VALIDATION_FAILED` when the envelope holds no valid index snapshot.
462
+ * @param indexName - The index that receives the documents.
463
+ * @param data - The export's bytes, as one array, a stream, or an async
464
+ * iterable of chunks.
465
+ * @returns The number of imported documents, the number of refused lines,
466
+ * and the 1,000 refusals with the lowest line numbers.
467
+ * @throws A `NarsilError` with `CONFIG_INVALID` when the first line is
468
+ * anything other than a version 1 header, or when the named index exists
469
+ * with a different schema.
446
470
  */
447
- restore(indexName: string, data: Uint8Array): Promise<void>;
471
+ importDocumentExport(indexName: string, data: ByteSource): Promise<DocumentImportResult>;
448
472
  /**
449
473
  * Writes an index's current state to the durability directory, so a restart
450
474
  * replays less of the log.
@@ -550,4 +574,4 @@ interface Narsil extends IndexLifecycleOperations, DocumentWriteOperations {
550
574
  shutdown(): Promise<void>;
551
575
  }
552
576
 
553
- export type { DocumentWriteOperations as D, IndexLifecycleOperations as I, NarsilEventMap as N, Narsil as a };
577
+ export type { AdapterRegistrationOperations as A, ByteSource as B, DocumentWriteOperations as D, IndexLifecycleOperations as I, Narsil as N, SynonymOperations as S };