@delali/narsil 0.3.0 → 0.3.1-next.101
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -8
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/{adapters-79OIa4xm.d.ts → adapters-BQFQ4PRz.d.ts} +27 -1
- package/dist/browser/chunk-BDOSMEEP.mjs +493 -0
- package/dist/browser/chunk-EJDUFWSO.mjs +602 -0
- package/dist/browser/chunk-HOZW3PKP.mjs +9 -0
- package/dist/browser/chunk-JOVZ6HMT.mjs +4358 -0
- package/dist/browser/chunk-JPTQZWJQ.mjs +10 -0
- package/dist/browser/chunk-KPJMDPHG.mjs +24558 -0
- package/dist/browser/chunk-PGIM3HIM.mjs +2743 -0
- package/dist/browser/chunk-RHG4O77A.mjs +206 -0
- package/dist/browser/chunk-RIJSLY2I.mjs +360 -0
- package/dist/browser/durability-wiring-SRUYT3VV.mjs +3875 -0
- package/dist/browser/module-CBB2ETYP.mjs +3111 -0
- package/dist/browser/release-76MXRPGV.mjs +33 -0
- package/dist/browser/replace-RURXTGVG.mjs +57 -0
- package/dist/browser/restore-queue-EEZQO6RF.mjs +35 -0
- package/dist/browser/runtime-7ZAHCHWR.mjs +557 -0
- package/dist/browser/snapshot-OAAA5UJ7.mjs +1210 -0
- package/dist/browser/write-7D6VIK6O.mjs +208 -0
- package/dist/{capabilities-DSsOGz6F.d.ts → capabilities-DiGlB6Sr.d.ts} +68 -7
- package/dist/chunk-2RIRGOUL.mjs +113 -0
- package/dist/chunk-37CD3OKS.mjs +87 -0
- package/dist/chunk-44W3CBNG.mjs +7 -0
- package/dist/chunk-46D6JQIG.mjs +426 -0
- package/dist/{chunk-VMRR3DQU.mjs → chunk-5JNA5SKS.mjs} +546 -35
- package/dist/chunk-747OHOUA.mjs +11 -0
- package/dist/chunk-75XIEF6H.mjs +126 -0
- package/dist/chunk-7X5JST5L.mjs +114 -0
- package/dist/chunk-BOFBLJFZ.mjs +209 -0
- package/dist/{chunk-USKFU7J5.mjs → chunk-BWJVNZPR.mjs} +8 -1
- package/dist/chunk-C7SLA6CM.mjs +19 -0
- package/dist/{chunk-POWOTUKJ.mjs → chunk-CQUMNIIG.mjs} +3 -6
- package/dist/chunk-CVXLLEQ3.mjs +1011 -0
- package/dist/chunk-DJHHF2L5.mjs +17 -0
- package/dist/chunk-E74YLAIG.mjs +24 -0
- package/dist/{chunk-UXNWQZST.mjs → chunk-EMRMGYRE.mjs} +236 -18
- package/dist/chunk-EWZLIYHV.mjs +57 -0
- package/dist/chunk-GBEEG5AN.mjs +35 -0
- package/dist/chunk-GES7U2PM.mjs +68 -0
- package/dist/chunk-HJSN7UYG.mjs +10 -0
- package/dist/chunk-HLPRHOSO.mjs +17 -0
- package/dist/chunk-HMFNFVP7.mjs +14 -0
- package/dist/chunk-J5QP52HA.mjs +1751 -0
- package/dist/{chunk-YNVIUGF5.mjs → chunk-JZVARCRZ.mjs} +3 -28
- package/dist/chunk-KOKM4PQN.mjs +52 -0
- package/dist/{chunk-B2TFROFX.mjs → chunk-KWTW6VTG.mjs} +465 -250
- package/dist/chunk-KZ3NX2P7.mjs +21 -0
- package/dist/chunk-L5ORED7V.mjs +25 -0
- package/dist/{chunk-THYRHHO2.mjs → chunk-LGAMKHA2.mjs} +2009 -2351
- package/dist/chunk-LKOB5COA.mjs +117 -0
- package/dist/chunk-LPQOEU5M.mjs +96 -0
- package/dist/{chunk-3LHGNRYE.mjs → chunk-LXWDQLTW.mjs} +7 -2
- package/dist/{chunk-L6TNPLAR.mjs → chunk-OEYI3KRB.mjs} +270 -108
- package/dist/chunk-OM362BRO.mjs +948 -0
- package/dist/chunk-OZDFA4WC.mjs +67 -0
- package/dist/chunk-P36MUFEJ.mjs +10 -0
- package/dist/chunk-PC2G76VW.mjs +2540 -0
- package/dist/{chunk-GCM3JFVA.mjs → chunk-PNEYIHA4.mjs} +11 -2
- package/dist/chunk-PNFKUCG3.mjs +1130 -0
- package/dist/chunk-Q6XWVUZX.mjs +44 -0
- package/dist/chunk-R27EMH5M.mjs +617 -0
- package/dist/{chunk-YMZCMH5M.mjs → chunk-R6A773WK.mjs} +19 -24
- package/dist/chunk-RF7PLG43.mjs +106 -0
- package/dist/chunk-RFBLQLLZ.mjs +16 -0
- package/dist/{chunk-Y3L5MDRV.mjs → chunk-RFVUDM5E.mjs} +17 -2
- package/dist/chunk-S7FL6UAH.mjs +521 -0
- package/dist/chunk-S7GNDRPS.mjs +651 -0
- package/dist/chunk-SP3NM3ZG.mjs +3537 -0
- package/dist/chunk-U476IXKQ.mjs +10 -0
- package/dist/chunk-UA3AUGNV.mjs +462 -0
- package/dist/chunk-UTG5YNXV.mjs +507 -0
- package/dist/chunk-VI4R6W25.mjs +8484 -0
- package/dist/chunk-WQBCBKY3.mjs +8 -0
- package/dist/{chunk-24RXHMEL.mjs → chunk-WVCOCKBU.mjs} +3794 -7570
- package/dist/chunk-X5CV2E2B.mjs +1 -0
- package/dist/chunk-XADZPPZI.mjs +614 -0
- package/dist/chunk-YEEQHT4V.mjs +212 -0
- package/dist/chunk-YTJCGKM4.mjs +56 -0
- package/dist/chunk-YYQIODMF.mjs +275 -0
- package/dist/chunk-ZI34TVOA.mjs +144 -0
- package/dist/{chunk-OOSIDHA3.mjs → chunk-ZKWYWW4Y.mjs} +1 -1
- package/dist/chunk-ZM7KU4YW.mjs +35 -0
- package/dist/{client-BQK6Tf6A.d.ts → client-BOStAkc5.d.ts} +160 -37
- package/dist/client.d.ts +11 -10
- package/dist/client.mjs +209 -45
- package/dist/{config-DmXULPa-.d.ts → config-Cjvzbs_j.d.ts} +51 -3
- package/dist/config-DffQlmRq.d.ts +421 -0
- package/dist/direct-executor-ZD64PPEZ.mjs +24 -0
- package/dist/distribution/coordinator/etcd.d.ts +79 -18
- package/dist/distribution/coordinator/etcd.mjs +553 -109
- package/dist/distribution/coordinator/in-memory.d.ts +4 -3
- package/dist/distribution/coordinator/in-memory.mjs +22 -3
- package/dist/distribution/transport/grpc.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +79 -14
- package/dist/distribution/transport/in-memory.d.ts +2 -2
- package/dist/distribution/transport/in-memory.mjs +12 -7
- package/dist/distribution/transport/tcp.d.ts +2 -2
- package/dist/distribution/transport/tcp.mjs +50 -28
- package/dist/distribution.d.ts +144 -23
- package/dist/distribution.mjs +14479 -8173
- package/dist/durability-wiring-FEUBR5BT.mjs +3330 -0
- package/dist/embeddings/openai.d.ts +2 -2
- package/dist/embeddings/openai.mjs +22 -133
- package/dist/{engine-CIOyFozK.d.ts → engine-rC0vjjY-.d.ts} +165 -141
- package/dist/{memory-yFaVn2xZ.d.ts → events-DqOIg5xa.d.ts} +645 -136
- package/dist/{types-DAK7mnUZ.d.ts → index-Bkdt8nDw.d.ts} +182 -45
- package/dist/index.browser.mjs +4 -32180
- package/dist/index.d.ts +91 -20
- package/dist/index.mjs +33 -16
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +2 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/module-3S65MVCM.mjs +2151 -0
- package/dist/partial-words-DTIHO8G4.d.ts +19 -0
- package/dist/partial-words.d.ts +22 -0
- package/dist/partial-words.mjs +17 -0
- package/dist/pattern-Cledccbl.d.ts +18 -0
- package/dist/pattern.d.ts +22 -0
- package/dist/pattern.mjs +12 -0
- package/dist/persistence/durability/checkpoint-worker.d.ts +119 -72
- package/dist/persistence/durability/checkpoint-worker.mjs +51 -18
- package/dist/react.d.ts +279 -183
- package/dist/react.mjs +201 -48
- package/dist/release-BH3DKW3T.mjs +33 -0
- package/dist/replace-CEO5KJJ3.mjs +57 -0
- package/dist/rerank-K7C4WZFG.mjs +60 -0
- package/dist/rerankers/http.d.ts +50 -0
- package/dist/rerankers/http.mjs +104 -0
- package/dist/restore-queue-KQHA7ZV2.mjs +35 -0
- package/dist/runtime-6Y4ZYXOG.mjs +177 -0
- package/dist/{schema-d5Afu03A.d.ts → schema-4W_7tk1f.d.ts} +81 -11
- package/dist/server.d.ts +13 -12
- package/dist/server.mjs +611 -80
- package/dist/snapshot-QGZPODM5.mjs +799 -0
- package/dist/synonyms-CpOAHPa-.d.ts +52 -0
- package/dist/synonyms.d.ts +22 -0
- package/dist/synonyms.mjs +12 -0
- package/dist/{tasks-CH2Dod8w.d.ts → tasks-CwPTz-tP.d.ts} +10 -3
- package/dist/thread-N3IUGUS3.mjs +697 -0
- package/dist/{types-BQDlIvfS.d.ts → types-2qoitTXg.d.ts} +1 -1
- package/dist/{types-D5OwhtsQ.d.ts → types-Cz5okLml.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +46 -4
- package/dist/vector/search-worker.mjs +21 -11
- package/dist/workers/entry.mjs +9 -6
- package/dist/write-U5RI7MAX.mjs +241 -0
- package/package.json +55 -9
- package/dist/chunk-3L4U6EJQ.mjs +0 -49
- package/dist/chunk-IFBKLBGB.mjs +0 -7
- package/dist/chunk-ORBLD3EL.mjs +0 -6
- package/dist/chunk-U7KYW4BA.mjs +0 -135
- package/dist/chunk-ZJ4CSAB2.mjs +0 -13
- package/dist/chunk-ZNZBXMP5.mjs +0 -2117
- package/dist/chunk-ZZT3EAUC.mjs +0 -11168
- package/dist/config-BhldZgaK.d.ts +0 -194
- package/dist/direct-executor-ALAVE7FC.mjs +0 -12
- package/dist/serialization/crc32-worker.d.ts +0 -20
- package/dist/serialization/crc32-worker.mjs +0 -62
- package/dist/simd-DW5AtJ6B.d.ts +0 -41
- package/dist/thread-DYHEQRVY.mjs +0 -402
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { E as EmbeddingAdapter } from '../adapters-
|
|
1
|
+
import { E as EmbeddingAdapter } from '../adapters-BQFQ4PRz.js';
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* How {@link createOpenAIEmbedding} reaches an embeddings endpoint.
|
|
@@ -29,7 +29,7 @@ interface OpenAIEmbeddingConfig {
|
|
|
29
29
|
*
|
|
30
30
|
* @param config - Endpoint, credentials, model, and vector length.
|
|
31
31
|
* @returns An adapter you pass as `embedding`, or register by name with
|
|
32
|
-
* {@link
|
|
32
|
+
* {@link AdapterRegistrationOperations.registerEmbeddingAdapter}.
|
|
33
33
|
* @throws A `NarsilError` with `EMBEDDING_CONFIG_INVALID` when the base URL,
|
|
34
34
|
* the model, or the dimension count is missing or out of range.
|
|
35
35
|
*
|
|
@@ -1,64 +1,11 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { resolveApiKey, buildSignal, postJsonWithRetry } from '../chunk-75XIEF6H.mjs';
|
|
2
|
+
import { NarsilError, ErrorCodes } from '../chunk-BWJVNZPR.mjs';
|
|
2
3
|
|
|
3
4
|
// src/embeddings/constants.ts
|
|
4
|
-
var RETRYABLE_STATUS_CODES = /* @__PURE__ */ new Set([429, 500, 502, 503, 504]);
|
|
5
|
-
var MAX_BACKOFF_MS = 3e4;
|
|
6
|
-
var BASE_BACKOFF_MS = 1e3;
|
|
7
|
-
var MAX_JITTER_MS = 1e3;
|
|
8
5
|
var DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
|
|
9
6
|
var MAX_INPUTS_PER_REQUEST = 2048;
|
|
10
7
|
|
|
11
8
|
// src/embeddings/openai.ts
|
|
12
|
-
function isRetryableStatus(status) {
|
|
13
|
-
return RETRYABLE_STATUS_CODES.has(status);
|
|
14
|
-
}
|
|
15
|
-
function computeBackoffMs(attempt, retryAfterSeconds) {
|
|
16
|
-
if (retryAfterSeconds !== null && retryAfterSeconds > 0) {
|
|
17
|
-
return Math.min(retryAfterSeconds * 1e3, MAX_BACKOFF_MS);
|
|
18
|
-
}
|
|
19
|
-
const exponential = Math.min(BASE_BACKOFF_MS * 2 ** attempt, MAX_BACKOFF_MS);
|
|
20
|
-
const jitter = Math.random() * MAX_JITTER_MS;
|
|
21
|
-
return exponential + jitter;
|
|
22
|
-
}
|
|
23
|
-
function parseRetryAfterHeader(headers) {
|
|
24
|
-
const raw = headers.get("retry-after");
|
|
25
|
-
if (raw === null) return null;
|
|
26
|
-
const seconds = Number(raw);
|
|
27
|
-
if (Number.isFinite(seconds) && seconds > 0) return seconds;
|
|
28
|
-
return null;
|
|
29
|
-
}
|
|
30
|
-
async function resolveApiKey(apiKey) {
|
|
31
|
-
const resolved = typeof apiKey === "function" ? await apiKey() : apiKey;
|
|
32
|
-
if (typeof resolved !== "string" || resolved.trim().length === 0) {
|
|
33
|
-
throw new NarsilError(
|
|
34
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
35
|
-
"OpenAI embedding adapter apiKey resolved to an empty or non-string value"
|
|
36
|
-
);
|
|
37
|
-
}
|
|
38
|
-
return resolved;
|
|
39
|
-
}
|
|
40
|
-
function buildSignal(callerSignal, timeoutMs) {
|
|
41
|
-
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
|
42
|
-
if (callerSignal === void 0) return timeoutSignal;
|
|
43
|
-
return AbortSignal.any([callerSignal, timeoutSignal]);
|
|
44
|
-
}
|
|
45
|
-
async function sleep(ms, signal) {
|
|
46
|
-
return new Promise((resolve, reject) => {
|
|
47
|
-
if (signal?.aborted) {
|
|
48
|
-
reject(signal.reason);
|
|
49
|
-
return;
|
|
50
|
-
}
|
|
51
|
-
const onAbort = () => {
|
|
52
|
-
clearTimeout(timer);
|
|
53
|
-
reject(signal?.reason);
|
|
54
|
-
};
|
|
55
|
-
const timer = setTimeout(() => {
|
|
56
|
-
signal?.removeEventListener("abort", onAbort);
|
|
57
|
-
resolve();
|
|
58
|
-
}, ms);
|
|
59
|
-
signal?.addEventListener("abort", onAbort, { once: true });
|
|
60
|
-
});
|
|
61
|
-
}
|
|
62
9
|
function parseEmbeddingItems(data) {
|
|
63
10
|
if (!Array.isArray(data)) {
|
|
64
11
|
throw new NarsilError(ErrorCodes.EMBEDDING_FAILED, 'OpenAI response missing "data" array');
|
|
@@ -85,80 +32,6 @@ function parseEmbeddingItems(data) {
|
|
|
85
32
|
}
|
|
86
33
|
return items;
|
|
87
34
|
}
|
|
88
|
-
async function parseErrorBody(response) {
|
|
89
|
-
try {
|
|
90
|
-
const body = await response.json();
|
|
91
|
-
const errorType = body?.error?.type ?? "unknown";
|
|
92
|
-
const errorMessage = body?.error?.message ?? `HTTP ${response.status}`;
|
|
93
|
-
return { type: errorType, message: errorMessage };
|
|
94
|
-
} catch {
|
|
95
|
-
return { type: "unknown", message: `HTTP ${response.status}` };
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
async function executeWithRetry(url, requestBody, apiKey, maxRetries, signal) {
|
|
99
|
-
let lastError = null;
|
|
100
|
-
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
101
|
-
let response;
|
|
102
|
-
try {
|
|
103
|
-
response = await fetch(url, {
|
|
104
|
-
method: "POST",
|
|
105
|
-
headers: {
|
|
106
|
-
Authorization: `Bearer ${apiKey}`,
|
|
107
|
-
"Content-Type": "application/json"
|
|
108
|
-
},
|
|
109
|
-
body: requestBody,
|
|
110
|
-
signal
|
|
111
|
-
});
|
|
112
|
-
} catch (err) {
|
|
113
|
-
if (err instanceof DOMException || err instanceof Error && err.name === "AbortError") {
|
|
114
|
-
throw err;
|
|
115
|
-
}
|
|
116
|
-
lastError = err instanceof Error ? err : new Error(String(err));
|
|
117
|
-
if (attempt < maxRetries) {
|
|
118
|
-
await sleep(computeBackoffMs(attempt, null), signal);
|
|
119
|
-
continue;
|
|
120
|
-
}
|
|
121
|
-
throw new NarsilError(
|
|
122
|
-
ErrorCodes.EMBEDDING_FAILED,
|
|
123
|
-
`OpenAI request failed after ${maxRetries + 1} attempts: network error`,
|
|
124
|
-
{
|
|
125
|
-
cause: lastError.message
|
|
126
|
-
}
|
|
127
|
-
);
|
|
128
|
-
}
|
|
129
|
-
if (response.ok) {
|
|
130
|
-
let parsed;
|
|
131
|
-
try {
|
|
132
|
-
parsed = await response.json();
|
|
133
|
-
} catch {
|
|
134
|
-
lastError = new Error("invalid JSON in a 2xx response body");
|
|
135
|
-
if (attempt < maxRetries) {
|
|
136
|
-
await sleep(computeBackoffMs(attempt, null), signal);
|
|
137
|
-
continue;
|
|
138
|
-
}
|
|
139
|
-
throw new NarsilError(ErrorCodes.EMBEDDING_FAILED, "OpenAI returned invalid JSON in response body");
|
|
140
|
-
}
|
|
141
|
-
if (parsed === null || typeof parsed !== "object") {
|
|
142
|
-
throw new NarsilError(ErrorCodes.EMBEDDING_FAILED, "OpenAI returned non-object JSON response");
|
|
143
|
-
}
|
|
144
|
-
return parsed;
|
|
145
|
-
}
|
|
146
|
-
const retryAfter = parseRetryAfterHeader(response.headers);
|
|
147
|
-
const errorInfo = await parseErrorBody(response);
|
|
148
|
-
if (!isRetryableStatus(response.status) || attempt >= maxRetries) {
|
|
149
|
-
throw new NarsilError(
|
|
150
|
-
ErrorCodes.EMBEDDING_FAILED,
|
|
151
|
-
`OpenAI API error (${response.status}): [${errorInfo.type}] ${errorInfo.message}`,
|
|
152
|
-
{ status: response.status, errorType: errorInfo.type }
|
|
153
|
-
);
|
|
154
|
-
}
|
|
155
|
-
lastError = new Error(`HTTP ${response.status}: ${errorInfo.type}`);
|
|
156
|
-
await sleep(computeBackoffMs(attempt, retryAfter), signal);
|
|
157
|
-
}
|
|
158
|
-
throw new NarsilError(ErrorCodes.EMBEDDING_FAILED, "OpenAI request failed: exhausted all retries", {
|
|
159
|
-
cause: lastError?.message
|
|
160
|
-
});
|
|
161
|
-
}
|
|
162
35
|
function createOpenAIEmbedding(config) {
|
|
163
36
|
if (!config.baseUrl || config.baseUrl.trim().length === 0) {
|
|
164
37
|
throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, "OpenAI embedding adapter requires a non-empty baseUrl");
|
|
@@ -203,10 +76,18 @@ function createOpenAIEmbedding(config) {
|
|
|
203
76
|
return dimensionCount;
|
|
204
77
|
},
|
|
205
78
|
async embed(input, _purpose, signal) {
|
|
206
|
-
const apiKey = await resolveApiKey(config.apiKey);
|
|
79
|
+
const apiKey = await resolveApiKey(config.apiKey, "OpenAI embedding adapter", ErrorCodes.EMBEDDING_CONFIG_INVALID);
|
|
207
80
|
const body = JSON.stringify({ input: [input], model, dimensions: dimensionCount });
|
|
208
81
|
const combinedSignal = buildSignal(signal, timeoutMs);
|
|
209
|
-
const result = await
|
|
82
|
+
const result = await postJsonWithRetry({
|
|
83
|
+
url,
|
|
84
|
+
body,
|
|
85
|
+
apiKey,
|
|
86
|
+
maxRetries,
|
|
87
|
+
signal: combinedSignal,
|
|
88
|
+
service: "OpenAI",
|
|
89
|
+
failureCode: ErrorCodes.EMBEDDING_FAILED
|
|
90
|
+
});
|
|
210
91
|
const items = parseEmbeddingItems(result.data);
|
|
211
92
|
if (items.length === 0) {
|
|
212
93
|
throw new NarsilError(
|
|
@@ -218,11 +99,19 @@ function createOpenAIEmbedding(config) {
|
|
|
218
99
|
},
|
|
219
100
|
async embedBatch(inputs, _purpose, signal) {
|
|
220
101
|
if (inputs.length === 0) return [];
|
|
221
|
-
const apiKey = await resolveApiKey(config.apiKey);
|
|
102
|
+
const apiKey = await resolveApiKey(config.apiKey, "OpenAI embedding adapter", ErrorCodes.EMBEDDING_CONFIG_INVALID);
|
|
222
103
|
const combinedSignal = buildSignal(signal, timeoutMs);
|
|
223
104
|
const executeChunk = async (chunk) => {
|
|
224
105
|
const body = JSON.stringify({ input: chunk, model, dimensions: dimensionCount });
|
|
225
|
-
const result = await
|
|
106
|
+
const result = await postJsonWithRetry({
|
|
107
|
+
url,
|
|
108
|
+
body,
|
|
109
|
+
apiKey,
|
|
110
|
+
maxRetries,
|
|
111
|
+
signal: combinedSignal,
|
|
112
|
+
service: "OpenAI",
|
|
113
|
+
failureCode: ErrorCodes.EMBEDDING_FAILED
|
|
114
|
+
});
|
|
226
115
|
const items = parseEmbeddingItems(result.data);
|
|
227
116
|
if (items.length !== chunk.length) {
|
|
228
117
|
throw new NarsilError(
|
|
@@ -1,122 +1,46 @@
|
|
|
1
|
-
import { E as EmbeddingAdapter } from './adapters-
|
|
2
|
-
import { B as BatchResult, I as IndexInfo,
|
|
3
|
-
import { A as AnyDocument, a as InsertOptions, W as WriteOptions, I as IndexConfig, P as PartitionConfig } from './schema-
|
|
1
|
+
import { E as EmbeddingAdapter, R as RerankAdapter } from './adapters-BQFQ4PRz.js';
|
|
2
|
+
import { B as BatchResult, I as IndexInfo, k as IndexStats, l as PartitionStatsResult, L as ListParams, g as ListResult, Q as QueryParams, e as QueryResult, P as PreflightResult, h as SuggestParams, j as SuggestResult, D as DocumentImportResult, M as MemoryStats, V as VectorMaintenanceResult, m as NarsilEventMap } from './events-DqOIg5xa.js';
|
|
3
|
+
import { A as AnyDocument, a as InsertOptions, W as WriteOptions, I as IndexConfig, P as PartitionConfig } from './schema-4W_7tk1f.js';
|
|
4
|
+
import { a as SynonymRule } from './synonyms-CpOAHPa-.js';
|
|
4
5
|
|
|
5
6
|
/**
|
|
6
|
-
*
|
|
7
|
+
* Registers the adapters that indexes and queries reach by name, which is
|
|
8
|
+
* what lets a binding outlive a restart and reach a worker thread or another
|
|
9
|
+
* node.
|
|
7
10
|
*
|
|
8
|
-
*
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
11
|
+
* @public
|
|
12
|
+
*/
|
|
13
|
+
interface AdapterRegistrationOperations {
|
|
14
|
+
/**
|
|
15
|
+
* Registers an embedding adapter under a name that an index config can
|
|
16
|
+
* give and that durability metadata keeps. Registering a name again binds
|
|
17
|
+
* every index that gives that name to the new adapter.
|
|
18
|
+
*
|
|
19
|
+
* @param name - The name that an index config gives in `embedding.adapter`.
|
|
20
|
+
* @param adapter - The adapter that embeds documents and queries.
|
|
21
|
+
*/
|
|
22
|
+
registerEmbeddingAdapter(name: string, adapter: EmbeddingAdapter): void;
|
|
23
|
+
/**
|
|
24
|
+
* Registers a rerank adapter under a name that a query gives in
|
|
25
|
+
* `rerank.adapter`. Registering a name again sends every later query that
|
|
26
|
+
* gives that name to the new adapter. The engine throws
|
|
27
|
+
* `RERANK_CONFIG_INVALID` for an empty name and for an adapter without a
|
|
28
|
+
* `rerank` function.
|
|
29
|
+
*
|
|
30
|
+
* @param name - The name that a query gives in `rerank.adapter`.
|
|
31
|
+
* @param adapter - The adapter that scores the top hits of a query.
|
|
32
|
+
*/
|
|
33
|
+
registerRerankAdapter(name: string, adapter: RerankAdapter): void;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* The bytes that {@link Narsil.restore} and {@link Narsil.importDocumentExport}
|
|
38
|
+
* accept, as one array, a stream of chunks, or an async iterable of chunks,
|
|
39
|
+
* such as a Node.js file stream.
|
|
12
40
|
*
|
|
13
41
|
* @public
|
|
14
42
|
*/
|
|
15
|
-
type
|
|
16
|
-
/**
|
|
17
|
-
* A worker thread died. The pool drops it and fails the requests that it was
|
|
18
|
-
* serving with `WORKER_CRASHED`, and the engine answers each of those queries
|
|
19
|
-
* again on the main thread, so the caller receives a result and no error.
|
|
20
|
-
* The remaining workers keep answering, because each holds a full worker
|
|
21
|
-
* copy of every promoted index. After a delay the
|
|
22
|
-
* engine spawns a replacement, loads every copy onto it, and puts it back
|
|
23
|
-
* into rotation. Once no worker is left, queries fall back to the main
|
|
24
|
-
* thread, which holds every document, and the next request after the delay
|
|
25
|
-
* starts a new pool. The delay starts at one second and doubles up to a
|
|
26
|
-
* minute while replacements keep failing.
|
|
27
|
-
*/
|
|
28
|
-
workerCrash: {
|
|
29
|
-
/** This worker died. */
|
|
30
|
-
workerId: number;
|
|
31
|
-
/** That worker was holding these indexes. */
|
|
32
|
-
indexNames: string[];
|
|
33
|
-
/** This ended the worker. */
|
|
34
|
-
error: Error;
|
|
35
|
-
};
|
|
36
|
-
/** An index gained worker copies, whether it reached the copy threshold or loaded its copies again after an idle spell. */
|
|
37
|
-
workerPromote: {
|
|
38
|
-
/** The pool holds this many workers. */
|
|
39
|
-
workerCount: number;
|
|
40
|
-
/** The index gained its copies for this reason, such as the copy threshold it reached. */
|
|
41
|
-
reason: string;
|
|
42
|
-
};
|
|
43
|
-
/** An index could not gain worker copies. It keeps answering on the main thread. */
|
|
44
|
-
workerPromoteFailure: {
|
|
45
|
-
/** The engine tried to load the copies for this reason. */
|
|
46
|
-
reason: string;
|
|
47
|
-
/** This stopped it. */
|
|
48
|
-
error: Error;
|
|
49
|
-
/** This turns true when the engine will try again on a later threshold. */
|
|
50
|
-
retryable: boolean;
|
|
51
|
-
};
|
|
52
|
-
/** An index finished spreading its documents across a new partition count. */
|
|
53
|
-
partitionRebalance: {
|
|
54
|
-
/** This index was rebalanced. */
|
|
55
|
-
indexName: string;
|
|
56
|
-
/** The index held this many partitions before. */
|
|
57
|
-
oldCount: number;
|
|
58
|
-
/** The index holds this many partitions now. */
|
|
59
|
-
newCount: number;
|
|
60
|
-
};
|
|
61
|
-
/** A partition passed its watermark, so the index is close to needing another one. */
|
|
62
|
-
partitionWatermark: {
|
|
63
|
-
/** This index crossed the mark. */
|
|
64
|
-
indexName: string;
|
|
65
|
-
/** The index holds this many documents. */
|
|
66
|
-
documentCount: number;
|
|
67
|
-
/** The index can hold this many across its current partitions. */
|
|
68
|
-
capacity: number;
|
|
69
|
-
/** The index holds this many partitions. */
|
|
70
|
-
partitionCount: number;
|
|
71
|
-
};
|
|
72
|
-
/**
|
|
73
|
-
* The process uses nine tenths of its heap, measured during a write or a
|
|
74
|
-
* load, so the next large index may end the process with an out-of-memory
|
|
75
|
-
* error. Where `--max-old-space-size` or `--max-old-space-size-percentage`
|
|
76
|
-
* sets a limit, on the command line or in `NODE_OPTIONS`, the engine measures
|
|
77
|
-
* the used bytes against that figure and `heapLimit` reports it. Where
|
|
78
|
-
* neither flag sets one, the engine compares the used bytes with the headroom
|
|
79
|
-
* that V8 reports, which runs about 192 MB above the true ceiling, so a
|
|
80
|
-
* default heap below about 2 GB may end before this event fires. The engine
|
|
81
|
-
* emits the event once per crossing and arms it again once the headroom
|
|
82
|
-
* recovers to two tenths. Raise the limit with
|
|
83
|
-
* `--max-old-space-size-percentage` or `--max-old-space-size`, or close an
|
|
84
|
-
* idle index; see {@link ProcessMemoryReport.heapLimit}.
|
|
85
|
-
*/
|
|
86
|
-
heapPressure: {
|
|
87
|
-
/** The heap reaches that point during a write to this index or a load of it. */
|
|
88
|
-
indexName: string;
|
|
89
|
-
/** The process uses this much heap, in bytes. */
|
|
90
|
-
heapUsed: number;
|
|
91
|
-
/** The heap may grow to this many bytes. */
|
|
92
|
-
heapLimit: number;
|
|
93
|
-
/** The engine estimates that the index holds this many bytes. */
|
|
94
|
-
estimatedMemoryBytes: number;
|
|
95
|
-
};
|
|
96
|
-
/** A rebuild is bringing an index's terms up to its language module's current analysis. */
|
|
97
|
-
analysisRebuild: {
|
|
98
|
-
/** This index is being rebuilt. */
|
|
99
|
-
indexName: string;
|
|
100
|
-
/** The rebuild stands here. */
|
|
101
|
-
status: 'started' | 'completed' | 'failed';
|
|
102
|
-
/** The rebuild has covered this many partitions so far. */
|
|
103
|
-
partitionsRebuilt: number;
|
|
104
|
-
/** The rebuild covers this many partitions in total. */
|
|
105
|
-
partitionCount: number;
|
|
106
|
-
/** This stopped the rebuild, and only a `failed` status carries it. */
|
|
107
|
-
error?: Error;
|
|
108
|
-
};
|
|
109
|
-
/** The write-ahead log or a checkpoint failed, so writes since the last checkpoint are at risk. */
|
|
110
|
-
durabilityError: {
|
|
111
|
-
/** The durability layer threw this. */
|
|
112
|
-
error: Error;
|
|
113
|
-
};
|
|
114
|
-
/** The invalidation channel failed, so this instance may be reading partitions another instance has changed. */
|
|
115
|
-
invalidationError: {
|
|
116
|
-
/** The adapter threw this. */
|
|
117
|
-
error: Error;
|
|
118
|
-
};
|
|
119
|
-
};
|
|
43
|
+
type ByteSource = Uint8Array | ReadableStream<Uint8Array> | AsyncIterable<Uint8Array>;
|
|
120
44
|
|
|
121
45
|
/**
|
|
122
46
|
* Opens and closes durable indexes without deleting them.
|
|
@@ -140,6 +64,42 @@ interface IndexLifecycleOperations {
|
|
|
140
64
|
close(indexName: string): Promise<void>;
|
|
141
65
|
}
|
|
142
66
|
|
|
67
|
+
/**
|
|
68
|
+
* Reads and replaces the synonym set of an index, which
|
|
69
|
+
* {@link IndexConfig.synonyms} sets when the index is created.
|
|
70
|
+
*
|
|
71
|
+
* @public
|
|
72
|
+
*/
|
|
73
|
+
interface SynonymOperations {
|
|
74
|
+
/**
|
|
75
|
+
* Replaces the synonym set of an index, so that every search that starts
|
|
76
|
+
* after the promise resolves expands its words with the new set.
|
|
77
|
+
*
|
|
78
|
+
* The engine applies the set without rebuilding the index, because a
|
|
79
|
+
* synonym set changes no stored token. It saves the set with the index, so
|
|
80
|
+
* the set stays in place after a restart, and every snapshot of the index
|
|
81
|
+
* includes it. Where the index has copies on worker threads, the promise
|
|
82
|
+
* resolves only once every copy uses the new set. Pass an empty list to
|
|
83
|
+
* remove the set.
|
|
84
|
+
*
|
|
85
|
+
* @param indexName - The index whose set the engine replaces.
|
|
86
|
+
* @param synonyms - The new set, in the shape that {@link SynonymRule}
|
|
87
|
+
* describes.
|
|
88
|
+
* @throws A `NarsilError` with `CONFIG_INVALID` when a rule breaks that
|
|
89
|
+
* shape, when a word analyses to no token or to more than one, or when the
|
|
90
|
+
* set passes 100,000 rules, 500,000 words, or 4,194,304 bytes of words in
|
|
91
|
+
* UTF-8. The index then keeps the set that it held before the call.
|
|
92
|
+
*/
|
|
93
|
+
updateSynonyms(indexName: string, synonyms: SynonymRule[]): Promise<void>;
|
|
94
|
+
/**
|
|
95
|
+
* Returns the synonym set of an index with every word as the caller wrote
|
|
96
|
+
* it, or an empty list when the index holds no set.
|
|
97
|
+
*
|
|
98
|
+
* @param indexName - The index whose set the engine returns.
|
|
99
|
+
*/
|
|
100
|
+
getSynonyms(indexName: string): Promise<SynonymRule[]>;
|
|
101
|
+
}
|
|
102
|
+
|
|
143
103
|
/**
|
|
144
104
|
* Adds, replaces, and removes documents, and waits for the worker copies to
|
|
145
105
|
* catch up with those writes.
|
|
@@ -249,13 +209,14 @@ interface DocumentWriteOperations {
|
|
|
249
209
|
*
|
|
250
210
|
* One instance holds every index you create, so an application usually keeps a
|
|
251
211
|
* single engine for its lifetime and reaches each index by name. Build one
|
|
252
|
-
* with {@link createNarsil}.
|
|
253
|
-
* {@link DocumentWriteOperations}
|
|
254
|
-
* {@link IndexLifecycleOperations}
|
|
212
|
+
* with {@link createNarsil}. Its write, open and close, synonym, and adapter
|
|
213
|
+
* registration methods come from {@link DocumentWriteOperations},
|
|
214
|
+
* {@link IndexLifecycleOperations}, {@link SynonymOperations}, and
|
|
215
|
+
* {@link AdapterRegistrationOperations}.
|
|
255
216
|
*
|
|
256
217
|
* @public
|
|
257
218
|
*/
|
|
258
|
-
interface Narsil extends IndexLifecycleOperations, DocumentWriteOperations {
|
|
219
|
+
interface Narsil extends IndexLifecycleOperations, DocumentWriteOperations, SynonymOperations, AdapterRegistrationOperations {
|
|
259
220
|
/**
|
|
260
221
|
* Creates an index you can insert documents into and query.
|
|
261
222
|
*
|
|
@@ -268,9 +229,6 @@ interface Narsil extends IndexLifecycleOperations, DocumentWriteOperations {
|
|
|
268
229
|
* and scoring settings.
|
|
269
230
|
*/
|
|
270
231
|
createIndex(name: string, config: IndexConfig): Promise<void>;
|
|
271
|
-
/** Registers an adapter under a name that index configs can reference and
|
|
272
|
-
* durability metadata can persist; re-registering rebinds referencing indexes. */
|
|
273
|
-
registerEmbeddingAdapter(name: string, adapter: EmbeddingAdapter): void;
|
|
274
232
|
/**
|
|
275
233
|
* Removes an index and everything it holds, including whatever persistence
|
|
276
234
|
* wrote for it. The documents do not come back.
|
|
@@ -423,28 +381,94 @@ interface Narsil extends IndexLifecycleOperations, DocumentWriteOperations {
|
|
|
423
381
|
*/
|
|
424
382
|
rebuildAnalysis(indexName: string): Promise<void>;
|
|
425
383
|
/**
|
|
426
|
-
*
|
|
427
|
-
*
|
|
384
|
+
* Returns a stream of the index's snapshot archive. The archive is a pax
|
|
385
|
+
* file containing the index's configuration and the files of one
|
|
386
|
+
* checkpoint, and any implementation of the Narsil format can restore it.
|
|
387
|
+
*
|
|
388
|
+
* With write-ahead log durability, the engine takes a checkpoint before it
|
|
389
|
+
* starts the stream, so the archive contains every write that the engine
|
|
390
|
+
* acknowledged before the call. The engine keeps that checkpoint's files on
|
|
391
|
+
* disk until the stream ends or you cancel it. Without write-ahead log
|
|
392
|
+
* durability, the engine builds the archive's files from the index in
|
|
393
|
+
* memory.
|
|
394
|
+
*
|
|
395
|
+
* @param indexName - The index to stream.
|
|
396
|
+
* @returns A stream of the archive's bytes, which you can write to a file or
|
|
397
|
+
* send over a network.
|
|
398
|
+
*/
|
|
399
|
+
snapshot(indexName: string): Promise<ReadableStream<Uint8Array>>;
|
|
400
|
+
/**
|
|
401
|
+
* Creates an index from a snapshot under the name that you pass, replacing
|
|
402
|
+
* any index of that name.
|
|
403
|
+
*
|
|
404
|
+
* The engine accepts the archive that {@link Narsil.snapshot} streams and
|
|
405
|
+
* the `.nrsl` file of release 0.3. It checks each archive file against its
|
|
406
|
+
* manifest and loads the restored index beside the existing one, which
|
|
407
|
+
* answers requests until the swap. A failed restore leaves the existing index
|
|
408
|
+
* serving, and after a crash the engine reopens either the old or the new one.
|
|
409
|
+
*
|
|
410
|
+
* @param indexName - The name that the restored index takes.
|
|
411
|
+
* @param data - The snapshot's bytes, as an array, a stream, or async chunks.
|
|
412
|
+
* @throws A `NarsilError` with `PERSISTENCE_LOAD_FAILED` when the archive is
|
|
413
|
+
* malformed or a file's length differs from the manifest. The engine raises
|
|
414
|
+
* `PERSISTENCE_CRC_MISMATCH` when a file's checksum differs from its
|
|
415
|
+
* contents, and `ENVELOPE_VERSION_MISMATCH` when a file has a format version
|
|
416
|
+
* other than the ones that this release implements. It raises
|
|
417
|
+
* `CONFIG_INVALID` when the archive's configuration is invalid,
|
|
418
|
+
* `DOC_VALIDATION_FAILED` when the data is empty or the release 0.3 file is
|
|
419
|
+
* invalid, `RESTORE_MEMORY_EXCEEDED` when the restored index needs more
|
|
420
|
+
* memory than the process has free, and `WORKER_TIMEOUT` when a
|
|
421
|
+
* worker still holds the old index 30 seconds after the engine asks for it.
|
|
422
|
+
*/
|
|
423
|
+
restore(indexName: string, data: ByteSource): Promise<void>;
|
|
424
|
+
/**
|
|
425
|
+
* Returns a stream of the index's document export. The export is JSON Lines
|
|
426
|
+
* text whose format stays the same in every release, so you can use it to
|
|
427
|
+
* move an index across releases or between implementations.
|
|
428
|
+
*
|
|
429
|
+
* The first line is a header containing the format name, the format
|
|
430
|
+
* version, and the index configuration. Each later line contains one
|
|
431
|
+
* document, with its ID under `id` and the value of each vector field as an
|
|
432
|
+
* array of numbers. With write-ahead log durability, the engine exports every
|
|
433
|
+
* document from one checkpoint; otherwise, it exports them from the index in
|
|
434
|
+
* memory at one moment.
|
|
435
|
+
*
|
|
436
|
+
* JSON has no type for a date, for raw bytes, or for an infinite number or
|
|
437
|
+
* `NaN`, so the engine writes each such value as a tag. Each tag is an
|
|
438
|
+
* object with one key that begins with `$`, such as
|
|
439
|
+
* `{"$date":[1700000000,123000000]}`. The engine also wraps a stored object
|
|
440
|
+
* whose only key begins with `$` in an `$object` tag, because the import
|
|
441
|
+
* decodes every other object of that shape as a tag.
|
|
442
|
+
*
|
|
443
|
+
* @param indexName - The index to export.
|
|
444
|
+
* @returns A stream of the export's UTF-8 bytes.
|
|
445
|
+
*/
|
|
446
|
+
exportDocuments(indexName: string): Promise<ReadableStream<Uint8Array>>;
|
|
447
|
+
/**
|
|
448
|
+
* Imports a document export into the named index. Where the name is free,
|
|
449
|
+
* the engine creates the index from the export's configuration.
|
|
450
|
+
*
|
|
451
|
+
* The engine writes the documents in batches and replaces any document
|
|
452
|
+
* whose ID the index already contains. Because it replaces existing
|
|
453
|
+
* documents, you can run an interrupted import again to make the index equal
|
|
454
|
+
* to the export. When a line is invalid or the engine rejects its document,
|
|
455
|
+
* the engine records the refusal and continues with the next line.
|
|
456
|
+
*
|
|
457
|
+
* The engine converts each tag that {@link Narsil.exportDocuments} writes
|
|
458
|
+
* into the value that the tag stands for. When an object has one key that
|
|
459
|
+
* begins with `$` and that key names no tag, or when a tag holds an invalid
|
|
460
|
+
* value, the engine refuses the line.
|
|
428
461
|
*
|
|
429
|
-
* @param indexName - The index
|
|
430
|
-
* @
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
*
|
|
435
|
-
*
|
|
436
|
-
*
|
|
437
|
-
*
|
|
438
|
-
* @param indexName - The name the restored index takes.
|
|
439
|
-
* @param data - Bytes that {@link Narsil.snapshot} produced.
|
|
440
|
-
* @throws A `NarsilError` with `ENVELOPE_INVALID_MAGIC` when the bytes are
|
|
441
|
-
* shorter than an envelope header or open with anything other than the
|
|
442
|
-
* `NRSL` magic bytes, `ENVELOPE_VERSION_MISMATCH` when the envelope carries a
|
|
443
|
-
* format version other than the index snapshot format,
|
|
444
|
-
* `PERSISTENCE_CRC_MISMATCH` when the payload fails its checksum, and
|
|
445
|
-
* `DOC_VALIDATION_FAILED` when the envelope holds no valid index snapshot.
|
|
462
|
+
* @param indexName - The index that receives the documents.
|
|
463
|
+
* @param data - The export's bytes, as one array, a stream, or an async
|
|
464
|
+
* iterable of chunks.
|
|
465
|
+
* @returns The number of imported documents, the number of refused lines,
|
|
466
|
+
* and the 1,000 refusals with the lowest line numbers.
|
|
467
|
+
* @throws A `NarsilError` with `CONFIG_INVALID` when the first line is
|
|
468
|
+
* anything other than a version 1 header, or when the named index exists
|
|
469
|
+
* with a different schema.
|
|
446
470
|
*/
|
|
447
|
-
|
|
471
|
+
importDocumentExport(indexName: string, data: ByteSource): Promise<DocumentImportResult>;
|
|
448
472
|
/**
|
|
449
473
|
* Writes an index's current state to the durability directory, so a restart
|
|
450
474
|
* replays less of the log.
|
|
@@ -550,4 +574,4 @@ interface Narsil extends IndexLifecycleOperations, DocumentWriteOperations {
|
|
|
550
574
|
shutdown(): Promise<void>;
|
|
551
575
|
}
|
|
552
576
|
|
|
553
|
-
export type { DocumentWriteOperations as D, IndexLifecycleOperations as I,
|
|
577
|
+
export type { AdapterRegistrationOperations as A, ByteSource as B, DocumentWriteOperations as D, IndexLifecycleOperations as I, Narsil as N, SynonymOperations as S };
|