@delali/narsil 0.2.4-next.48 → 0.2.4-next.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/adapters/indexeddb.mjs +15 -10
- package/dist/{chunk-BFIH7OV5.mjs → chunk-2GWVQWGA.mjs} +3 -1
- package/dist/{chunk-UZTT2BKQ.mjs → chunk-56PJBIJ6.mjs} +891 -542
- package/dist/chunk-CBLS46IE.mjs +10 -0
- package/dist/{chunk-JQUL2T6I.mjs → chunk-GCM3JFVA.mjs} +1 -2
- package/dist/{chunk-4COAUBLJ.mjs → chunk-HGJC5L65.mjs} +4 -7
- package/dist/{chunk-DFZDV5FV.mjs → chunk-JXHUHFJ5.mjs} +27 -14
- package/dist/{chunk-XWICBGYT.mjs → chunk-KGRYQVZ5.mjs} +2 -5
- package/dist/{chunk-EV45L35L.mjs → chunk-M6JGMKA5.mjs} +391 -183
- package/dist/{chunk-PM52SMBT.mjs → chunk-OOSIDHA3.mjs} +2 -1
- package/dist/chunk-ORBLD3EL.mjs +6 -0
- package/dist/chunk-OWCFK42Q.mjs +30 -0
- package/dist/{chunk-WTONLDU3.mjs → chunk-RDGXYRF4.mjs} +1 -0
- package/dist/chunk-UNEEFTEU.mjs +7 -0
- package/dist/chunk-YEO6G2NG.mjs +6 -0
- package/dist/{client-C5BkUqB8.d.ts → client-DX7xPcNA.d.ts} +1 -1
- package/dist/client.d.ts +5 -5
- package/dist/client.mjs +224 -222
- package/dist/{config-B-ZI-ft5.d.ts → config-B41W23Us.d.ts} +1 -1
- package/dist/{config-BZRExglL.d.ts → config-Co-zAXqO.d.ts} +27 -11
- package/dist/direct-executor-HRTDWMVB.mjs +9 -0
- package/dist/distribution/coordinator/etcd.mjs +2 -3
- package/dist/distribution/coordinator/in-memory.mjs +2 -2
- package/dist/distribution/transport/grpc.mjs +7 -9
- package/dist/distribution/transport/in-memory.mjs +2 -1
- package/dist/distribution/transport/tcp.mjs +3 -2
- package/dist/distribution.d.ts +5 -5
- package/dist/distribution.mjs +121 -97
- package/dist/embeddings/openai.mjs +5 -3
- package/dist/{engine-Cjr1uBLa.d.ts → engine-CABZg25p.d.ts} +12 -8
- package/dist/index.browser.mjs +1514 -894
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +9 -8
- package/dist/invalidation/broadcast-channel.mjs +2 -1
- package/dist/invalidation/filesystem.mjs +3 -4
- package/dist/{options-BC68VPoE.d.ts → options-Dtk99qUs.d.ts} +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +9 -8
- package/dist/react.d.ts +3 -3
- package/dist/react.mjs +7 -5
- package/dist/{search-DDHjfXhJ.d.ts → search-FjWhZN3f.d.ts} +20 -3
- package/dist/server.d.ts +6 -6
- package/dist/server.mjs +43 -30
- package/dist/vector/hnsw-build-worker.mjs +2 -4
- package/dist/vector/search-worker.mjs +3 -4
- package/dist/workers/entry.mjs +2 -2
- package/package.json +1 -1
- package/dist/direct-executor-EOTFGXTO.mjs +0 -8
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { MAX_MESSAGE_SIZE_BYTES
|
|
1
|
+
import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
|
|
2
|
+
import { TransportError, TransportErrorCodes } from './chunk-GCM3JFVA.mjs';
|
|
2
3
|
import { encode, decode } from '@msgpack/msgpack';
|
|
3
4
|
|
|
4
5
|
// src/distribution/transport/tcp/types.ts
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { DEFAULT_PAGE_SIZE } from './chunk-CBLS46IE.mjs';
|
|
2
|
+
|
|
3
|
+
// src/distribution/constants.ts
|
|
4
|
+
var WIRE_BATCH_BUDGET = {
|
|
5
|
+
maxCount: 1e3,
|
|
6
|
+
maxBytes: 8388608,
|
|
7
|
+
overheadBytes: 256
|
|
8
|
+
};
|
|
9
|
+
var MAX_PARTITION_COUNT = 65536;
|
|
10
|
+
var MAX_REPLICATION_FACTOR = 255;
|
|
11
|
+
|
|
12
|
+
// src/engine/constants.ts
|
|
13
|
+
var MIN_DOCUMENTS_FOR_SEGMENTS = 64;
|
|
14
|
+
var BATCH_CHUNK_SIZE = 1e3;
|
|
15
|
+
var DEFAULT_LIMIT = DEFAULT_PAGE_SIZE;
|
|
16
|
+
var DEFAULT_OFFSET = 0;
|
|
17
|
+
var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
|
|
18
|
+
var MAX_INDEX_NAME_LENGTH = 255;
|
|
19
|
+
var MAX_DOC_ID_LENGTH = 512;
|
|
20
|
+
var MIN_SWEEP_INTERVAL_MS = 100;
|
|
21
|
+
var MAX_SWEEP_INTERVAL_MS = 6e4;
|
|
22
|
+
var STATISTICS_BROADCAST_INTERVAL_MS = 5e3;
|
|
23
|
+
var FOREIGN_STATS_TTL_MS = 6e4;
|
|
24
|
+
var DEFAULT_SUGGEST_LIMIT = 10;
|
|
25
|
+
var MAX_SUGGEST_LIMIT = 100;
|
|
26
|
+
var SUGGEST_OVERSAMPLE_FACTOR = 1.5;
|
|
27
|
+
var SUGGEST_OVERSAMPLE_PADDING = 10;
|
|
28
|
+
var MAX_SUGGEST_SCATTER_LIMIT = Math.ceil(MAX_SUGGEST_LIMIT * SUGGEST_OVERSAMPLE_FACTOR) + SUGGEST_OVERSAMPLE_PADDING;
|
|
29
|
+
|
|
30
|
+
export { BATCH_CHUNK_SIZE, DEFAULT_LIMIT, DEFAULT_OFFSET, DEFAULT_SUGGEST_LIMIT, FOREIGN_STATS_TTL_MS, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, MAX_INDEX_NAME_LENGTH, MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_SWEEP_INTERVAL_MS, MIN_DOCUMENTS_FOR_SEGMENTS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, WIRE_BATCH_BUDGET };
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
// src/distribution/transport/constants.ts
|
|
2
|
+
var MAX_MESSAGE_SIZE_BYTES = 67108864;
|
|
3
|
+
var GRPC_ENVELOPE_OVERHEAD_BYTES = 1024;
|
|
4
|
+
var GRPC_KEEPALIVE_TIME_MS = 3e4;
|
|
5
|
+
var GRPC_KEEPALIVE_TIMEOUT_MS = 1e4;
|
|
6
|
+
|
|
7
|
+
export { GRPC_ENVELOPE_OVERHEAD_BYTES, GRPC_KEEPALIVE_TIMEOUT_MS, GRPC_KEEPALIVE_TIME_MS, MAX_MESSAGE_SIZE_BYTES };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { V as VectorMaintenanceResult, M as MemoryStats, B as BatchResult, L as ListParams, g as ListResult, I as IndexInfo, f as IndexStats, P as PartitionStatsResult, Q as QueryParams, e as QueryResult, h as PreflightResult, i as SuggestParams, j as SuggestResult } from './search-DDHjfXhJ.js';
|
|
2
1
|
import { P as PartitionConfig, A as AnyDocument, b as InsertOptions } from './schema-B7juWg8Q.js';
|
|
2
|
+
import { V as VectorMaintenanceResult, M as MemoryStats, B as BatchResult, L as ListParams, g as ListResult, I as IndexInfo, f as IndexStats, P as PartitionStatsResult, Q as QueryParams, e as QueryResult, h as PreflightResult, i as SuggestParams, j as SuggestResult } from './search-FjWhZN3f.js';
|
|
3
3
|
import { e as TaskRecord, b as ImportResult, H as HttpIndexConfig, c as TaskListQuery, T as TaskListPage } from './tasks-Bn4R1NPN.js';
|
|
4
4
|
|
|
5
5
|
/**
|
package/dist/client.d.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes } from './search-
|
|
2
|
-
export { A as ASYNC_IMPORT_CAPABILITY, E as ErrorEnvelope, I as INDEX_LIFECYCLE_CAPABILITY, R as REBUILD_ANALYSIS_CAPABILITY, S as SERVER_CAPABILITIES, T as TASK_CANCEL_CAPABILITY, a as TASK_FILTER_CAPABILITY } from './options-
|
|
1
|
+
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes } from './search-FjWhZN3f.js';
|
|
2
|
+
export { A as ASYNC_IMPORT_CAPABILITY, E as ErrorEnvelope, I as INDEX_LIFECYCLE_CAPABILITY, R as REBUILD_ANALYSIS_CAPABILITY, S as SERVER_CAPABILITIES, T as TASK_CANCEL_CAPABILITY, a as TASK_FILTER_CAPABILITY } from './options-Dtk99qUs.js';
|
|
3
3
|
export { C as CapabilitiesResponse, a as CreateIndexEmbedding, H as HttpIndexConfig, I as ImportError, b as ImportResult, T as TaskListPage, c as TaskListQuery, d as TaskProgress, e as TaskRecord, f as TaskStatus, g as TaskType } from './tasks-Bn4R1NPN.js';
|
|
4
|
-
export { A as AdminOperations, B as BulkOperations, D as DocumentOperations, F as FetchFunction, I as ImportSource, a as IndexOperations, N as NarsilClient, b as NarsilClientOptions, P as PutResult, R as RequestOptions, S as SearchOperations, c as ServerOperations, d as ServerVersion, T as TaskOperations, W as WaitForTaskOptions, e as createNarsilClient } from './client-
|
|
4
|
+
export { A as AdminOperations, B as BulkOperations, D as DocumentOperations, F as FetchFunction, I as ImportSource, a as IndexOperations, N as NarsilClient, b as NarsilClientOptions, P as PutResult, R as RequestOptions, S as SearchOperations, c as ServerOperations, d as ServerVersion, T as TaskOperations, W as WaitForTaskOptions, e as createNarsilClient } from './client-DX7xPcNA.js';
|
|
5
5
|
import './schema-B7juWg8Q.js';
|
|
6
6
|
import './adapters-79OIa4xm.js';
|
|
7
|
-
import './config-
|
|
8
|
-
import './config-
|
|
7
|
+
import './config-B41W23Us.js';
|
|
8
|
+
import './config-Co-zAXqO.js';
|
|
9
9
|
import './types-DsSSjY4d.js';
|
|
10
10
|
import './types-D5OwhtsQ.js';
|
package/dist/client.mjs
CHANGED
|
@@ -166,220 +166,10 @@ function encodeJson(value) {
|
|
|
166
166
|
return JSON.stringify(value, toPlainNumbers);
|
|
167
167
|
}
|
|
168
168
|
|
|
169
|
-
// src/client/
|
|
170
|
-
var
|
|
169
|
+
// src/client/constants.ts
|
|
170
|
+
var DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
|
|
171
171
|
var NO_TIMEOUT = 0;
|
|
172
|
-
|
|
173
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
174
|
-
}
|
|
175
|
-
function unrefTimer(timer) {
|
|
176
|
-
const unrefable = timer;
|
|
177
|
-
unrefable.unref?.();
|
|
178
|
-
}
|
|
179
|
-
function startDeadline(timeoutMs, caller) {
|
|
180
|
-
const controller = new AbortController();
|
|
181
|
-
let expired = false;
|
|
182
|
-
const abortWithCaller = () => controller.abort();
|
|
183
|
-
if (caller?.aborted) controller.abort();
|
|
184
|
-
else caller?.addEventListener("abort", abortWithCaller);
|
|
185
|
-
let timer;
|
|
186
|
-
if (timeoutMs > 0 && !controller.signal.aborted) {
|
|
187
|
-
timer = setTimeout(() => {
|
|
188
|
-
expired = true;
|
|
189
|
-
controller.abort();
|
|
190
|
-
}, timeoutMs);
|
|
191
|
-
unrefTimer(timer);
|
|
192
|
-
}
|
|
193
|
-
return {
|
|
194
|
-
signal: controller.signal,
|
|
195
|
-
expired: () => expired,
|
|
196
|
-
release: () => {
|
|
197
|
-
if (timer !== void 0) clearTimeout(timer);
|
|
198
|
-
caller?.removeEventListener("abort", abortWithCaller);
|
|
199
|
-
}
|
|
200
|
-
};
|
|
201
|
-
}
|
|
202
|
-
var ResponseTooLargeError = class extends Error {
|
|
203
|
-
constructor(limitBytes) {
|
|
204
|
-
super(`The answer passed the ${limitBytes} byte ceiling this client reads`);
|
|
205
|
-
this.limitBytes = limitBytes;
|
|
206
|
-
}
|
|
207
|
-
};
|
|
208
|
-
async function readStreamCapped(stream, limitBytes) {
|
|
209
|
-
const reader = stream.getReader();
|
|
210
|
-
const chunks = [];
|
|
211
|
-
let total = 0;
|
|
212
|
-
try {
|
|
213
|
-
for (; ; ) {
|
|
214
|
-
const { done, value } = await reader.read();
|
|
215
|
-
if (done) break;
|
|
216
|
-
total += value.byteLength;
|
|
217
|
-
if (total > limitBytes) throw new ResponseTooLargeError(limitBytes);
|
|
218
|
-
chunks.push(value);
|
|
219
|
-
}
|
|
220
|
-
} finally {
|
|
221
|
-
reader.releaseLock();
|
|
222
|
-
if (total > limitBytes) await stream.cancel().catch(() => void 0);
|
|
223
|
-
}
|
|
224
|
-
const body = new Uint8Array(total);
|
|
225
|
-
let offset = 0;
|
|
226
|
-
for (const chunk of chunks) {
|
|
227
|
-
body.set(chunk, offset);
|
|
228
|
-
offset += chunk.byteLength;
|
|
229
|
-
}
|
|
230
|
-
return body;
|
|
231
|
-
}
|
|
232
|
-
async function readAnswer(response, limitBytes, wantsBytes) {
|
|
233
|
-
const stream = response.body;
|
|
234
|
-
if (limitBytes > 0 && stream !== null && stream !== void 0) {
|
|
235
|
-
const body = await readStreamCapped(stream, limitBytes);
|
|
236
|
-
return wantsBytes ? { text: "", bytes: body } : { text: new TextDecoder().decode(body), bytes: null };
|
|
237
|
-
}
|
|
238
|
-
if (wantsBytes) {
|
|
239
|
-
const bytes = new Uint8Array(await response.arrayBuffer());
|
|
240
|
-
if (limitBytes > 0 && bytes.byteLength > limitBytes) throw new ResponseTooLargeError(limitBytes);
|
|
241
|
-
return { text: "", bytes };
|
|
242
|
-
}
|
|
243
|
-
const text = await response.text();
|
|
244
|
-
if (limitBytes > 0 && text.length > limitBytes) throw new ResponseTooLargeError(limitBytes);
|
|
245
|
-
return { text, bytes: null };
|
|
246
|
-
}
|
|
247
|
-
function invalidResponse(message, details) {
|
|
248
|
-
return new NarsilError(ClientErrorCodes.CLIENT_INVALID_RESPONSE, message, details);
|
|
249
|
-
}
|
|
250
|
-
function errorFromBody(status, payload, url) {
|
|
251
|
-
const envelope = isRecord(payload) && isRecord(payload.error) ? payload.error : void 0;
|
|
252
|
-
if (envelope === void 0 || typeof envelope.code !== "string" || typeof envelope.message !== "string") {
|
|
253
|
-
return invalidResponse(`The server answered HTTP ${status} without a Narsil error envelope`, { url, status });
|
|
254
|
-
}
|
|
255
|
-
const details = isRecord(envelope.details) ? envelope.details : {};
|
|
256
|
-
return new NarsilError(envelope.code, envelope.message, { ...details, status });
|
|
257
|
-
}
|
|
258
|
-
function isAbsoluteUrl(value) {
|
|
259
|
-
try {
|
|
260
|
-
new URL(value);
|
|
261
|
-
return true;
|
|
262
|
-
} catch {
|
|
263
|
-
return false;
|
|
264
|
-
}
|
|
265
|
-
}
|
|
266
|
-
function normaliseBase(url) {
|
|
267
|
-
const trimmed = url.trim();
|
|
268
|
-
if (trimmed.length === 0) {
|
|
269
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'The client needs a server address, and "url" is empty');
|
|
270
|
-
}
|
|
271
|
-
if (!trimmed.startsWith("/") && !isAbsoluteUrl(trimmed)) {
|
|
272
|
-
throw new NarsilError(
|
|
273
|
-
ErrorCodes.CONFIG_INVALID,
|
|
274
|
-
`The client cannot read "${trimmed}" as a server address; pass an absolute URL or a path starting with "/"`,
|
|
275
|
-
{ url: trimmed }
|
|
276
|
-
);
|
|
277
|
-
}
|
|
278
|
-
return trimmed.endsWith("/") ? trimmed.slice(0, -1) : trimmed;
|
|
279
|
-
}
|
|
280
|
-
function withQuery(path, query) {
|
|
281
|
-
if (query === void 0) return path;
|
|
282
|
-
const params = new URLSearchParams();
|
|
283
|
-
for (const [key, value] of Object.entries(query)) {
|
|
284
|
-
if (value !== void 0) params.set(key, value);
|
|
285
|
-
}
|
|
286
|
-
const encoded = params.toString();
|
|
287
|
-
return encoded.length === 0 ? path : `${path}?${encoded}`;
|
|
288
|
-
}
|
|
289
|
-
function createTransport(options) {
|
|
290
|
-
const base = normaliseBase(options.url);
|
|
291
|
-
const send = options.fetch ?? ((input, init) => fetch(input, init));
|
|
292
|
-
const baseHeaders = { ...options.headers };
|
|
293
|
-
if (options.apiKey !== void 0 && options.apiKey.length > 0) {
|
|
294
|
-
baseHeaders.authorization = `Bearer ${options.apiKey}`;
|
|
295
|
-
}
|
|
296
|
-
function resolveTimeout(spec) {
|
|
297
|
-
const perCall = spec.options?.timeoutMs;
|
|
298
|
-
if (perCall !== void 0) return perCall;
|
|
299
|
-
if (options.timeoutMs !== void 0) return options.timeoutMs;
|
|
300
|
-
return spec.defaultTimeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
301
|
-
}
|
|
302
|
-
function resolveResponseCeiling(spec) {
|
|
303
|
-
const perCall = spec.options?.maxResponseBytes;
|
|
304
|
-
if (perCall !== void 0) return perCall;
|
|
305
|
-
return options.maxResponseBytes ?? 0;
|
|
306
|
-
}
|
|
307
|
-
async function exchange(spec) {
|
|
308
|
-
const url = `${base}${withQuery(spec.path, spec.query)}`;
|
|
309
|
-
const headers = { ...baseHeaders, ...spec.options?.headers };
|
|
310
|
-
if (spec.contentType !== void 0) headers["content-type"] = spec.contentType;
|
|
311
|
-
const timeoutMs = resolveTimeout(spec);
|
|
312
|
-
const deadline = startDeadline(timeoutMs, spec.options?.signal);
|
|
313
|
-
try {
|
|
314
|
-
const init = { method: spec.method, headers, signal: deadline.signal };
|
|
315
|
-
if (spec.body !== void 0) init.body = spec.body;
|
|
316
|
-
const response = await send(url, init);
|
|
317
|
-
const wantsBytes = spec.binaryAnswer === true && response.ok;
|
|
318
|
-
const answer = await readAnswer(response, resolveResponseCeiling(spec), wantsBytes);
|
|
319
|
-
return { status: response.status, ...answer };
|
|
320
|
-
} catch (err) {
|
|
321
|
-
if (err instanceof ResponseTooLargeError) {
|
|
322
|
-
throw invalidResponse(`The server at ${url} answered more than this client reads: ${err.message}`, {
|
|
323
|
-
url,
|
|
324
|
-
maxResponseBytes: err.limitBytes
|
|
325
|
-
});
|
|
326
|
-
}
|
|
327
|
-
if (deadline.expired()) {
|
|
328
|
-
throw new NarsilError(
|
|
329
|
-
ClientErrorCodes.CLIENT_REQUEST_TIMEOUT,
|
|
330
|
-
`The server at ${url} did not answer within ${timeoutMs} ms`,
|
|
331
|
-
{ url, timeoutMs }
|
|
332
|
-
);
|
|
333
|
-
}
|
|
334
|
-
if (spec.options?.signal?.aborted === true) {
|
|
335
|
-
throw new NarsilError(ClientErrorCodes.CLIENT_REQUEST_ABORTED, `The caller cancelled the request to ${url}`, {
|
|
336
|
-
url
|
|
337
|
-
});
|
|
338
|
-
}
|
|
339
|
-
throw new NarsilError(
|
|
340
|
-
ClientErrorCodes.CLIENT_CONNECTION_FAILED,
|
|
341
|
-
`The client could not reach ${url}: ${describeError(err)}`,
|
|
342
|
-
{ url }
|
|
343
|
-
);
|
|
344
|
-
} finally {
|
|
345
|
-
deadline.release();
|
|
346
|
-
}
|
|
347
|
-
}
|
|
348
|
-
function parse(status, text, path) {
|
|
349
|
-
if (text.length === 0) return void 0;
|
|
350
|
-
try {
|
|
351
|
-
return JSON.parse(text);
|
|
352
|
-
} catch {
|
|
353
|
-
throw invalidResponse(`The server answered HTTP ${status} with a body that is not JSON`, { url: path, status });
|
|
354
|
-
}
|
|
355
|
-
}
|
|
356
|
-
async function json(spec) {
|
|
357
|
-
const { status, text } = await exchange(spec);
|
|
358
|
-
const payload = parse(status, text, spec.path);
|
|
359
|
-
if (status >= 200 && status < 300) return payload;
|
|
360
|
-
throw errorFromBody(status, payload, spec.path);
|
|
361
|
-
}
|
|
362
|
-
return {
|
|
363
|
-
json,
|
|
364
|
-
async jsonOrNull(spec, absentCode) {
|
|
365
|
-
try {
|
|
366
|
-
return await json(spec);
|
|
367
|
-
} catch (err) {
|
|
368
|
-
if (err instanceof NarsilError && err.code === absentCode) return null;
|
|
369
|
-
throw err;
|
|
370
|
-
}
|
|
371
|
-
},
|
|
372
|
-
async binary(spec) {
|
|
373
|
-
const { status, text, bytes } = await exchange(spec);
|
|
374
|
-
if (bytes !== null) return bytes;
|
|
375
|
-
throw errorFromBody(status, parse(status, text, spec.path), spec.path);
|
|
376
|
-
},
|
|
377
|
-
async probe(spec) {
|
|
378
|
-
const { status } = await exchange(spec);
|
|
379
|
-
return status;
|
|
380
|
-
}
|
|
381
|
-
};
|
|
382
|
-
}
|
|
172
|
+
var DEFAULT_TASK_POLL_INTERVAL_MS = 250;
|
|
383
173
|
|
|
384
174
|
// src/client/paths.ts
|
|
385
175
|
function indexPath(indexName) {
|
|
@@ -393,7 +183,7 @@ function taskPath(taskId) {
|
|
|
393
183
|
}
|
|
394
184
|
|
|
395
185
|
// src/client/response-shape.ts
|
|
396
|
-
function
|
|
186
|
+
function isRecord(value) {
|
|
397
187
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
398
188
|
}
|
|
399
189
|
function malformed(path, expected) {
|
|
@@ -402,27 +192,27 @@ function malformed(path, expected) {
|
|
|
402
192
|
});
|
|
403
193
|
}
|
|
404
194
|
function readArray(payload, key, path) {
|
|
405
|
-
if (!
|
|
195
|
+
if (!isRecord(payload) || !Array.isArray(payload[key])) throw malformed(path, `"${key}" array`);
|
|
406
196
|
return payload[key];
|
|
407
197
|
}
|
|
408
198
|
function readNumber(payload, key, path) {
|
|
409
|
-
if (!
|
|
199
|
+
if (!isRecord(payload) || typeof payload[key] !== "number") throw malformed(path, `"${key}" number`);
|
|
410
200
|
return payload[key];
|
|
411
201
|
}
|
|
412
202
|
function readBoolean(payload, key, path) {
|
|
413
|
-
if (!
|
|
203
|
+
if (!isRecord(payload) || typeof payload[key] !== "boolean") throw malformed(path, `"${key}" flag`);
|
|
414
204
|
return payload[key];
|
|
415
205
|
}
|
|
416
206
|
function readString(payload, key, path) {
|
|
417
|
-
if (!
|
|
207
|
+
if (!isRecord(payload) || typeof payload[key] !== "string") throw malformed(path, `"${key}" string`);
|
|
418
208
|
return payload[key];
|
|
419
209
|
}
|
|
420
210
|
function readObject(payload, key, path) {
|
|
421
|
-
if (!
|
|
211
|
+
if (!isRecord(payload) || !isRecord(payload[key])) throw malformed(path, `"${key}" object`);
|
|
422
212
|
return payload[key];
|
|
423
213
|
}
|
|
424
214
|
function readBody(payload, path) {
|
|
425
|
-
if (!
|
|
215
|
+
if (!isRecord(payload)) throw malformed(path, "JSON object");
|
|
426
216
|
return payload;
|
|
427
217
|
}
|
|
428
218
|
|
|
@@ -642,6 +432,219 @@ function createDocumentOperations(transport) {
|
|
|
642
432
|
};
|
|
643
433
|
}
|
|
644
434
|
|
|
435
|
+
// src/client/http.ts
|
|
436
|
+
function isRecord2(value) {
|
|
437
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
438
|
+
}
|
|
439
|
+
function unrefTimer(timer) {
|
|
440
|
+
const unrefable = timer;
|
|
441
|
+
unrefable.unref?.();
|
|
442
|
+
}
|
|
443
|
+
function startDeadline(timeoutMs, caller) {
|
|
444
|
+
const controller = new AbortController();
|
|
445
|
+
let expired = false;
|
|
446
|
+
const abortWithCaller = () => controller.abort();
|
|
447
|
+
if (caller?.aborted) controller.abort();
|
|
448
|
+
else caller?.addEventListener("abort", abortWithCaller);
|
|
449
|
+
let timer;
|
|
450
|
+
if (timeoutMs > 0 && !controller.signal.aborted) {
|
|
451
|
+
timer = setTimeout(() => {
|
|
452
|
+
expired = true;
|
|
453
|
+
controller.abort();
|
|
454
|
+
}, timeoutMs);
|
|
455
|
+
unrefTimer(timer);
|
|
456
|
+
}
|
|
457
|
+
return {
|
|
458
|
+
signal: controller.signal,
|
|
459
|
+
expired: () => expired,
|
|
460
|
+
release: () => {
|
|
461
|
+
if (timer !== void 0) clearTimeout(timer);
|
|
462
|
+
caller?.removeEventListener("abort", abortWithCaller);
|
|
463
|
+
}
|
|
464
|
+
};
|
|
465
|
+
}
|
|
466
|
+
var ResponseTooLargeError = class extends Error {
|
|
467
|
+
constructor(limitBytes) {
|
|
468
|
+
super(`The answer passed the ${limitBytes} byte ceiling this client reads`);
|
|
469
|
+
this.limitBytes = limitBytes;
|
|
470
|
+
}
|
|
471
|
+
};
|
|
472
|
+
async function readStreamCapped(stream, limitBytes) {
|
|
473
|
+
const reader = stream.getReader();
|
|
474
|
+
const chunks = [];
|
|
475
|
+
let total = 0;
|
|
476
|
+
try {
|
|
477
|
+
for (; ; ) {
|
|
478
|
+
const { done, value } = await reader.read();
|
|
479
|
+
if (done) break;
|
|
480
|
+
total += value.byteLength;
|
|
481
|
+
if (total > limitBytes) throw new ResponseTooLargeError(limitBytes);
|
|
482
|
+
chunks.push(value);
|
|
483
|
+
}
|
|
484
|
+
} finally {
|
|
485
|
+
reader.releaseLock();
|
|
486
|
+
if (total > limitBytes) await stream.cancel().catch(() => void 0);
|
|
487
|
+
}
|
|
488
|
+
const body = new Uint8Array(total);
|
|
489
|
+
let offset = 0;
|
|
490
|
+
for (const chunk of chunks) {
|
|
491
|
+
body.set(chunk, offset);
|
|
492
|
+
offset += chunk.byteLength;
|
|
493
|
+
}
|
|
494
|
+
return body;
|
|
495
|
+
}
|
|
496
|
+
async function readAnswer(response, limitBytes, wantsBytes) {
|
|
497
|
+
const stream = response.body;
|
|
498
|
+
if (limitBytes > 0 && stream !== null && stream !== void 0) {
|
|
499
|
+
const body = await readStreamCapped(stream, limitBytes);
|
|
500
|
+
return wantsBytes ? { text: "", bytes: body } : { text: new TextDecoder().decode(body), bytes: null };
|
|
501
|
+
}
|
|
502
|
+
if (wantsBytes) {
|
|
503
|
+
const bytes = new Uint8Array(await response.arrayBuffer());
|
|
504
|
+
if (limitBytes > 0 && bytes.byteLength > limitBytes) throw new ResponseTooLargeError(limitBytes);
|
|
505
|
+
return { text: "", bytes };
|
|
506
|
+
}
|
|
507
|
+
const text = await response.text();
|
|
508
|
+
if (limitBytes > 0 && text.length > limitBytes) throw new ResponseTooLargeError(limitBytes);
|
|
509
|
+
return { text, bytes: null };
|
|
510
|
+
}
|
|
511
|
+
function invalidResponse(message, details) {
|
|
512
|
+
return new NarsilError(ClientErrorCodes.CLIENT_INVALID_RESPONSE, message, details);
|
|
513
|
+
}
|
|
514
|
+
function errorFromBody(status, payload, url) {
|
|
515
|
+
const envelope = isRecord2(payload) && isRecord2(payload.error) ? payload.error : void 0;
|
|
516
|
+
if (envelope === void 0 || typeof envelope.code !== "string" || typeof envelope.message !== "string") {
|
|
517
|
+
return invalidResponse(`The server answered HTTP ${status} without a Narsil error envelope`, { url, status });
|
|
518
|
+
}
|
|
519
|
+
const details = isRecord2(envelope.details) ? envelope.details : {};
|
|
520
|
+
return new NarsilError(envelope.code, envelope.message, { ...details, status });
|
|
521
|
+
}
|
|
522
|
+
function isAbsoluteUrl(value) {
|
|
523
|
+
try {
|
|
524
|
+
new URL(value);
|
|
525
|
+
return true;
|
|
526
|
+
} catch {
|
|
527
|
+
return false;
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
function normaliseBase(url) {
|
|
531
|
+
const trimmed = url.trim();
|
|
532
|
+
if (trimmed.length === 0) {
|
|
533
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'The client needs a server address, and "url" is empty');
|
|
534
|
+
}
|
|
535
|
+
if (!trimmed.startsWith("/") && !isAbsoluteUrl(trimmed)) {
|
|
536
|
+
throw new NarsilError(
|
|
537
|
+
ErrorCodes.CONFIG_INVALID,
|
|
538
|
+
`The client cannot read "${trimmed}" as a server address; pass an absolute URL or a path starting with "/"`,
|
|
539
|
+
{ url: trimmed }
|
|
540
|
+
);
|
|
541
|
+
}
|
|
542
|
+
return trimmed.endsWith("/") ? trimmed.slice(0, -1) : trimmed;
|
|
543
|
+
}
|
|
544
|
+
function withQuery(path, query) {
|
|
545
|
+
if (query === void 0) return path;
|
|
546
|
+
const params = new URLSearchParams();
|
|
547
|
+
for (const [key, value] of Object.entries(query)) {
|
|
548
|
+
if (value !== void 0) params.set(key, value);
|
|
549
|
+
}
|
|
550
|
+
const encoded = params.toString();
|
|
551
|
+
return encoded.length === 0 ? path : `${path}?${encoded}`;
|
|
552
|
+
}
|
|
553
|
+
function createTransport(options) {
|
|
554
|
+
const base = normaliseBase(options.url);
|
|
555
|
+
const send = options.fetch ?? ((input, init) => fetch(input, init));
|
|
556
|
+
const baseHeaders = { ...options.headers };
|
|
557
|
+
if (options.apiKey !== void 0 && options.apiKey.length > 0) {
|
|
558
|
+
baseHeaders.authorization = `Bearer ${options.apiKey}`;
|
|
559
|
+
}
|
|
560
|
+
function resolveTimeout(spec) {
|
|
561
|
+
const perCall = spec.options?.timeoutMs;
|
|
562
|
+
if (perCall !== void 0) return perCall;
|
|
563
|
+
if (options.timeoutMs !== void 0) return options.timeoutMs;
|
|
564
|
+
return spec.defaultTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
|
|
565
|
+
}
|
|
566
|
+
function resolveResponseCeiling(spec) {
|
|
567
|
+
const perCall = spec.options?.maxResponseBytes;
|
|
568
|
+
if (perCall !== void 0) return perCall;
|
|
569
|
+
return options.maxResponseBytes ?? 0;
|
|
570
|
+
}
|
|
571
|
+
async function exchange(spec) {
|
|
572
|
+
const url = `${base}${withQuery(spec.path, spec.query)}`;
|
|
573
|
+
const headers = { ...baseHeaders, ...spec.options?.headers };
|
|
574
|
+
if (spec.contentType !== void 0) headers["content-type"] = spec.contentType;
|
|
575
|
+
const timeoutMs = resolveTimeout(spec);
|
|
576
|
+
const deadline = startDeadline(timeoutMs, spec.options?.signal);
|
|
577
|
+
try {
|
|
578
|
+
const init = { method: spec.method, headers, signal: deadline.signal };
|
|
579
|
+
if (spec.body !== void 0) init.body = spec.body;
|
|
580
|
+
const response = await send(url, init);
|
|
581
|
+
const wantsBytes = spec.binaryAnswer === true && response.ok;
|
|
582
|
+
const answer = await readAnswer(response, resolveResponseCeiling(spec), wantsBytes);
|
|
583
|
+
return { status: response.status, ...answer };
|
|
584
|
+
} catch (err) {
|
|
585
|
+
if (err instanceof ResponseTooLargeError) {
|
|
586
|
+
throw invalidResponse(`The server at ${url} answered more than this client reads: ${err.message}`, {
|
|
587
|
+
url,
|
|
588
|
+
maxResponseBytes: err.limitBytes
|
|
589
|
+
});
|
|
590
|
+
}
|
|
591
|
+
if (deadline.expired()) {
|
|
592
|
+
throw new NarsilError(
|
|
593
|
+
ClientErrorCodes.CLIENT_REQUEST_TIMEOUT,
|
|
594
|
+
`The server at ${url} did not answer within ${timeoutMs} ms`,
|
|
595
|
+
{ url, timeoutMs }
|
|
596
|
+
);
|
|
597
|
+
}
|
|
598
|
+
if (spec.options?.signal?.aborted === true) {
|
|
599
|
+
throw new NarsilError(ClientErrorCodes.CLIENT_REQUEST_ABORTED, `The caller cancelled the request to ${url}`, {
|
|
600
|
+
url
|
|
601
|
+
});
|
|
602
|
+
}
|
|
603
|
+
throw new NarsilError(
|
|
604
|
+
ClientErrorCodes.CLIENT_CONNECTION_FAILED,
|
|
605
|
+
`The client could not reach ${url}: ${describeError(err)}`,
|
|
606
|
+
{ url }
|
|
607
|
+
);
|
|
608
|
+
} finally {
|
|
609
|
+
deadline.release();
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
function parse(status, text, path) {
|
|
613
|
+
if (text.length === 0) return void 0;
|
|
614
|
+
try {
|
|
615
|
+
return JSON.parse(text);
|
|
616
|
+
} catch {
|
|
617
|
+
throw invalidResponse(`The server answered HTTP ${status} with a body that is not JSON`, { url: path, status });
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
async function json(spec) {
|
|
621
|
+
const { status, text } = await exchange(spec);
|
|
622
|
+
const payload = parse(status, text, spec.path);
|
|
623
|
+
if (status >= 200 && status < 300) return payload;
|
|
624
|
+
throw errorFromBody(status, payload, spec.path);
|
|
625
|
+
}
|
|
626
|
+
return {
|
|
627
|
+
json,
|
|
628
|
+
async jsonOrNull(spec, absentCode) {
|
|
629
|
+
try {
|
|
630
|
+
return await json(spec);
|
|
631
|
+
} catch (err) {
|
|
632
|
+
if (err instanceof NarsilError && err.code === absentCode) return null;
|
|
633
|
+
throw err;
|
|
634
|
+
}
|
|
635
|
+
},
|
|
636
|
+
async binary(spec) {
|
|
637
|
+
const { status, text, bytes } = await exchange(spec);
|
|
638
|
+
if (bytes !== null) return bytes;
|
|
639
|
+
throw errorFromBody(status, parse(status, text, spec.path), spec.path);
|
|
640
|
+
},
|
|
641
|
+
async probe(spec) {
|
|
642
|
+
const { status } = await exchange(spec);
|
|
643
|
+
return status;
|
|
644
|
+
}
|
|
645
|
+
};
|
|
646
|
+
}
|
|
647
|
+
|
|
645
648
|
// src/client/indexes.ts
|
|
646
649
|
function createIndexOperations(transport) {
|
|
647
650
|
return {
|
|
@@ -746,7 +749,6 @@ function createServerOperations(transport) {
|
|
|
746
749
|
}
|
|
747
750
|
|
|
748
751
|
// src/client/tasks.ts
|
|
749
|
-
var DEFAULT_POLL_INTERVAL_MS = 250;
|
|
750
752
|
function isTerminal(record) {
|
|
751
753
|
return record.status === "succeeded" || record.status === "failed" || record.status === "cancelled";
|
|
752
754
|
}
|
|
@@ -794,7 +796,7 @@ function createTaskOperations(transport) {
|
|
|
794
796
|
return readBody(await transport.json({ method: "POST", path, options }), path);
|
|
795
797
|
},
|
|
796
798
|
async waitForTask(taskId, options) {
|
|
797
|
-
const interval = options?.pollIntervalMs ??
|
|
799
|
+
const interval = options?.pollIntervalMs ?? DEFAULT_TASK_POLL_INTERVAL_MS;
|
|
798
800
|
const waitTimeoutMs = options?.waitTimeoutMs ?? 0;
|
|
799
801
|
const startedAt = Date.now();
|
|
800
802
|
let reported = null;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { N as NarsilConfig } from './config-
|
|
1
|
+
import { N as NarsilConfig } from './config-Co-zAXqO.js';
|
|
2
2
|
import { A as AllocationTable, N as NodeRole, a as NodeRegistration, C as ClusterCoordinator, b as NodeCapacity } from './types-DsSSjY4d.js';
|
|
3
3
|
import { N as NodeTransport } from './types-D5OwhtsQ.js';
|
|
4
4
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { P as PersistenceAdapter, I as InvalidationAdapter, E as EmbeddingAdapter } from './adapters-79OIa4xm.js';
|
|
2
|
-
import { Q as QueryParams, e as QueryResult } from './search-
|
|
2
|
+
import { Q as QueryParams, e as QueryResult } from './search-FjWhZN3f.js';
|
|
3
3
|
import { I as IndexConfig, A as AnyDocument } from './schema-B7juWg8Q.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -148,7 +148,7 @@ interface NarsilConfig {
|
|
|
148
148
|
plugins?: NarsilPlugin[];
|
|
149
149
|
/** This supplies the id for a document inserted without one. The engine generates a UUID v7 by default. */
|
|
150
150
|
idGenerator?: () => string;
|
|
151
|
-
/** These settings control when
|
|
151
|
+
/** These settings control when an index gains worker copies, how many threads they share, and when an idle index gives them up. */
|
|
152
152
|
workers?: WorkerConfig;
|
|
153
153
|
/** Every index uses this adapter unless its own config names another. */
|
|
154
154
|
embedding?: EmbeddingAdapter;
|
|
@@ -249,23 +249,39 @@ interface DurabilityConfig {
|
|
|
249
249
|
compactionThreshold?: number;
|
|
250
250
|
}
|
|
251
251
|
/**
|
|
252
|
-
*
|
|
252
|
+
* How the engine holds worker copies of its indexes, and how many threads
|
|
253
|
+
* they may use.
|
|
253
254
|
*
|
|
254
|
-
*
|
|
255
|
-
*
|
|
256
|
-
*
|
|
255
|
+
* A small index answers fastest on the main thread. Once an index holds as
|
|
256
|
+
* many documents as the copy threshold, the engine loads a copy of it onto
|
|
257
|
+
* every worker thread, and each query then runs whole on the copy with the
|
|
258
|
+
* fewest queries in flight. The same switch governs the vector search pool.
|
|
257
259
|
*
|
|
258
260
|
* @public
|
|
259
261
|
*/
|
|
260
262
|
interface WorkerConfig {
|
|
261
|
-
/**
|
|
263
|
+
/**
|
|
264
|
+
* Whether the engine may hold worker copies and a vector search pool. It is
|
|
265
|
+
* on everywhere but a browser page. Set it to false to hold the process to
|
|
266
|
+
* one thread, which leaves both pools absent.
|
|
267
|
+
*/
|
|
262
268
|
enabled?: boolean;
|
|
263
|
-
/**
|
|
269
|
+
/**
|
|
270
|
+
* The keyword copies and the vector search pool share this many threads
|
|
271
|
+
* between them, half each. The engine takes the host's cores minus one by
|
|
272
|
+
* default, between 2 and 8.
|
|
273
|
+
*/
|
|
264
274
|
count?: number;
|
|
265
|
-
/**
|
|
275
|
+
/** An index gains worker copies once it holds this many documents, 1,000 by default. */
|
|
266
276
|
promotionThreshold?: number;
|
|
267
|
-
/**
|
|
268
|
-
|
|
277
|
+
/**
|
|
278
|
+
* The engine drops an index's copies after this many milliseconds without a
|
|
279
|
+
* read or a write, five minutes by default, and loads them again on the
|
|
280
|
+
* next request while the main copy answers it. It may not exceed
|
|
281
|
+
* `lifecycle.idleTimeoutMs`, and where it is unset the copies take the
|
|
282
|
+
* smaller of five minutes and that interval.
|
|
283
|
+
*/
|
|
284
|
+
idleTimeoutMs?: number;
|
|
269
285
|
/** Each worker imports this module on start-up, which is how a worker reaches a custom tokeniser or language. */
|
|
270
286
|
bootstrapModule?: string;
|
|
271
287
|
}
|