@delali/narsil 0.2.4-next.51 → 0.2.4-next.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -27
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/{capabilities-BWm4IITU.d.ts → capabilities-DlYw0Ov0.d.ts} +13 -2
- package/dist/{chunk-KMMJMDUA.mjs → chunk-3L4U6EJQ.mjs} +15 -10
- package/dist/chunk-4NVPQ6S5.mjs +366 -0
- package/dist/{chunk-CMVG2UEF.mjs → chunk-5T6AOR37.mjs} +56 -518
- package/dist/{chunk-ND34I3SV.mjs → chunk-BH5SYB5W.mjs} +3295 -2149
- package/dist/{chunk-XE6XR2OE.mjs → chunk-D2S642IV.mjs} +4 -2
- package/dist/{chunk-P4ELXVJQ.mjs → chunk-FRQVMFY7.mjs} +7 -1
- package/dist/chunk-IFBKLBGB.mjs +7 -0
- package/dist/chunk-L225SHNB.mjs +513 -0
- package/dist/{chunk-MHE7YWAU.mjs → chunk-NTZYA3SJ.mjs} +3 -3
- package/dist/{chunk-WYJXSLER.mjs → chunk-POWOTUKJ.mjs} +7 -1
- package/dist/{chunk-CV5H7XL7.mjs → chunk-RGS7FMCB.mjs} +53 -10
- package/dist/{chunk-ZHUQKZBE.mjs → chunk-TVT5V4E7.mjs} +101 -34
- package/dist/chunk-U2AL3SON.mjs +4198 -0
- package/dist/chunk-UTOFOJOR.mjs +135 -0
- package/dist/{chunk-BEIK3EOZ.mjs → chunk-XNQAIO74.mjs} +5523 -3847
- package/dist/chunk-Y3L5MDRV.mjs +96 -0
- package/dist/chunk-ZJ4CSAB2.mjs +13 -0
- package/dist/{client-B_X6_AbT.d.ts → client-CQUqt9Ir.d.ts} +13 -4
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +20 -14
- package/dist/{config-Czh2YNDi.d.ts → config-CNas6wJH.d.ts} +2 -2
- package/dist/{config-OON7IJ_B.d.ts → config-bXxnJrQc.d.ts} +80 -35
- package/dist/direct-executor-RBA5O7EN.mjs +12 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +32 -17
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/{engine-9PtFY50q.d.ts → engine-Dj4I91Jv.d.ts} +21 -8
- package/dist/index.browser.mjs +12962 -7786
- package/dist/index.d.ts +9 -21
- package/dist/index.mjs +14 -13
- package/dist/{memory-Bzd5xcEW.d.ts → memory-xrXOZMbw.d.ts} +68 -13
- package/dist/persistence/durability/checkpoint-worker.d.ts +138 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +32 -18
- package/dist/react.d.ts +11 -11
- package/dist/react.mjs +16 -4
- package/dist/{schema-BqRx9t4E.d.ts → schema-mLpXZBEh.d.ts} +43 -12
- package/dist/serialization/crc32-worker.mjs +2 -1
- package/dist/server.d.ts +13 -10
- package/dist/server.mjs +136 -47
- package/dist/simd-DW5AtJ6B.d.ts +41 -0
- package/dist/{tasks-DT7L1p6w.d.ts → tasks-Df73Tjgq.d.ts} +9 -5
- package/dist/{thread-PZLEO6JL.mjs → thread-5UP6MGED.mjs} +131 -43
- package/dist/{types-DXPxaXPO.d.ts → types-BTIVO8DI.d.ts} +1 -1
- package/dist/vector/search-worker.d.ts +292 -128
- package/dist/vector/search-worker.mjs +70 -54
- package/dist/workers/entry.mjs +30 -14
- package/package.json +52 -31
- package/dist/brute-force-DbHYCTc3.d.ts +0 -8
- package/dist/chunk-CBLS46IE.mjs +0 -10
- package/dist/chunk-PIWKZPP7.mjs +0 -64
- package/dist/chunk-PMBHH44R.mjs +0 -492
- package/dist/chunk-RP23PWMV.mjs +0 -11
- package/dist/chunk-X6YZO4MQ.mjs +0 -1572
- package/dist/direct-executor-OREXU72C.mjs +0 -12
- package/dist/vector/hnsw-build-worker.d.ts +0 -105
- package/dist/vector/hnsw-build-worker.mjs +0 -123
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { performance } from 'perf_hooks';
|
|
2
|
+
import { getHeapStatistics, setFlagsFromString } from 'v8';
|
|
3
|
+
import { runInNewContext } from 'vm';
|
|
4
|
+
|
|
5
|
+
// src/runtime/idle-heap-cleanup.ts
|
|
6
|
+
|
|
7
|
+
// src/runtime/constants.ts
|
|
8
|
+
var IDLE_HEAP_CLEANUP_CHECK_MS = 1e3;
|
|
9
|
+
var IDLE_HEAP_CLEANUP_QUIET_CHECKS = 2;
|
|
10
|
+
var IDLE_HEAP_CLEANUP_MAX_UTILISATION = 0.1;
|
|
11
|
+
var IDLE_HEAP_CLEANUP_MAX_GROWTH_BYTES = 4 * 1024 * 1024;
|
|
12
|
+
var IDLE_HEAP_CLEANUP_MIN_RECLAIMABLE_BYTES = 64 * 1024 * 1024;
|
|
13
|
+
|
|
14
|
+
// src/runtime/idle-heap-cleaner.ts
|
|
15
|
+
function createIdleHeapCleaner(deps) {
|
|
16
|
+
let quietChecks = 0;
|
|
17
|
+
let liveBytesAfterLastCleanup = 0;
|
|
18
|
+
let heldBytesAtLastCheck = Number.POSITIVE_INFINITY;
|
|
19
|
+
return {
|
|
20
|
+
check() {
|
|
21
|
+
const busy = deps.utilisationSinceLastCheck() > IDLE_HEAP_CLEANUP_MAX_UTILISATION;
|
|
22
|
+
const before = deps.readHeap();
|
|
23
|
+
if (before === null) return false;
|
|
24
|
+
const heapGrew = before.heldBytes > heldBytesAtLastCheck + IDLE_HEAP_CLEANUP_MAX_GROWTH_BYTES;
|
|
25
|
+
heldBytesAtLastCheck = before.heldBytes;
|
|
26
|
+
if (busy || heapGrew) {
|
|
27
|
+
quietChecks = 0;
|
|
28
|
+
return false;
|
|
29
|
+
}
|
|
30
|
+
quietChecks += 1;
|
|
31
|
+
if (quietChecks < IDLE_HEAP_CLEANUP_QUIET_CHECKS) return false;
|
|
32
|
+
if (before.heldBytes - liveBytesAfterLastCleanup < IDLE_HEAP_CLEANUP_MIN_RECLAIMABLE_BYTES) return false;
|
|
33
|
+
if (!deps.collectAndReturnPages()) return false;
|
|
34
|
+
const after = deps.readHeap();
|
|
35
|
+
liveBytesAfterLastCleanup = after?.liveBytes ?? before.liveBytes;
|
|
36
|
+
heldBytesAtLastCheck = after?.heldBytes ?? before.heldBytes;
|
|
37
|
+
quietChecks = 0;
|
|
38
|
+
return true;
|
|
39
|
+
}
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// src/runtime/idle-heap-cleanup.ts
|
|
44
|
+
var collectorOfThisThread = null;
|
|
45
|
+
function acquireCollector() {
|
|
46
|
+
if (collectorOfThisThread !== null) return collectorOfThisThread;
|
|
47
|
+
try {
|
|
48
|
+
setFlagsFromString("--expose-gc");
|
|
49
|
+
const collect = runInNewContext("gc");
|
|
50
|
+
setFlagsFromString("--no-expose-gc");
|
|
51
|
+
if (typeof collect !== "function") return null;
|
|
52
|
+
collectorOfThisThread = collect;
|
|
53
|
+
return collectorOfThisThread;
|
|
54
|
+
} catch {
|
|
55
|
+
return null;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
function collectAndReturnPages() {
|
|
59
|
+
const collect = acquireCollector();
|
|
60
|
+
if (collect === null) return false;
|
|
61
|
+
try {
|
|
62
|
+
collect({ type: "major", execution: "sync", flavor: "last-resort" });
|
|
63
|
+
return true;
|
|
64
|
+
} catch {
|
|
65
|
+
return false;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
function readHeap() {
|
|
69
|
+
try {
|
|
70
|
+
const stats = getHeapStatistics();
|
|
71
|
+
return { heldBytes: stats.total_physical_size, liveBytes: stats.used_heap_size };
|
|
72
|
+
} catch {
|
|
73
|
+
return null;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
function startIdleHeapCleanup() {
|
|
77
|
+
if (typeof performance.eventLoopUtilization !== "function") return () => void 0;
|
|
78
|
+
let lastReading = performance.eventLoopUtilization();
|
|
79
|
+
const cleaner = createIdleHeapCleaner({
|
|
80
|
+
utilisationSinceLastCheck() {
|
|
81
|
+
const reading = performance.eventLoopUtilization();
|
|
82
|
+
const { utilization } = performance.eventLoopUtilization(reading, lastReading);
|
|
83
|
+
lastReading = reading;
|
|
84
|
+
return utilization;
|
|
85
|
+
},
|
|
86
|
+
readHeap,
|
|
87
|
+
collectAndReturnPages
|
|
88
|
+
});
|
|
89
|
+
const timer = setInterval(() => {
|
|
90
|
+
cleaner.check();
|
|
91
|
+
}, IDLE_HEAP_CLEANUP_CHECK_MS);
|
|
92
|
+
timer.unref();
|
|
93
|
+
return () => clearInterval(timer);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export { startIdleHeapCleanup };
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
// src/vector/osq/constants.ts
|
|
2
|
+
var OSQ_GRID = { 1: 0.798, 2: 1.493, 4: 2.514, 8: 3.922 };
|
|
3
|
+
var OSQ_STEPS = { 1: 1, 2: 3, 4: 15, 8: 255 };
|
|
4
|
+
var OSQ_QUERY_BITS = { 1: 4, 2: 4, 4: 4, 8: 8 };
|
|
5
|
+
var OSQ_LAMBDA = 0.1;
|
|
6
|
+
var OSQ_REFINE_ROUNDS = 5;
|
|
7
|
+
var OSQ_REFINE_TOLERANCE = 1e-8;
|
|
8
|
+
var DEFAULT_OVERSAMPLE_BY_BITS = { 1: 3, 2: 3, 4: 2, 8: 2 };
|
|
9
|
+
var DEFAULT_OSQ4_NARROW_OVERSAMPLE = 5;
|
|
10
|
+
var OSQ4_NARROW_DIMENSION_LIMIT = 1024;
|
|
11
|
+
var MIN_OVERSAMPLE = 1;
|
|
12
|
+
|
|
13
|
+
export { DEFAULT_OSQ4_NARROW_OVERSAMPLE, DEFAULT_OVERSAMPLE_BY_BITS, MIN_OVERSAMPLE, OSQ4_NARROW_DIMENSION_LIMIT, OSQ_GRID, OSQ_LAMBDA, OSQ_QUERY_BITS, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, OSQ_STEPS };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { P as PartitionConfig, A as AnyDocument,
|
|
2
|
-
import { V as VectorMaintenanceResult, B as BatchResult, L as ListParams, g as ListResult, I as IndexInfo, f as IndexStats, P as PartitionStatsResult, Q as QueryParams, e as QueryResult, h as PreflightResult,
|
|
3
|
-
import { e as TaskRecord, M as MemoryStatsResponse, b as ImportResult, H as HttpIndexConfig, c as TaskListQuery, T as TaskListPage } from './tasks-
|
|
1
|
+
import { P as PartitionConfig, A as AnyDocument, a as InsertOptions, W as WriteOptions } from './schema-mLpXZBEh.js';
|
|
2
|
+
import { V as VectorMaintenanceResult, B as BatchResult, L as ListParams, g as ListResult, I as IndexInfo, f as IndexStats, P as PartitionStatsResult, Q as QueryParams, e as QueryResult, h as PreflightResult, j as SuggestParams, k as SuggestResult } from './memory-xrXOZMbw.js';
|
|
3
|
+
import { e as TaskRecord, M as MemoryStatsResponse, b as ImportResult, H as HttpIndexConfig, c as TaskListQuery, T as TaskListPage } from './tasks-Df73Tjgq.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* This is the `fetch` implementation the client sends every request through.
|
|
@@ -596,9 +596,11 @@ interface SearchOperations {
|
|
|
596
596
|
* this request.
|
|
597
597
|
* @returns The result holds the hits, the total match count, and whatever
|
|
598
598
|
* else the query asked for.
|
|
599
|
-
* @throws A `NarsilError` with `SEARCH_RESULT_WINDOW_EXCEEDED`
|
|
599
|
+
* @throws A `NarsilError` with `SEARCH_RESULT_WINDOW_EXCEEDED` where `offset`
|
|
600
600
|
* plus `limit` passes the server's result window, which the cursor pages
|
|
601
601
|
* beyond.
|
|
602
|
+
* @throws A `NarsilError` with `INVALID_REQUEST` where `limit` alone passes
|
|
603
|
+
* `limits.maxFetchDocuments`, which the server checks before the window.
|
|
602
604
|
*/
|
|
603
605
|
query<T = AnyDocument>(indexName: string, params: QueryParams, options?: RequestOptions): Promise<QueryResult<T>>;
|
|
604
606
|
/**
|
|
@@ -644,6 +646,13 @@ interface ServerVersion {
|
|
|
644
646
|
gitSha: string | null;
|
|
645
647
|
/** This is true when the working tree held uncommitted changes at build time. */
|
|
646
648
|
dirty: boolean;
|
|
649
|
+
/**
|
|
650
|
+
* The server thread that answered searches vector graphs through this path.
|
|
651
|
+
* `native` is the native search core in C, and `wasm` is WebAssembly. Both
|
|
652
|
+
* paths return the same results. A server that predates this field leaves it
|
|
653
|
+
* out.
|
|
654
|
+
*/
|
|
655
|
+
vectorSearch?: 'native' | 'wasm';
|
|
647
656
|
}
|
|
648
657
|
/**
|
|
649
658
|
* These methods ask a server what it is and what it serves, before you ask it
|
package/dist/client.d.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes } from './memory-
|
|
2
|
-
export { A as ASYNC_IMPORT_CAPABILITY, E as ErrorEnvelope, I as INDEX_LIFECYCLE_CAPABILITY, R as REBUILD_ANALYSIS_CAPABILITY, S as SERVER_CAPABILITIES, T as TASK_CANCEL_CAPABILITY, a as TASK_FILTER_CAPABILITY } from './capabilities-
|
|
3
|
-
export { C as CapabilitiesResponse, a as CreateIndexEmbedding, H as HttpIndexConfig, I as ImportError, b as ImportResult, M as MemoryStatsResponse, T as TaskListPage, c as TaskListQuery, d as TaskProgress, e as TaskRecord, f as TaskStatus, g as TaskType } from './tasks-
|
|
4
|
-
export { A as AdminOperations, B as BulkOperations, D as DocumentOperations, F as FetchFunction, I as ImportSource, a as IndexOperations, N as NarsilClient, b as NarsilClientOptions, P as PutResult, R as RequestOptions, S as SearchOperations, c as ServerOperations, d as ServerVersion, T as TaskOperations, W as WaitForTaskOptions, e as createNarsilClient } from './client-
|
|
5
|
-
import './schema-
|
|
1
|
+
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, i as isNarsilError } from './memory-xrXOZMbw.js';
|
|
2
|
+
export { A as ASYNC_IMPORT_CAPABILITY, E as ErrorEnvelope, I as INDEX_LIFECYCLE_CAPABILITY, R as REBUILD_ANALYSIS_CAPABILITY, S as SERVER_CAPABILITIES, T as TASK_CANCEL_CAPABILITY, a as TASK_FILTER_CAPABILITY } from './capabilities-DlYw0Ov0.js';
|
|
3
|
+
export { C as CapabilitiesResponse, a as CreateIndexEmbedding, H as HttpIndexConfig, I as ImportError, b as ImportResult, M as MemoryStatsResponse, T as TaskListPage, c as TaskListQuery, d as TaskProgress, e as TaskRecord, f as TaskStatus, g as TaskType } from './tasks-Df73Tjgq.js';
|
|
4
|
+
export { A as AdminOperations, B as BulkOperations, D as DocumentOperations, F as FetchFunction, I as ImportSource, a as IndexOperations, N as NarsilClient, b as NarsilClientOptions, P as PutResult, R as RequestOptions, S as SearchOperations, c as ServerOperations, d as ServerVersion, T as TaskOperations, W as WaitForTaskOptions, e as createNarsilClient } from './client-CQUqt9Ir.js';
|
|
5
|
+
import './schema-mLpXZBEh.js';
|
|
6
6
|
import './adapters-79OIa4xm.js';
|
|
7
|
-
import './config-
|
|
8
|
-
import './config-
|
|
9
|
-
import './types-
|
|
7
|
+
import './config-CNas6wJH.js';
|
|
8
|
+
import './config-bXxnJrQc.js';
|
|
9
|
+
import './types-BTIVO8DI.js';
|
|
10
10
|
import './types-D5OwhtsQ.js';
|
package/dist/client.mjs
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
// src/errors.ts
|
|
2
|
+
var NARSIL_ERROR_MARKER = /* @__PURE__ */ Symbol.for("@delali/narsil.NarsilError");
|
|
2
3
|
var ErrorCodes = {
|
|
3
4
|
SCHEMA_INVALID_TYPE: "SCHEMA_INVALID_TYPE",
|
|
4
5
|
SCHEMA_DEPTH_EXCEEDED: "SCHEMA_DEPTH_EXCEEDED",
|
|
@@ -16,6 +17,7 @@ var ErrorCodes = {
|
|
|
16
17
|
WORKER_CRASHED: "WORKER_CRASHED",
|
|
17
18
|
WORKER_BUSY: "WORKER_BUSY",
|
|
18
19
|
WORKER_TIMEOUT: "WORKER_TIMEOUT",
|
|
20
|
+
WORKER_ACTION_FAILED: "WORKER_ACTION_FAILED",
|
|
19
21
|
PERSISTENCE_SAVE_FAILED: "PERSISTENCE_SAVE_FAILED",
|
|
20
22
|
PERSISTENCE_LOAD_FAILED: "PERSISTENCE_LOAD_FAILED",
|
|
21
23
|
PERSISTENCE_DELETE_FAILED: "PERSISTENCE_DELETE_FAILED",
|
|
@@ -135,8 +137,12 @@ var NarsilError = class extends Error {
|
|
|
135
137
|
this.name = "NarsilError";
|
|
136
138
|
this.code = code;
|
|
137
139
|
this.details = details ?? {};
|
|
140
|
+
Object.defineProperty(this, NARSIL_ERROR_MARKER, { value: true });
|
|
138
141
|
}
|
|
139
142
|
};
|
|
143
|
+
function isNarsilError(value) {
|
|
144
|
+
return typeof value === "object" && value !== null && NARSIL_ERROR_MARKER in value;
|
|
145
|
+
}
|
|
140
146
|
function describeError(error) {
|
|
141
147
|
return error instanceof Error ? error.message : String(error);
|
|
142
148
|
}
|
|
@@ -188,7 +194,7 @@ function isRecord(value) {
|
|
|
188
194
|
}
|
|
189
195
|
function malformed(path, expected) {
|
|
190
196
|
return new NarsilError(ClientErrorCodes.CLIENT_INVALID_RESPONSE, `The answer from ${path} holds no ${expected}`, {
|
|
191
|
-
|
|
197
|
+
route: path
|
|
192
198
|
});
|
|
193
199
|
}
|
|
194
200
|
function readArray(payload, key, path) {
|
|
@@ -535,10 +541,10 @@ function errorFromBody(status, payload, url) {
|
|
|
535
541
|
const details = isRecord2(envelope.details) ? envelope.details : {};
|
|
536
542
|
return new NarsilError(envelope.code, envelope.message, { ...details, status });
|
|
537
543
|
}
|
|
538
|
-
function
|
|
544
|
+
function isHttpUrl(value) {
|
|
539
545
|
try {
|
|
540
|
-
new URL(value);
|
|
541
|
-
return
|
|
546
|
+
const parsed = new URL(value);
|
|
547
|
+
return parsed.protocol === "http:" || parsed.protocol === "https:";
|
|
542
548
|
} catch {
|
|
543
549
|
return false;
|
|
544
550
|
}
|
|
@@ -548,14 +554,14 @@ function normaliseBase(url) {
|
|
|
548
554
|
if (trimmed.length === 0) {
|
|
549
555
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'The client needs a server address, and "url" is empty');
|
|
550
556
|
}
|
|
551
|
-
if (!trimmed.startsWith("/") && !
|
|
557
|
+
if (!trimmed.startsWith("/") && !isHttpUrl(trimmed)) {
|
|
552
558
|
throw new NarsilError(
|
|
553
559
|
ErrorCodes.CONFIG_INVALID,
|
|
554
|
-
`The client cannot read "${trimmed}" as a server address; pass an
|
|
560
|
+
`The client cannot read "${trimmed}" as a server address; pass an http or https URL, or a path starting with "/" that a browser resolves against the page`,
|
|
555
561
|
{ url: trimmed }
|
|
556
562
|
);
|
|
557
563
|
}
|
|
558
|
-
return trimmed.
|
|
564
|
+
return trimmed.replace(/\/+$/, "");
|
|
559
565
|
}
|
|
560
566
|
function withQuery(path, query) {
|
|
561
567
|
if (query === void 0) return path;
|
|
@@ -596,7 +602,7 @@ function createTransport(options) {
|
|
|
596
602
|
const response = await send(url, init);
|
|
597
603
|
const wantsBytes = spec.binaryAnswer === true && response.ok;
|
|
598
604
|
const answer = await readAnswer(response, resolveResponseCeiling(spec), wantsBytes);
|
|
599
|
-
return { status: response.status, ...answer };
|
|
605
|
+
return { status: response.status, url, ...answer };
|
|
600
606
|
} catch (err) {
|
|
601
607
|
if (err instanceof ResponseTooLargeError) {
|
|
602
608
|
throw invalidResponse(`The server at ${url} answered more than this client reads: ${err.message}`, {
|
|
@@ -634,10 +640,10 @@ function createTransport(options) {
|
|
|
634
640
|
}
|
|
635
641
|
}
|
|
636
642
|
async function json(spec) {
|
|
637
|
-
const { status, text } = await exchange(spec);
|
|
638
|
-
const payload = parse(status, text,
|
|
643
|
+
const { status, text, url } = await exchange(spec);
|
|
644
|
+
const payload = parse(status, text, url);
|
|
639
645
|
if (status >= 200 && status < 300) return payload;
|
|
640
|
-
throw errorFromBody(status, payload,
|
|
646
|
+
throw errorFromBody(status, payload, url);
|
|
641
647
|
}
|
|
642
648
|
return {
|
|
643
649
|
json,
|
|
@@ -650,9 +656,9 @@ function createTransport(options) {
|
|
|
650
656
|
}
|
|
651
657
|
},
|
|
652
658
|
async binary(spec) {
|
|
653
|
-
const { status, text, bytes } = await exchange(spec);
|
|
659
|
+
const { status, text, bytes, url } = await exchange(spec);
|
|
654
660
|
if (bytes !== null) return bytes;
|
|
655
|
-
throw errorFromBody(status, parse(status, text,
|
|
661
|
+
throw errorFromBody(status, parse(status, text, url), url);
|
|
656
662
|
},
|
|
657
663
|
async probe(spec) {
|
|
658
664
|
const { status } = await exchange(spec);
|
|
@@ -866,4 +872,4 @@ function createNarsilClient(options) {
|
|
|
866
872
|
};
|
|
867
873
|
}
|
|
868
874
|
|
|
869
|
-
export { ASYNC_IMPORT_CAPABILITY, ClientErrorCodes, ErrorCodes, INDEX_LIFECYCLE_CAPABILITY, NarsilError, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, ServerErrorCodes, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, createNarsilClient };
|
|
875
|
+
export { ASYNC_IMPORT_CAPABILITY, ClientErrorCodes, ErrorCodes, INDEX_LIFECYCLE_CAPABILITY, NarsilError, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, ServerErrorCodes, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, createNarsilClient, isNarsilError };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { N as NarsilConfig } from './config-
|
|
2
|
-
import { A as AllocationTable, N as NodeRole, a as NodeRegistration, C as ClusterCoordinator, b as NodeCapacity } from './types-
|
|
1
|
+
import { N as NarsilConfig } from './config-bXxnJrQc.js';
|
|
2
|
+
import { A as AllocationTable, N as NodeRole, a as NodeRegistration, C as ClusterCoordinator, b as NodeCapacity } from './types-BTIVO8DI.js';
|
|
3
3
|
import { N as NodeTransport } from './types-D5OwhtsQ.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -1,44 +1,72 @@
|
|
|
1
1
|
import { P as PersistenceAdapter, I as InvalidationAdapter, E as EmbeddingAdapter } from './adapters-79OIa4xm.js';
|
|
2
|
-
import { Q as QueryParams, e as QueryResult } from './memory-
|
|
3
|
-
import { I as IndexConfig, A as AnyDocument } from './schema-
|
|
2
|
+
import { Q as QueryParams, e as QueryResult } from './memory-xrXOZMbw.js';
|
|
3
|
+
import { I as IndexConfig, A as AnyDocument } from './schema-mLpXZBEh.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Hooks that run alongside the engine's own work, which is how you add
|
|
7
7
|
* validation, auditing, enrichment, or metrics without forking the engine.
|
|
8
8
|
*
|
|
9
|
-
* Every hook is optional. A `before` hook
|
|
10
|
-
* by throwing
|
|
11
|
-
* awaits a hook that returns a promise, so slow work in
|
|
12
|
-
* or search it covers.
|
|
9
|
+
* Every hook is optional. A `before` hook fires first and rejects the
|
|
10
|
+
* operation by throwing, while an `after` hook fires once the operation
|
|
11
|
+
* succeeds. The engine awaits a hook that returns a promise, so slow work in
|
|
12
|
+
* one hook slows every write or search that it covers.
|
|
13
13
|
*
|
|
14
14
|
* @public
|
|
15
15
|
*/
|
|
16
16
|
interface NarsilPlugin {
|
|
17
17
|
/** This name identifies the plugin in errors and in event payloads. */
|
|
18
18
|
name: string;
|
|
19
|
-
/**
|
|
19
|
+
/** Fires before the engine indexes a document. Throw to reject the write. */
|
|
20
20
|
beforeInsert?(ctx: InsertContext): void | Promise<void>;
|
|
21
|
-
/**
|
|
21
|
+
/** Fires once the engine has indexed a document. */
|
|
22
22
|
afterInsert?(ctx: InsertContext): void | Promise<void>;
|
|
23
|
-
/**
|
|
23
|
+
/** Fires before the engine removes a document. Throw to reject the removal. */
|
|
24
24
|
beforeRemove?(ctx: RemoveContext): void | Promise<void>;
|
|
25
|
-
/**
|
|
25
|
+
/** Fires once the engine has removed a document. */
|
|
26
26
|
afterRemove?(ctx: RemoveContext): void | Promise<void>;
|
|
27
|
-
/**
|
|
27
|
+
/** Fires before the engine replaces a document. Throw to reject the update. */
|
|
28
28
|
beforeUpdate?(ctx: UpdateContext): void | Promise<void>;
|
|
29
|
-
/**
|
|
29
|
+
/** Fires once the engine has replaced a document. */
|
|
30
30
|
afterUpdate?(ctx: UpdateContext): void | Promise<void>;
|
|
31
|
-
/**
|
|
31
|
+
/**
|
|
32
|
+
* Fires before every search, and before every preflight. Throw to reject the
|
|
33
|
+
* query. It fires before the engine embeds a `vector.text` query, so a hook
|
|
34
|
+
* that counts or refuses work stops the call to the embedding provider, and
|
|
35
|
+
* `ctx.params.vector` still holds the text that the caller passed.
|
|
36
|
+
* {@link Narsil.suggest} scans the term dictionary and searches no document,
|
|
37
|
+
* so it fires no hook.
|
|
38
|
+
*/
|
|
32
39
|
beforeSearch?(ctx: SearchContext): void | Promise<void>;
|
|
33
|
-
/**
|
|
40
|
+
/**
|
|
41
|
+
* Fires once a search returns. The context holds a copy of the results, so a
|
|
42
|
+
* change that you make here leaves the caller's own result untouched. Where
|
|
43
|
+
* this hook throws, the engine logs a warning and answers the query all the
|
|
44
|
+
* same, because the search is already over.
|
|
45
|
+
*/
|
|
34
46
|
afterSearch?(ctx: SearchContext): void | Promise<void>;
|
|
35
|
-
/**
|
|
47
|
+
/**
|
|
48
|
+
* Fires once the engine creates an index, and a `restore` fires it too,
|
|
49
|
+
* because a restore recreates the index that it replaces. Where this hook
|
|
50
|
+
* throws, the engine logs a warning and keeps the index, because the hook
|
|
51
|
+
* fires after the index exists.
|
|
52
|
+
*/
|
|
36
53
|
onIndexCreate?(ctx: IndexContext): void | Promise<void>;
|
|
37
|
-
/**
|
|
54
|
+
/**
|
|
55
|
+
* Fires once the engine drops an index, and a `restore` fires it too,
|
|
56
|
+
* because a restore drops the index that it replaces. Where this hook
|
|
57
|
+
* throws, the engine logs a warning, because the hook fires after the index
|
|
58
|
+
* is gone.
|
|
59
|
+
*/
|
|
38
60
|
onIndexDrop?(ctx: IndexContext): void | Promise<void>;
|
|
39
|
-
/**
|
|
61
|
+
/**
|
|
62
|
+
* Fires once {@link Narsil.rebalance} moves an index onto a new partition
|
|
63
|
+
* count, which is the only way that the count changes. An index opens with
|
|
64
|
+
* the count that `partitions.maxPartitions` sets, because the engine adds no
|
|
65
|
+
* partition on its own. Subscribe to the `partitionWatermark` event to learn
|
|
66
|
+
* when an index fills its capacity.
|
|
67
|
+
*/
|
|
40
68
|
onPartitionSplit?(ctx: PartitionContext): void | Promise<void>;
|
|
41
|
-
/**
|
|
69
|
+
/** Fires once the engine loads worker copies of its indexes. */
|
|
42
70
|
onWorkerPromote?(ctx: WorkerContext): void | Promise<void>;
|
|
43
71
|
}
|
|
44
72
|
/**
|
|
@@ -62,7 +90,7 @@ interface InsertContext {
|
|
|
62
90
|
interface RemoveContext {
|
|
63
91
|
/** The engine removes the document from this index. */
|
|
64
92
|
indexName: string;
|
|
65
|
-
/**
|
|
93
|
+
/** The engine removes this document. */
|
|
66
94
|
docId: string;
|
|
67
95
|
}
|
|
68
96
|
/**
|
|
@@ -73,9 +101,9 @@ interface RemoveContext {
|
|
|
73
101
|
interface UpdateContext {
|
|
74
102
|
/** The document belongs to this index. */
|
|
75
103
|
indexName: string;
|
|
76
|
-
/**
|
|
104
|
+
/** The engine replaces this document. */
|
|
77
105
|
docId: string;
|
|
78
|
-
/** The index
|
|
106
|
+
/** The index holds this document before the update. */
|
|
79
107
|
oldDocument: AnyDocument;
|
|
80
108
|
/** This document replaces it, and the engine indexes whatever a `before` hook leaves here. */
|
|
81
109
|
newDocument: AnyDocument;
|
|
@@ -90,29 +118,33 @@ interface SearchContext {
|
|
|
90
118
|
indexName: string;
|
|
91
119
|
/** These are the parameters the query runs with, and a `before` hook that changes them rewrites the search. */
|
|
92
120
|
params: QueryParams;
|
|
93
|
-
/**
|
|
121
|
+
/**
|
|
122
|
+
* These are the results, which an `after` hook alone receives. A preflight
|
|
123
|
+
* fires `beforeSearch` and stops there, because it answers with a count and
|
|
124
|
+
* no hits. Read this field inside an `afterSearch` hook alone.
|
|
125
|
+
*/
|
|
94
126
|
results?: QueryResult;
|
|
95
127
|
}
|
|
96
128
|
/**
|
|
97
|
-
* What a plugin receives when
|
|
129
|
+
* What a plugin receives when the engine creates or drops an index.
|
|
98
130
|
*
|
|
99
131
|
* @public
|
|
100
132
|
*/
|
|
101
133
|
interface IndexContext {
|
|
102
|
-
/**
|
|
134
|
+
/** The engine creates or drops this index. */
|
|
103
135
|
indexName: string;
|
|
104
|
-
/** The index
|
|
136
|
+
/** The index holds this configuration. */
|
|
105
137
|
config: IndexConfig;
|
|
106
138
|
}
|
|
107
139
|
/**
|
|
108
|
-
* What a plugin receives when
|
|
140
|
+
* What a plugin receives when a rebalance changes an index's partition count.
|
|
109
141
|
*
|
|
110
142
|
* @public
|
|
111
143
|
*/
|
|
112
144
|
interface PartitionContext {
|
|
113
|
-
/**
|
|
145
|
+
/** The rebalance moved this index. */
|
|
114
146
|
indexName: string;
|
|
115
|
-
/** The index held this many partitions before. */
|
|
147
|
+
/** The index held this many partitions before the rebalance. */
|
|
116
148
|
oldPartitionCount: number;
|
|
117
149
|
/** The index holds this many partitions now. */
|
|
118
150
|
newPartitionCount: number;
|
|
@@ -123,9 +155,9 @@ interface PartitionContext {
|
|
|
123
155
|
* @public
|
|
124
156
|
*/
|
|
125
157
|
interface WorkerContext {
|
|
126
|
-
/** The pool
|
|
158
|
+
/** The pool holds this many workers. */
|
|
127
159
|
workerCount: number;
|
|
128
|
-
/** The engine promoted for this reason, such as the threshold the index
|
|
160
|
+
/** The engine promoted for this reason, such as the threshold that the index crossed. */
|
|
129
161
|
reason: string;
|
|
130
162
|
}
|
|
131
163
|
|
|
@@ -245,7 +277,11 @@ interface DurabilityConfig {
|
|
|
245
277
|
checkpointIntervalMs?: number;
|
|
246
278
|
/** This many mutations trigger a checkpoint before the interval elapses. */
|
|
247
279
|
checkpointMutationThreshold?: number;
|
|
248
|
-
/**
|
|
280
|
+
/**
|
|
281
|
+
* The engine merges a partition's checkpoint segments once that partition
|
|
282
|
+
* holds more than this many of them, 12 by default. The value is a whole
|
|
283
|
+
* number of segments.
|
|
284
|
+
*/
|
|
249
285
|
compactionThreshold?: number;
|
|
250
286
|
}
|
|
251
287
|
/**
|
|
@@ -280,9 +316,12 @@ interface WorkerConfig {
|
|
|
280
316
|
enabled?: boolean;
|
|
281
317
|
/**
|
|
282
318
|
* The keyword copies and the vector search pool share this many threads
|
|
283
|
-
* between them, half each, in an embedded engine
|
|
284
|
-
*
|
|
285
|
-
*
|
|
319
|
+
* between them, half each, in an embedded engine, so a budget of 4 gives 2
|
|
320
|
+
* keyword copies and 2 vector search workers. The HTTP server holds a copy
|
|
321
|
+
* on every one of them and receives requests there. The engine takes the
|
|
322
|
+
* host's cores minus one by default, between 2 and 8. It caps a value that
|
|
323
|
+
* you set at 31, which is how many scratch slots the shared vector arena
|
|
324
|
+
* reserves.
|
|
286
325
|
*/
|
|
287
326
|
count?: number;
|
|
288
327
|
/** An index gains worker copies once it holds this many documents, 1,000 by default. */
|
|
@@ -303,7 +342,13 @@ interface WorkerConfig {
|
|
|
303
342
|
* an engine that leaves this unset.
|
|
304
343
|
*/
|
|
305
344
|
mainCopyQueries?: MainCopyQueries;
|
|
306
|
-
/**
|
|
345
|
+
/**
|
|
346
|
+
* Each worker imports this module on start-up, which is how a worker reaches
|
|
347
|
+
* a custom tokeniser or language. Give an absolute path or a URL, such as
|
|
348
|
+
* `new URL('./bootstrap.js', import.meta.url).href`, because a worker
|
|
349
|
+
* resolves a relative specifier against Narsil's own directory. A relative
|
|
350
|
+
* value raises `CONFIG_INVALID`.
|
|
351
|
+
*/
|
|
307
352
|
bootstrapModule?: string;
|
|
308
353
|
}
|
|
309
354
|
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export { createDirectExecutor } from './chunk-XNQAIO74.mjs';
|
|
2
|
+
import './chunk-AV4IJKD2.mjs';
|
|
3
|
+
import './chunk-5T6AOR37.mjs';
|
|
4
|
+
import './chunk-L225SHNB.mjs';
|
|
5
|
+
import './chunk-TGP5M5R7.mjs';
|
|
6
|
+
import './chunk-U2AL3SON.mjs';
|
|
7
|
+
import './chunk-ZJ4CSAB2.mjs';
|
|
8
|
+
import './chunk-UTOFOJOR.mjs';
|
|
9
|
+
import './chunk-IFBKLBGB.mjs';
|
|
10
|
+
import './chunk-U2BKYEC7.mjs';
|
|
11
|
+
import './chunk-2GWVQWGA.mjs';
|
|
12
|
+
import './chunk-FRQVMFY7.mjs';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { MAX_NODE_ID_LENGTH, MAX_WATCHERS } from '../../chunk-ORBLD3EL.mjs';
|
|
2
2
|
import { compareCodePoints } from '../../chunk-2GWVQWGA.mjs';
|
|
3
|
-
import { NarsilError, ErrorCodes } from '../../chunk-
|
|
3
|
+
import { NarsilError, ErrorCodes } from '../../chunk-FRQVMFY7.mjs';
|
|
4
4
|
import { decode, encode } from '@msgpack/msgpack';
|
|
5
5
|
|
|
6
6
|
// src/distribution/coordinator/etcd/types.ts
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { decodeTransportMessage, encodeTransportMessage } from '../../chunk-OOSIDHA3.mjs';
|
|
2
2
|
import { MAX_MESSAGE_SIZE_BYTES, GRPC_KEEPALIVE_TIMEOUT_MS, GRPC_KEEPALIVE_TIME_MS, GRPC_ENVELOPE_OVERHEAD_BYTES } from '../../chunk-UNEEFTEU.mjs';
|
|
3
3
|
import { TransportError, TransportErrorCodes } from '../../chunk-GCM3JFVA.mjs';
|
|
4
|
-
import { NarsilError, ErrorCodes } from '../../chunk-
|
|
4
|
+
import { NarsilError, ErrorCodes } from '../../chunk-FRQVMFY7.mjs';
|
|
5
5
|
|
|
6
6
|
// src/distribution/transport/grpc/codec.ts
|
|
7
7
|
var FIELD_ONE_LENGTH_DELIMITED_KEY = 10;
|
package/dist/distribution.d.ts
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import { a as CreateIndexOptions, C as ClusterNamespace, b as ClusterNodeConfig } from './config-
|
|
2
|
-
export { c as ClusterControllerConfig, d as ClusterNodeInfo, e as ClusterQueryConfig, N as NodeReadiness, R as ReplicationConfig } from './config-
|
|
3
|
-
import { N as NarsilEventMap, a as Narsil } from './engine-
|
|
4
|
-
import { B as BatchResult, L as ListParams, g as ListResult,
|
|
5
|
-
import { I as IndexConfig, A as AnyDocument,
|
|
6
|
-
import { b as NodeCapacity, N as NodeRole } from './types-
|
|
7
|
-
export { c as AllocationConstraints, d as AllocationEvent, A as AllocationTable, C as ClusterCoordinator, e as NodeEvent, a as NodeRegistration, P as PartitionAssignment, f as PartitionState, S as SchemaEvent, U as UnassignedReason } from './types-
|
|
1
|
+
import { a as CreateIndexOptions, C as ClusterNamespace, b as ClusterNodeConfig } from './config-CNas6wJH.js';
|
|
2
|
+
export { c as ClusterControllerConfig, d as ClusterNodeInfo, e as ClusterQueryConfig, N as NodeReadiness, R as ReplicationConfig } from './config-CNas6wJH.js';
|
|
3
|
+
import { N as NarsilEventMap, a as Narsil } from './engine-Dj4I91Jv.js';
|
|
4
|
+
import { B as BatchResult, L as ListParams, g as ListResult, j as SuggestParams, k as SuggestResult, Q as QueryParams, h as PreflightResult, f as IndexStats, P as PartitionStatsResult, M as MemoryStats, e as QueryResult } from './memory-xrXOZMbw.js';
|
|
5
|
+
import { I as IndexConfig, A as AnyDocument, a as InsertOptions } from './schema-mLpXZBEh.js';
|
|
6
|
+
import { b as NodeCapacity, N as NodeRole } from './types-BTIVO8DI.js';
|
|
7
|
+
export { c as AllocationConstraints, d as AllocationEvent, A as AllocationTable, C as ClusterCoordinator, e as NodeEvent, a as NodeRegistration, P as PartitionAssignment, f as PartitionState, S as SchemaEvent, U as UnassignedReason } from './types-BTIVO8DI.js';
|
|
8
8
|
export { L as ListenHandler, N as NodeTransport, R as RespondFn, T as TransportConfig, a as TransportMessage } from './types-D5OwhtsQ.js';
|
|
9
|
-
import './config-
|
|
9
|
+
import './config-bXxnJrQc.js';
|
|
10
10
|
import './adapters-79OIa4xm.js';
|
|
11
11
|
|
|
12
12
|
/**
|