@delali/narsil 0.2.4-next.52 → 0.2.4-next.55
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -4
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/{capabilities-CuTSXvbN.d.ts → capabilities-DlYw0Ov0.d.ts} +13 -2
- package/dist/{chunk-U36IDJN7.mjs → chunk-4NVPQ6S5.mjs} +2 -2
- package/dist/{chunk-WM7IS45M.mjs → chunk-5T6AOR37.mjs} +2 -2
- package/dist/{chunk-ULJ36FK4.mjs → chunk-BH5SYB5W.mjs} +73 -46
- package/dist/{chunk-3GWP2TO6.mjs → chunk-D2S642IV.mjs} +4 -2
- package/dist/{chunk-P4ELXVJQ.mjs → chunk-FRQVMFY7.mjs} +7 -1
- package/dist/{chunk-ZT5BRDFW.mjs → chunk-L225SHNB.mjs} +2 -2
- package/dist/{chunk-72AXTIBW.mjs → chunk-NTZYA3SJ.mjs} +3 -3
- package/dist/{chunk-EY2ULUMT.mjs → chunk-RGS7FMCB.mjs} +25 -5
- package/dist/{chunk-YPYULM5E.mjs → chunk-TVT5V4E7.mjs} +84 -33
- package/dist/{chunk-T62YOSGS.mjs → chunk-U2AL3SON.mjs} +16 -98
- package/dist/chunk-UTOFOJOR.mjs +135 -0
- package/dist/{chunk-2FO7KZ6K.mjs → chunk-XNQAIO74.mjs} +53 -8
- package/dist/{client-BMJWVxOP.d.ts → client-CQUqt9Ir.d.ts} +13 -4
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +20 -14
- package/dist/{config-qMLpi5T4.d.ts → config-CNas6wJH.d.ts} +2 -2
- package/dist/{config-BcYZkdMw.d.ts → config-bXxnJrQc.d.ts} +80 -35
- package/dist/direct-executor-RBA5O7EN.mjs +12 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +12 -12
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/{engine-zhms1qId.d.ts → engine-Dj4I91Jv.d.ts} +21 -8
- package/dist/index.browser.mjs +169 -72
- package/dist/index.d.ts +7 -7
- package/dist/index.mjs +10 -10
- package/dist/{memory-OmgkX_Z2.d.ts → memory-xrXOZMbw.d.ts} +58 -13
- package/dist/persistence/durability/checkpoint-worker.d.ts +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +10 -10
- package/dist/react.d.ts +11 -11
- package/dist/react.mjs +16 -4
- package/dist/{schema-OSWaaZzx.d.ts → schema-mLpXZBEh.d.ts} +13 -6
- package/dist/server.d.ts +13 -10
- package/dist/server.mjs +68 -20
- package/dist/{tasks-DiF1c9v2.d.ts → tasks-Df73Tjgq.d.ts} +9 -5
- package/dist/{thread-TLCX43WV.mjs → thread-5UP6MGED.mjs} +8 -8
- package/dist/{types-LEzMyjVX.d.ts → types-BTIVO8DI.d.ts} +1 -1
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +3 -3
- package/dist/workers/entry.mjs +23 -10
- package/package.json +7 -7
- package/dist/chunk-WFKIGEBY.mjs +0 -11
- package/dist/direct-executor-UJC36S2D.mjs +0 -12
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { P as PartitionConfig, A as AnyDocument, a as InsertOptions, W as WriteOptions } from './schema-
|
|
2
|
-
import { V as VectorMaintenanceResult, B as BatchResult, L as ListParams, g as ListResult, I as IndexInfo, f as IndexStats, P as PartitionStatsResult, Q as QueryParams, e as QueryResult, h as PreflightResult,
|
|
3
|
-
import { e as TaskRecord, M as MemoryStatsResponse, b as ImportResult, H as HttpIndexConfig, c as TaskListQuery, T as TaskListPage } from './tasks-
|
|
1
|
+
import { P as PartitionConfig, A as AnyDocument, a as InsertOptions, W as WriteOptions } from './schema-mLpXZBEh.js';
|
|
2
|
+
import { V as VectorMaintenanceResult, B as BatchResult, L as ListParams, g as ListResult, I as IndexInfo, f as IndexStats, P as PartitionStatsResult, Q as QueryParams, e as QueryResult, h as PreflightResult, j as SuggestParams, k as SuggestResult } from './memory-xrXOZMbw.js';
|
|
3
|
+
import { e as TaskRecord, M as MemoryStatsResponse, b as ImportResult, H as HttpIndexConfig, c as TaskListQuery, T as TaskListPage } from './tasks-Df73Tjgq.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* This is the `fetch` implementation the client sends every request through.
|
|
@@ -596,9 +596,11 @@ interface SearchOperations {
|
|
|
596
596
|
* this request.
|
|
597
597
|
* @returns The result holds the hits, the total match count, and whatever
|
|
598
598
|
* else the query asked for.
|
|
599
|
-
* @throws A `NarsilError` with `SEARCH_RESULT_WINDOW_EXCEEDED`
|
|
599
|
+
* @throws A `NarsilError` with `SEARCH_RESULT_WINDOW_EXCEEDED` where `offset`
|
|
600
600
|
* plus `limit` passes the server's result window, which the cursor pages
|
|
601
601
|
* beyond.
|
|
602
|
+
* @throws A `NarsilError` with `INVALID_REQUEST` where `limit` alone passes
|
|
603
|
+
* `limits.maxFetchDocuments`, which the server checks before the window.
|
|
602
604
|
*/
|
|
603
605
|
query<T = AnyDocument>(indexName: string, params: QueryParams, options?: RequestOptions): Promise<QueryResult<T>>;
|
|
604
606
|
/**
|
|
@@ -644,6 +646,13 @@ interface ServerVersion {
|
|
|
644
646
|
gitSha: string | null;
|
|
645
647
|
/** This is true when the working tree held uncommitted changes at build time. */
|
|
646
648
|
dirty: boolean;
|
|
649
|
+
/**
|
|
650
|
+
* The server thread that answered searches vector graphs through this path.
|
|
651
|
+
* `native` is the native search core in C, and `wasm` is WebAssembly. Both
|
|
652
|
+
* paths return the same results. A server that predates this field leaves it
|
|
653
|
+
* out.
|
|
654
|
+
*/
|
|
655
|
+
vectorSearch?: 'native' | 'wasm';
|
|
647
656
|
}
|
|
648
657
|
/**
|
|
649
658
|
* These methods ask a server what it is and what it serves, before you ask it
|
package/dist/client.d.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes } from './memory-
|
|
2
|
-
export { A as ASYNC_IMPORT_CAPABILITY, E as ErrorEnvelope, I as INDEX_LIFECYCLE_CAPABILITY, R as REBUILD_ANALYSIS_CAPABILITY, S as SERVER_CAPABILITIES, T as TASK_CANCEL_CAPABILITY, a as TASK_FILTER_CAPABILITY } from './capabilities-
|
|
3
|
-
export { C as CapabilitiesResponse, a as CreateIndexEmbedding, H as HttpIndexConfig, I as ImportError, b as ImportResult, M as MemoryStatsResponse, T as TaskListPage, c as TaskListQuery, d as TaskProgress, e as TaskRecord, f as TaskStatus, g as TaskType } from './tasks-
|
|
4
|
-
export { A as AdminOperations, B as BulkOperations, D as DocumentOperations, F as FetchFunction, I as ImportSource, a as IndexOperations, N as NarsilClient, b as NarsilClientOptions, P as PutResult, R as RequestOptions, S as SearchOperations, c as ServerOperations, d as ServerVersion, T as TaskOperations, W as WaitForTaskOptions, e as createNarsilClient } from './client-
|
|
5
|
-
import './schema-
|
|
1
|
+
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, i as isNarsilError } from './memory-xrXOZMbw.js';
|
|
2
|
+
export { A as ASYNC_IMPORT_CAPABILITY, E as ErrorEnvelope, I as INDEX_LIFECYCLE_CAPABILITY, R as REBUILD_ANALYSIS_CAPABILITY, S as SERVER_CAPABILITIES, T as TASK_CANCEL_CAPABILITY, a as TASK_FILTER_CAPABILITY } from './capabilities-DlYw0Ov0.js';
|
|
3
|
+
export { C as CapabilitiesResponse, a as CreateIndexEmbedding, H as HttpIndexConfig, I as ImportError, b as ImportResult, M as MemoryStatsResponse, T as TaskListPage, c as TaskListQuery, d as TaskProgress, e as TaskRecord, f as TaskStatus, g as TaskType } from './tasks-Df73Tjgq.js';
|
|
4
|
+
export { A as AdminOperations, B as BulkOperations, D as DocumentOperations, F as FetchFunction, I as ImportSource, a as IndexOperations, N as NarsilClient, b as NarsilClientOptions, P as PutResult, R as RequestOptions, S as SearchOperations, c as ServerOperations, d as ServerVersion, T as TaskOperations, W as WaitForTaskOptions, e as createNarsilClient } from './client-CQUqt9Ir.js';
|
|
5
|
+
import './schema-mLpXZBEh.js';
|
|
6
6
|
import './adapters-79OIa4xm.js';
|
|
7
|
-
import './config-
|
|
8
|
-
import './config-
|
|
9
|
-
import './types-
|
|
7
|
+
import './config-CNas6wJH.js';
|
|
8
|
+
import './config-bXxnJrQc.js';
|
|
9
|
+
import './types-BTIVO8DI.js';
|
|
10
10
|
import './types-D5OwhtsQ.js';
|
package/dist/client.mjs
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
// src/errors.ts
|
|
2
|
+
var NARSIL_ERROR_MARKER = /* @__PURE__ */ Symbol.for("@delali/narsil.NarsilError");
|
|
2
3
|
var ErrorCodes = {
|
|
3
4
|
SCHEMA_INVALID_TYPE: "SCHEMA_INVALID_TYPE",
|
|
4
5
|
SCHEMA_DEPTH_EXCEEDED: "SCHEMA_DEPTH_EXCEEDED",
|
|
@@ -16,6 +17,7 @@ var ErrorCodes = {
|
|
|
16
17
|
WORKER_CRASHED: "WORKER_CRASHED",
|
|
17
18
|
WORKER_BUSY: "WORKER_BUSY",
|
|
18
19
|
WORKER_TIMEOUT: "WORKER_TIMEOUT",
|
|
20
|
+
WORKER_ACTION_FAILED: "WORKER_ACTION_FAILED",
|
|
19
21
|
PERSISTENCE_SAVE_FAILED: "PERSISTENCE_SAVE_FAILED",
|
|
20
22
|
PERSISTENCE_LOAD_FAILED: "PERSISTENCE_LOAD_FAILED",
|
|
21
23
|
PERSISTENCE_DELETE_FAILED: "PERSISTENCE_DELETE_FAILED",
|
|
@@ -135,8 +137,12 @@ var NarsilError = class extends Error {
|
|
|
135
137
|
this.name = "NarsilError";
|
|
136
138
|
this.code = code;
|
|
137
139
|
this.details = details ?? {};
|
|
140
|
+
Object.defineProperty(this, NARSIL_ERROR_MARKER, { value: true });
|
|
138
141
|
}
|
|
139
142
|
};
|
|
143
|
+
function isNarsilError(value) {
|
|
144
|
+
return typeof value === "object" && value !== null && NARSIL_ERROR_MARKER in value;
|
|
145
|
+
}
|
|
140
146
|
function describeError(error) {
|
|
141
147
|
return error instanceof Error ? error.message : String(error);
|
|
142
148
|
}
|
|
@@ -188,7 +194,7 @@ function isRecord(value) {
|
|
|
188
194
|
}
|
|
189
195
|
function malformed(path, expected) {
|
|
190
196
|
return new NarsilError(ClientErrorCodes.CLIENT_INVALID_RESPONSE, `The answer from ${path} holds no ${expected}`, {
|
|
191
|
-
|
|
197
|
+
route: path
|
|
192
198
|
});
|
|
193
199
|
}
|
|
194
200
|
function readArray(payload, key, path) {
|
|
@@ -535,10 +541,10 @@ function errorFromBody(status, payload, url) {
|
|
|
535
541
|
const details = isRecord2(envelope.details) ? envelope.details : {};
|
|
536
542
|
return new NarsilError(envelope.code, envelope.message, { ...details, status });
|
|
537
543
|
}
|
|
538
|
-
function
|
|
544
|
+
function isHttpUrl(value) {
|
|
539
545
|
try {
|
|
540
|
-
new URL(value);
|
|
541
|
-
return
|
|
546
|
+
const parsed = new URL(value);
|
|
547
|
+
return parsed.protocol === "http:" || parsed.protocol === "https:";
|
|
542
548
|
} catch {
|
|
543
549
|
return false;
|
|
544
550
|
}
|
|
@@ -548,14 +554,14 @@ function normaliseBase(url) {
|
|
|
548
554
|
if (trimmed.length === 0) {
|
|
549
555
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'The client needs a server address, and "url" is empty');
|
|
550
556
|
}
|
|
551
|
-
if (!trimmed.startsWith("/") && !
|
|
557
|
+
if (!trimmed.startsWith("/") && !isHttpUrl(trimmed)) {
|
|
552
558
|
throw new NarsilError(
|
|
553
559
|
ErrorCodes.CONFIG_INVALID,
|
|
554
|
-
`The client cannot read "${trimmed}" as a server address; pass an
|
|
560
|
+
`The client cannot read "${trimmed}" as a server address; pass an http or https URL, or a path starting with "/" that a browser resolves against the page`,
|
|
555
561
|
{ url: trimmed }
|
|
556
562
|
);
|
|
557
563
|
}
|
|
558
|
-
return trimmed.
|
|
564
|
+
return trimmed.replace(/\/+$/, "");
|
|
559
565
|
}
|
|
560
566
|
function withQuery(path, query) {
|
|
561
567
|
if (query === void 0) return path;
|
|
@@ -596,7 +602,7 @@ function createTransport(options) {
|
|
|
596
602
|
const response = await send(url, init);
|
|
597
603
|
const wantsBytes = spec.binaryAnswer === true && response.ok;
|
|
598
604
|
const answer = await readAnswer(response, resolveResponseCeiling(spec), wantsBytes);
|
|
599
|
-
return { status: response.status, ...answer };
|
|
605
|
+
return { status: response.status, url, ...answer };
|
|
600
606
|
} catch (err) {
|
|
601
607
|
if (err instanceof ResponseTooLargeError) {
|
|
602
608
|
throw invalidResponse(`The server at ${url} answered more than this client reads: ${err.message}`, {
|
|
@@ -634,10 +640,10 @@ function createTransport(options) {
|
|
|
634
640
|
}
|
|
635
641
|
}
|
|
636
642
|
async function json(spec) {
|
|
637
|
-
const { status, text } = await exchange(spec);
|
|
638
|
-
const payload = parse(status, text,
|
|
643
|
+
const { status, text, url } = await exchange(spec);
|
|
644
|
+
const payload = parse(status, text, url);
|
|
639
645
|
if (status >= 200 && status < 300) return payload;
|
|
640
|
-
throw errorFromBody(status, payload,
|
|
646
|
+
throw errorFromBody(status, payload, url);
|
|
641
647
|
}
|
|
642
648
|
return {
|
|
643
649
|
json,
|
|
@@ -650,9 +656,9 @@ function createTransport(options) {
|
|
|
650
656
|
}
|
|
651
657
|
},
|
|
652
658
|
async binary(spec) {
|
|
653
|
-
const { status, text, bytes } = await exchange(spec);
|
|
659
|
+
const { status, text, bytes, url } = await exchange(spec);
|
|
654
660
|
if (bytes !== null) return bytes;
|
|
655
|
-
throw errorFromBody(status, parse(status, text,
|
|
661
|
+
throw errorFromBody(status, parse(status, text, url), url);
|
|
656
662
|
},
|
|
657
663
|
async probe(spec) {
|
|
658
664
|
const { status } = await exchange(spec);
|
|
@@ -866,4 +872,4 @@ function createNarsilClient(options) {
|
|
|
866
872
|
};
|
|
867
873
|
}
|
|
868
874
|
|
|
869
|
-
export { ASYNC_IMPORT_CAPABILITY, ClientErrorCodes, ErrorCodes, INDEX_LIFECYCLE_CAPABILITY, NarsilError, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, ServerErrorCodes, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, createNarsilClient };
|
|
875
|
+
export { ASYNC_IMPORT_CAPABILITY, ClientErrorCodes, ErrorCodes, INDEX_LIFECYCLE_CAPABILITY, NarsilError, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, ServerErrorCodes, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, createNarsilClient, isNarsilError };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { N as NarsilConfig } from './config-
|
|
2
|
-
import { A as AllocationTable, N as NodeRole, a as NodeRegistration, C as ClusterCoordinator, b as NodeCapacity } from './types-
|
|
1
|
+
import { N as NarsilConfig } from './config-bXxnJrQc.js';
|
|
2
|
+
import { A as AllocationTable, N as NodeRole, a as NodeRegistration, C as ClusterCoordinator, b as NodeCapacity } from './types-BTIVO8DI.js';
|
|
3
3
|
import { N as NodeTransport } from './types-D5OwhtsQ.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -1,44 +1,72 @@
|
|
|
1
1
|
import { P as PersistenceAdapter, I as InvalidationAdapter, E as EmbeddingAdapter } from './adapters-79OIa4xm.js';
|
|
2
|
-
import { Q as QueryParams, e as QueryResult } from './memory-
|
|
3
|
-
import { I as IndexConfig, A as AnyDocument } from './schema-
|
|
2
|
+
import { Q as QueryParams, e as QueryResult } from './memory-xrXOZMbw.js';
|
|
3
|
+
import { I as IndexConfig, A as AnyDocument } from './schema-mLpXZBEh.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Hooks that run alongside the engine's own work, which is how you add
|
|
7
7
|
* validation, auditing, enrichment, or metrics without forking the engine.
|
|
8
8
|
*
|
|
9
|
-
* Every hook is optional. A `before` hook
|
|
10
|
-
* by throwing
|
|
11
|
-
* awaits a hook that returns a promise, so slow work in
|
|
12
|
-
* or search it covers.
|
|
9
|
+
* Every hook is optional. A `before` hook fires first and rejects the
|
|
10
|
+
* operation by throwing, while an `after` hook fires once the operation
|
|
11
|
+
* succeeds. The engine awaits a hook that returns a promise, so slow work in
|
|
12
|
+
* one hook slows every write or search that it covers.
|
|
13
13
|
*
|
|
14
14
|
* @public
|
|
15
15
|
*/
|
|
16
16
|
interface NarsilPlugin {
|
|
17
17
|
/** This name identifies the plugin in errors and in event payloads. */
|
|
18
18
|
name: string;
|
|
19
|
-
/**
|
|
19
|
+
/** Fires before the engine indexes a document. Throw to reject the write. */
|
|
20
20
|
beforeInsert?(ctx: InsertContext): void | Promise<void>;
|
|
21
|
-
/**
|
|
21
|
+
/** Fires once the engine has indexed a document. */
|
|
22
22
|
afterInsert?(ctx: InsertContext): void | Promise<void>;
|
|
23
|
-
/**
|
|
23
|
+
/** Fires before the engine removes a document. Throw to reject the removal. */
|
|
24
24
|
beforeRemove?(ctx: RemoveContext): void | Promise<void>;
|
|
25
|
-
/**
|
|
25
|
+
/** Fires once the engine has removed a document. */
|
|
26
26
|
afterRemove?(ctx: RemoveContext): void | Promise<void>;
|
|
27
|
-
/**
|
|
27
|
+
/** Fires before the engine replaces a document. Throw to reject the update. */
|
|
28
28
|
beforeUpdate?(ctx: UpdateContext): void | Promise<void>;
|
|
29
|
-
/**
|
|
29
|
+
/** Fires once the engine has replaced a document. */
|
|
30
30
|
afterUpdate?(ctx: UpdateContext): void | Promise<void>;
|
|
31
|
-
/**
|
|
31
|
+
/**
|
|
32
|
+
* Fires before every search, and before every preflight. Throw to reject the
|
|
33
|
+
* query. It fires before the engine embeds a `vector.text` query, so a hook
|
|
34
|
+
* that counts or refuses work stops the call to the embedding provider, and
|
|
35
|
+
* `ctx.params.vector` still holds the text that the caller passed.
|
|
36
|
+
* {@link Narsil.suggest} scans the term dictionary and searches no document,
|
|
37
|
+
* so it fires no hook.
|
|
38
|
+
*/
|
|
32
39
|
beforeSearch?(ctx: SearchContext): void | Promise<void>;
|
|
33
|
-
/**
|
|
40
|
+
/**
|
|
41
|
+
* Fires once a search returns. The context holds a copy of the results, so a
|
|
42
|
+
* change that you make here leaves the caller's own result untouched. Where
|
|
43
|
+
* this hook throws, the engine logs a warning and answers the query all the
|
|
44
|
+
* same, because the search is already over.
|
|
45
|
+
*/
|
|
34
46
|
afterSearch?(ctx: SearchContext): void | Promise<void>;
|
|
35
|
-
/**
|
|
47
|
+
/**
|
|
48
|
+
* Fires once the engine creates an index, and a `restore` fires it too,
|
|
49
|
+
* because a restore recreates the index that it replaces. Where this hook
|
|
50
|
+
* throws, the engine logs a warning and keeps the index, because the hook
|
|
51
|
+
* fires after the index exists.
|
|
52
|
+
*/
|
|
36
53
|
onIndexCreate?(ctx: IndexContext): void | Promise<void>;
|
|
37
|
-
/**
|
|
54
|
+
/**
|
|
55
|
+
* Fires once the engine drops an index, and a `restore` fires it too,
|
|
56
|
+
* because a restore drops the index that it replaces. Where this hook
|
|
57
|
+
* throws, the engine logs a warning, because the hook fires after the index
|
|
58
|
+
* is gone.
|
|
59
|
+
*/
|
|
38
60
|
onIndexDrop?(ctx: IndexContext): void | Promise<void>;
|
|
39
|
-
/**
|
|
61
|
+
/**
|
|
62
|
+
* Fires once {@link Narsil.rebalance} moves an index onto a new partition
|
|
63
|
+
* count, which is the only way that the count changes. An index opens with
|
|
64
|
+
* the count that `partitions.maxPartitions` sets, because the engine adds no
|
|
65
|
+
* partition on its own. Subscribe to the `partitionWatermark` event to learn
|
|
66
|
+
* when an index fills its capacity.
|
|
67
|
+
*/
|
|
40
68
|
onPartitionSplit?(ctx: PartitionContext): void | Promise<void>;
|
|
41
|
-
/**
|
|
69
|
+
/** Fires once the engine loads worker copies of its indexes. */
|
|
42
70
|
onWorkerPromote?(ctx: WorkerContext): void | Promise<void>;
|
|
43
71
|
}
|
|
44
72
|
/**
|
|
@@ -62,7 +90,7 @@ interface InsertContext {
|
|
|
62
90
|
interface RemoveContext {
|
|
63
91
|
/** The engine removes the document from this index. */
|
|
64
92
|
indexName: string;
|
|
65
|
-
/**
|
|
93
|
+
/** The engine removes this document. */
|
|
66
94
|
docId: string;
|
|
67
95
|
}
|
|
68
96
|
/**
|
|
@@ -73,9 +101,9 @@ interface RemoveContext {
|
|
|
73
101
|
interface UpdateContext {
|
|
74
102
|
/** The document belongs to this index. */
|
|
75
103
|
indexName: string;
|
|
76
|
-
/**
|
|
104
|
+
/** The engine replaces this document. */
|
|
77
105
|
docId: string;
|
|
78
|
-
/** The index
|
|
106
|
+
/** The index holds this document before the update. */
|
|
79
107
|
oldDocument: AnyDocument;
|
|
80
108
|
/** This document replaces it, and the engine indexes whatever a `before` hook leaves here. */
|
|
81
109
|
newDocument: AnyDocument;
|
|
@@ -90,29 +118,33 @@ interface SearchContext {
|
|
|
90
118
|
indexName: string;
|
|
91
119
|
/** These are the parameters the query runs with, and a `before` hook that changes them rewrites the search. */
|
|
92
120
|
params: QueryParams;
|
|
93
|
-
/**
|
|
121
|
+
/**
|
|
122
|
+
* These are the results, which an `after` hook alone receives. A preflight
|
|
123
|
+
* fires `beforeSearch` and stops there, because it answers with a count and
|
|
124
|
+
* no hits. Read this field inside an `afterSearch` hook alone.
|
|
125
|
+
*/
|
|
94
126
|
results?: QueryResult;
|
|
95
127
|
}
|
|
96
128
|
/**
|
|
97
|
-
* What a plugin receives when
|
|
129
|
+
* What a plugin receives when the engine creates or drops an index.
|
|
98
130
|
*
|
|
99
131
|
* @public
|
|
100
132
|
*/
|
|
101
133
|
interface IndexContext {
|
|
102
|
-
/**
|
|
134
|
+
/** The engine creates or drops this index. */
|
|
103
135
|
indexName: string;
|
|
104
|
-
/** The index
|
|
136
|
+
/** The index holds this configuration. */
|
|
105
137
|
config: IndexConfig;
|
|
106
138
|
}
|
|
107
139
|
/**
|
|
108
|
-
* What a plugin receives when
|
|
140
|
+
* What a plugin receives when a rebalance changes an index's partition count.
|
|
109
141
|
*
|
|
110
142
|
* @public
|
|
111
143
|
*/
|
|
112
144
|
interface PartitionContext {
|
|
113
|
-
/**
|
|
145
|
+
/** The rebalance moved this index. */
|
|
114
146
|
indexName: string;
|
|
115
|
-
/** The index held this many partitions before. */
|
|
147
|
+
/** The index held this many partitions before the rebalance. */
|
|
116
148
|
oldPartitionCount: number;
|
|
117
149
|
/** The index holds this many partitions now. */
|
|
118
150
|
newPartitionCount: number;
|
|
@@ -123,9 +155,9 @@ interface PartitionContext {
|
|
|
123
155
|
* @public
|
|
124
156
|
*/
|
|
125
157
|
interface WorkerContext {
|
|
126
|
-
/** The pool
|
|
158
|
+
/** The pool holds this many workers. */
|
|
127
159
|
workerCount: number;
|
|
128
|
-
/** The engine promoted for this reason, such as the threshold the index
|
|
160
|
+
/** The engine promoted for this reason, such as the threshold that the index crossed. */
|
|
129
161
|
reason: string;
|
|
130
162
|
}
|
|
131
163
|
|
|
@@ -245,7 +277,11 @@ interface DurabilityConfig {
|
|
|
245
277
|
checkpointIntervalMs?: number;
|
|
246
278
|
/** This many mutations trigger a checkpoint before the interval elapses. */
|
|
247
279
|
checkpointMutationThreshold?: number;
|
|
248
|
-
/**
|
|
280
|
+
/**
|
|
281
|
+
* The engine merges a partition's checkpoint segments once that partition
|
|
282
|
+
* holds more than this many of them, 12 by default. The value is a whole
|
|
283
|
+
* number of segments.
|
|
284
|
+
*/
|
|
249
285
|
compactionThreshold?: number;
|
|
250
286
|
}
|
|
251
287
|
/**
|
|
@@ -280,9 +316,12 @@ interface WorkerConfig {
|
|
|
280
316
|
enabled?: boolean;
|
|
281
317
|
/**
|
|
282
318
|
* The keyword copies and the vector search pool share this many threads
|
|
283
|
-
* between them, half each, in an embedded engine
|
|
284
|
-
*
|
|
285
|
-
*
|
|
319
|
+
* between them, half each, in an embedded engine, so a budget of 4 gives 2
|
|
320
|
+
* keyword copies and 2 vector search workers. The HTTP server holds a copy
|
|
321
|
+
* on every one of them and receives requests there. The engine takes the
|
|
322
|
+
* host's cores minus one by default, between 2 and 8. It caps a value that
|
|
323
|
+
* you set at 31, which is how many scratch slots the shared vector arena
|
|
324
|
+
* reserves.
|
|
286
325
|
*/
|
|
287
326
|
count?: number;
|
|
288
327
|
/** An index gains worker copies once it holds this many documents, 1,000 by default. */
|
|
@@ -303,7 +342,13 @@ interface WorkerConfig {
|
|
|
303
342
|
* an engine that leaves this unset.
|
|
304
343
|
*/
|
|
305
344
|
mainCopyQueries?: MainCopyQueries;
|
|
306
|
-
/**
|
|
345
|
+
/**
|
|
346
|
+
* Each worker imports this module on start-up, which is how a worker reaches
|
|
347
|
+
* a custom tokeniser or language. Give an absolute path or a URL, such as
|
|
348
|
+
* `new URL('./bootstrap.js', import.meta.url).href`, because a worker
|
|
349
|
+
* resolves a relative specifier against Narsil's own directory. A relative
|
|
350
|
+
* value raises `CONFIG_INVALID`.
|
|
351
|
+
*/
|
|
307
352
|
bootstrapModule?: string;
|
|
308
353
|
}
|
|
309
354
|
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export { createDirectExecutor } from './chunk-XNQAIO74.mjs';
|
|
2
|
+
import './chunk-AV4IJKD2.mjs';
|
|
3
|
+
import './chunk-5T6AOR37.mjs';
|
|
4
|
+
import './chunk-L225SHNB.mjs';
|
|
5
|
+
import './chunk-TGP5M5R7.mjs';
|
|
6
|
+
import './chunk-U2AL3SON.mjs';
|
|
7
|
+
import './chunk-ZJ4CSAB2.mjs';
|
|
8
|
+
import './chunk-UTOFOJOR.mjs';
|
|
9
|
+
import './chunk-IFBKLBGB.mjs';
|
|
10
|
+
import './chunk-U2BKYEC7.mjs';
|
|
11
|
+
import './chunk-2GWVQWGA.mjs';
|
|
12
|
+
import './chunk-FRQVMFY7.mjs';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { MAX_NODE_ID_LENGTH, MAX_WATCHERS } from '../../chunk-ORBLD3EL.mjs';
|
|
2
2
|
import { compareCodePoints } from '../../chunk-2GWVQWGA.mjs';
|
|
3
|
-
import { NarsilError, ErrorCodes } from '../../chunk-
|
|
3
|
+
import { NarsilError, ErrorCodes } from '../../chunk-FRQVMFY7.mjs';
|
|
4
4
|
import { decode, encode } from '@msgpack/msgpack';
|
|
5
5
|
|
|
6
6
|
// src/distribution/coordinator/etcd/types.ts
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { decodeTransportMessage, encodeTransportMessage } from '../../chunk-OOSIDHA3.mjs';
|
|
2
2
|
import { MAX_MESSAGE_SIZE_BYTES, GRPC_KEEPALIVE_TIMEOUT_MS, GRPC_KEEPALIVE_TIME_MS, GRPC_ENVELOPE_OVERHEAD_BYTES } from '../../chunk-UNEEFTEU.mjs';
|
|
3
3
|
import { TransportError, TransportErrorCodes } from '../../chunk-GCM3JFVA.mjs';
|
|
4
|
-
import { NarsilError, ErrorCodes } from '../../chunk-
|
|
4
|
+
import { NarsilError, ErrorCodes } from '../../chunk-FRQVMFY7.mjs';
|
|
5
5
|
|
|
6
6
|
// src/distribution/transport/grpc/codec.ts
|
|
7
7
|
var FIELD_ONE_LENGTH_DELIMITED_KEY = 10;
|
package/dist/distribution.d.ts
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import { a as CreateIndexOptions, C as ClusterNamespace, b as ClusterNodeConfig } from './config-
|
|
2
|
-
export { c as ClusterControllerConfig, d as ClusterNodeInfo, e as ClusterQueryConfig, N as NodeReadiness, R as ReplicationConfig } from './config-
|
|
3
|
-
import { N as NarsilEventMap, a as Narsil } from './engine-
|
|
4
|
-
import { B as BatchResult, L as ListParams, g as ListResult,
|
|
5
|
-
import { I as IndexConfig, A as AnyDocument, a as InsertOptions } from './schema-
|
|
6
|
-
import { b as NodeCapacity, N as NodeRole } from './types-
|
|
7
|
-
export { c as AllocationConstraints, d as AllocationEvent, A as AllocationTable, C as ClusterCoordinator, e as NodeEvent, a as NodeRegistration, P as PartitionAssignment, f as PartitionState, S as SchemaEvent, U as UnassignedReason } from './types-
|
|
1
|
+
import { a as CreateIndexOptions, C as ClusterNamespace, b as ClusterNodeConfig } from './config-CNas6wJH.js';
|
|
2
|
+
export { c as ClusterControllerConfig, d as ClusterNodeInfo, e as ClusterQueryConfig, N as NodeReadiness, R as ReplicationConfig } from './config-CNas6wJH.js';
|
|
3
|
+
import { N as NarsilEventMap, a as Narsil } from './engine-Dj4I91Jv.js';
|
|
4
|
+
import { B as BatchResult, L as ListParams, g as ListResult, j as SuggestParams, k as SuggestResult, Q as QueryParams, h as PreflightResult, f as IndexStats, P as PartitionStatsResult, M as MemoryStats, e as QueryResult } from './memory-xrXOZMbw.js';
|
|
5
|
+
import { I as IndexConfig, A as AnyDocument, a as InsertOptions } from './schema-mLpXZBEh.js';
|
|
6
|
+
import { b as NodeCapacity, N as NodeRole } from './types-BTIVO8DI.js';
|
|
7
|
+
export { c as AllocationConstraints, d as AllocationEvent, A as AllocationTable, C as ClusterCoordinator, e as NodeEvent, a as NodeRegistration, P as PartitionAssignment, f as PartitionState, S as SchemaEvent, U as UnassignedReason } from './types-BTIVO8DI.js';
|
|
8
8
|
export { L as ListenHandler, N as NodeTransport, R as RespondFn, T as TransportConfig, a as TransportMessage } from './types-D5OwhtsQ.js';
|
|
9
|
-
import './config-
|
|
9
|
+
import './config-bXxnJrQc.js';
|
|
10
10
|
import './adapters-79OIa4xm.js';
|
|
11
11
|
|
|
12
12
|
/**
|
package/dist/distribution.mjs
CHANGED
|
@@ -1,24 +1,24 @@
|
|
|
1
1
|
import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
|
|
2
|
-
import { validateIndexName, indexConfigKey, putIndexMetadata, getIndexMetadata } from './chunk-
|
|
3
|
-
import { createEngineCore, checkHeapAfterRecovery, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, applyIndexEntry, applyDeleteEntry, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, computeEntryChecksum, createEntryMessage, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-
|
|
2
|
+
import { validateIndexName, indexConfigKey, putIndexMetadata, getIndexMetadata } from './chunk-NTZYA3SJ.mjs';
|
|
3
|
+
import { createEngineCore, checkHeapAfterRecovery, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, applyIndexEntry, applyDeleteEntry, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, computeEntryChecksum, createEntryMessage, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-BH5SYB5W.mjs';
|
|
4
4
|
import { ReplicationMessageTypes, TransportErrorCodes, ClusterMessageTypes, QueryMessageTypes, TransportError } from './chunk-GCM3JFVA.mjs';
|
|
5
|
-
import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET } from './chunk-
|
|
6
|
-
import { generateId, deserializePayloadV2, createPartitionIndex, getLanguage, FILTER_EXPRESSION_KEYS, FIELD_FILTER_OPERATORS } from './chunk-
|
|
5
|
+
import { providedDocId, MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET } from './chunk-4NVPQ6S5.mjs';
|
|
6
|
+
import { generateId, deserializePayloadV2, createPartitionIndex, getLanguage, FILTER_EXPRESSION_KEYS, FIELD_FILTER_OPERATORS } from './chunk-XNQAIO74.mjs';
|
|
7
7
|
import './chunk-AV4IJKD2.mjs';
|
|
8
|
-
import { now, listBindingOf, queryBindingOf, foldGroupReducer, clampLimit, dedupePinnedEntries } from './chunk-
|
|
9
|
-
import { DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_INDEX_NAME_LENGTH, MAX_DOC_ID_LENGTH, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, INDEX_NAME_PATTERN } from './chunk-
|
|
10
|
-
import { resolveProjection, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, normalizeSort, encodePageCursor, applyProjection, readSortValues, requireWithinResultWindow, fnv1a, mergePartitionStats } from './chunk-
|
|
11
|
-
import { DEFAULT_PAGE_SIZE, MAX_SORT_FIELDS, MAX_CURSOR_LENGTH, RESULT_WINDOW, MAX_NESTING_DEPTH } from './chunk-
|
|
8
|
+
import { now, listBindingOf, queryBindingOf, foldGroupReducer, clampLimit, dedupePinnedEntries } from './chunk-TVT5V4E7.mjs';
|
|
9
|
+
import { DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_INDEX_NAME_LENGTH, MAX_DOC_ID_LENGTH, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, INDEX_NAME_PATTERN } from './chunk-D2S642IV.mjs';
|
|
10
|
+
import { resolveProjection, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, normalizeSort, encodePageCursor, applyProjection, readSortValues, requireWithinResultWindow, fnv1a, mergePartitionStats } from './chunk-5T6AOR37.mjs';
|
|
11
|
+
import { DEFAULT_PAGE_SIZE, MAX_SORT_FIELDS, MAX_CURSOR_LENGTH, RESULT_WINDOW, MAX_NESTING_DEPTH } from './chunk-L225SHNB.mjs';
|
|
12
12
|
import './chunk-TGP5M5R7.mjs';
|
|
13
|
-
import './chunk-
|
|
13
|
+
import './chunk-U2AL3SON.mjs';
|
|
14
14
|
import { MIN_OVERSAMPLE } from './chunk-ZJ4CSAB2.mjs';
|
|
15
|
-
import './chunk-
|
|
15
|
+
import './chunk-UTOFOJOR.mjs';
|
|
16
16
|
import { crc32 } from './chunk-3L4U6EJQ.mjs';
|
|
17
17
|
import './chunk-IFBKLBGB.mjs';
|
|
18
18
|
import './chunk-U2BKYEC7.mjs';
|
|
19
19
|
import { compareCodePoints, toComparableSortValue, compareComparableKeys, compareSortValues } from './chunk-2GWVQWGA.mjs';
|
|
20
20
|
import './chunk-POWOTUKJ.mjs';
|
|
21
|
-
import { ErrorCodes, NarsilError, describeError } from './chunk-
|
|
21
|
+
import { ErrorCodes, NarsilError, describeError } from './chunk-FRQVMFY7.mjs';
|
|
22
22
|
import { encode, decode } from '@msgpack/msgpack';
|
|
23
23
|
|
|
24
24
|
// src/distribution/cluster/constants.ts
|
|
@@ -9204,7 +9204,7 @@ async function routeDropIndex(name, coordinator, engine) {
|
|
|
9204
9204
|
await coordinator.dropSchema(name);
|
|
9205
9205
|
}
|
|
9206
9206
|
async function routeInsert(indexName, document, docId, deps, options) {
|
|
9207
|
-
const resolvedDocId = docId ?? generateId();
|
|
9207
|
+
const resolvedDocId = docId ?? providedDocId(document) ?? generateId();
|
|
9208
9208
|
const resolution = await resolvePrimaryAssignment(indexName, resolvedDocId, deps, false);
|
|
9209
9209
|
if (resolution === null) {
|
|
9210
9210
|
return deps.engine.insert(indexName, document, resolvedDocId, options);
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { E as EmbeddingAdapter } from './adapters-79OIa4xm.js';
|
|
2
|
-
import { B as BatchResult, I as IndexInfo, f as IndexStats, P as PartitionStatsResult, L as ListParams, g as ListResult, Q as QueryParams, e as QueryResult, h as PreflightResult,
|
|
3
|
-
import { A as AnyDocument, a as InsertOptions, W as WriteOptions, I as IndexConfig, P as PartitionConfig } from './schema-
|
|
2
|
+
import { B as BatchResult, I as IndexInfo, f as IndexStats, P as PartitionStatsResult, L as ListParams, g as ListResult, Q as QueryParams, e as QueryResult, h as PreflightResult, j as SuggestParams, k as SuggestResult, M as MemoryStats, V as VectorMaintenanceResult } from './memory-xrXOZMbw.js';
|
|
3
|
+
import { A as AnyDocument, a as InsertOptions, W as WriteOptions, I as IndexConfig, P as PartitionConfig } from './schema-mLpXZBEh.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Every event the engine emits, with the payload each one carries.
|
|
@@ -68,17 +68,23 @@ type NarsilEventMap = {
|
|
|
68
68
|
partitionCount: number;
|
|
69
69
|
};
|
|
70
70
|
/**
|
|
71
|
-
* The process
|
|
71
|
+
* The process uses nine tenths of its heap, measured during a write or a
|
|
72
72
|
* load, so the next large index may end the process with an out-of-memory
|
|
73
|
-
* error.
|
|
74
|
-
*
|
|
73
|
+
* error. Where `--max-old-space-size` or `--max-old-space-size-percentage`
|
|
74
|
+
* sets a limit, on the command line or in `NODE_OPTIONS`, the engine measures
|
|
75
|
+
* the used bytes against that figure and `heapLimit` reports it. Where
|
|
76
|
+
* neither flag sets one, the engine compares the used bytes with the headroom
|
|
77
|
+
* that V8 reports, which runs about 192 MB above the true ceiling, so a
|
|
78
|
+
* default heap below about 2 GB may end before this event fires. The engine
|
|
79
|
+
* emits the event once per crossing and arms it again once the headroom
|
|
80
|
+
* recovers to two tenths. Raise the limit with
|
|
75
81
|
* `--max-old-space-size-percentage` or `--max-old-space-size`, or close an
|
|
76
82
|
* idle index; see {@link ProcessMemoryReport.heapLimit}.
|
|
77
83
|
*/
|
|
78
84
|
heapPressure: {
|
|
79
|
-
/** The heap
|
|
85
|
+
/** The heap reaches that point during a write to this index or a load of it. */
|
|
80
86
|
indexName: string;
|
|
81
|
-
/** The process
|
|
87
|
+
/** The process uses this much heap, in bytes. */
|
|
82
88
|
heapUsed: number;
|
|
83
89
|
/** The heap may grow to this many bytes. */
|
|
84
90
|
heapLimit: number;
|
|
@@ -155,7 +161,9 @@ interface DocumentWriteOperations {
|
|
|
155
161
|
* @param indexName - The index that receives the document.
|
|
156
162
|
* @param document - Its fields must match the types the schema declares.
|
|
157
163
|
* @param docId - Pass an id to control it yourself, or omit it and read the
|
|
158
|
-
* returned value.
|
|
164
|
+
* returned value. An id that differs from the document's own `id` field
|
|
165
|
+
* raises `DOC_VALIDATION_FAILED`, because the stored document would
|
|
166
|
+
* otherwise contradict the key that it is stored under.
|
|
159
167
|
* @param options - Per-write settings, such as skipping the defensive copy
|
|
160
168
|
* or waiting for every worker copy to apply the write.
|
|
161
169
|
* @returns The id the document is stored under.
|
|
@@ -294,6 +302,11 @@ interface Narsil extends IndexLifecycleOperations, DocumentWriteOperations {
|
|
|
294
302
|
/**
|
|
295
303
|
* Fetches one stored document by id, without searching.
|
|
296
304
|
*
|
|
305
|
+
* A vector field comes back as a `Float32Array`, whatever form you give it.
|
|
306
|
+
* `JSON.stringify` writes one as an object keyed by index, so convert it
|
|
307
|
+
* with `Array.from`, or project the field away, before you serialise the
|
|
308
|
+
* document yourself. The HTTP routes convert it for you.
|
|
309
|
+
*
|
|
297
310
|
* @param indexName - The index holding the document.
|
|
298
311
|
* @param docId - The document to fetch.
|
|
299
312
|
* @returns The document, or `undefined` when the index holds no such id.
|