@mnmal-ai/substrate 0.0.0 → 1.0.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +118 -1
- package/dist/agent-token-BIboWOU0.js +200 -0
- package/dist/client.d.ts +782 -0
- package/dist/client.js +7 -0
- package/dist/cluster-worker.js +35 -0
- package/dist/content-type-C7T28xNb.js +25 -0
- package/dist/dataset-DOXGRz50.js +1535 -0
- package/dist/env-utils-B7SGmoeJ.js +21 -0
- package/dist/errors-D4vKuwz8.d.ts +130 -0
- package/dist/fake-DUVtKVSE.js +36 -0
- package/dist/index-D7mSOFlM.d.ts +69 -0
- package/dist/pack.d.ts +192 -0
- package/dist/pack.js +5 -0
- package/dist/plugin-BuQQEzL-.js +3844 -0
- package/dist/plugin-t2g2wX0b.d.ts +1568 -0
- package/dist/proposals-bootstrap.js +37 -0
- package/dist/proposals.config.schema.json +20 -0
- package/dist/proposals.d.ts +118 -0
- package/dist/proposals.default-config.json +4 -0
- package/dist/proposals.js +4 -0
- package/dist/recall-bootstrap.js +451 -0
- package/dist/recall.config.schema.json +232 -0
- package/dist/recall.d.ts +1981 -0
- package/dist/recall.default-config.json +4 -0
- package/dist/recall.js +5 -0
- package/dist/repl-C52wyfMs.js +2213 -0
- package/dist/row-ByLFE2Wx.d.ts +719 -0
- package/dist/row-DK7lPZVe.js +732 -0
- package/dist/runtime-metrics-DwpDJ8vz.d.ts +159 -0
- package/dist/schema-g4FQLr0V.d.ts +433 -0
- package/dist/sdk.d.ts +694 -0
- package/dist/sdk.js +112 -0
- package/dist/src-5hVbeiAu.js +1797 -0
- package/dist/src-C4DzaIQZ.js +837 -0
- package/dist/src-CjEwiKXw.js +2327 -0
- package/dist/src-DuopV8u7.js +2408 -0
- package/dist/src-YXlv_RyL.js +13524 -0
- package/dist/src-zcyUofMJ.js +1015 -0
- package/dist/substrate-admin.js +808 -0
- package/dist/substrate.js +5482 -0
- package/dist/testing.d.ts +1240 -0
- package/dist/testing.js +702 -0
- package/dist/types-8c0gtnd2.d.ts +35 -0
- package/dist/ws-C72iVS5T.d.ts +143 -0
- package/package.json +117 -4
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
|
|
2
|
+
import { Pt as defineBootstrap } from "./src-YXlv_RyL.js";
|
|
3
|
+
import { n as createNoopPubSub, t as createProposalsPlugin } from "./src-5hVbeiAu.js";
|
|
4
|
+
import { z } from "zod";
|
|
5
|
+
|
|
6
|
+
//#region ../../plugins/substrate-proposals/src/bootstrap.ts
|
|
7
|
+
/**
|
|
8
|
+
* Structured-config bootstrap for `@mnmal-ai/substrate-proposals`.
|
|
9
|
+
*
|
|
10
|
+
* The canonical entry consumed by the binary's `SUBSTRATE_PLUGIN_DIR` loader.
|
|
11
|
+
*
|
|
12
|
+
* `pubsub` and `datasetFor` are intentionally NOT in the schema — they're
|
|
13
|
+
* code-shaped:
|
|
14
|
+
* - `pubsub` defaults to no-op (single-instance).
|
|
15
|
+
* - `datasetFor` stays undefined; the plugin's `onStart` resolves it off
|
|
16
|
+
* `EngineHandle.datasetFor`.
|
|
17
|
+
*
|
|
18
|
+
* The committed `config.schema.json` (used by `package.json#substrate` for
|
|
19
|
+
* ops-surface declaration) is locked against `z.toJSONSchema(Schema)` by
|
|
20
|
+
* `test/schema-parity.test.ts` — drift fails the test and the schema can be
|
|
21
|
+
* regenerated via `UPDATE_SCHEMA=1 pnpm test`.
|
|
22
|
+
*/
|
|
23
|
+
const ProposalsBootstrapConfigSchema = z.object({
|
|
24
|
+
ownedNamespaces: z.array(z.string().min(1)).default([]),
|
|
25
|
+
proposeCapability: z.string().min(1).default("substrate.propose")
|
|
26
|
+
}).strict();
|
|
27
|
+
const bootstrap = defineBootstrap.coordinator({
|
|
28
|
+
schema: ProposalsBootstrapConfigSchema,
|
|
29
|
+
build: (config) => createProposalsPlugin({
|
|
30
|
+
ownedNamespaces: config.ownedNamespaces,
|
|
31
|
+
pubsub: createNoopPubSub(),
|
|
32
|
+
proposeCapability: config.proposeCapability
|
|
33
|
+
})
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
//#endregion
|
|
37
|
+
export { bootstrap as default };
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"type": "object",
|
|
4
|
+
"properties": {
|
|
5
|
+
"ownedNamespaces": {
|
|
6
|
+
"default": [],
|
|
7
|
+
"type": "array",
|
|
8
|
+
"items": {
|
|
9
|
+
"type": "string",
|
|
10
|
+
"minLength": 1
|
|
11
|
+
}
|
|
12
|
+
},
|
|
13
|
+
"proposeCapability": {
|
|
14
|
+
"default": "substrate.propose",
|
|
15
|
+
"type": "string",
|
|
16
|
+
"minLength": 1
|
|
17
|
+
}
|
|
18
|
+
},
|
|
19
|
+
"additionalProperties": false
|
|
20
|
+
}
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import { v as OnDelete } from "./schema-g4FQLr0V.js";
|
|
2
|
+
import { ct as Dataset, t as CoordinatorPlugin } from "./plugin-t2g2wX0b.js";
|
|
3
|
+
import { a as ResolverFrameError } from "./errors-D4vKuwz8.js";
|
|
4
|
+
//#region ../../plugins/substrate-proposals/src/config.d.ts
|
|
5
|
+
/**
|
|
6
|
+
* Multi-instance invalidation transport used by the plugin to broadcast
|
|
7
|
+
* TypeProposal and FieldProposal application across instances.
|
|
8
|
+
*
|
|
9
|
+
* `createNoopPubSub()` serves tests and single-instance deployments; no
|
|
10
|
+
* multi-instance (e.g. pg LISTEN/NOTIFY) implementation ships in this package.
|
|
11
|
+
*/
|
|
12
|
+
interface PubSub {
|
|
13
|
+
/**
|
|
14
|
+
* Subscribe to `channel`. Handler is invoked with the raw NOTIFY payload
|
|
15
|
+
* (a string, typically a proposal id). Implementations MAY reconnect and
|
|
16
|
+
* re-invoke subscriptions; handlers must be idempotent.
|
|
17
|
+
*/
|
|
18
|
+
listen(channel: string, handler: (payload: string) => void | Promise<void>): Promise<void>;
|
|
19
|
+
/**
|
|
20
|
+
* Emit `payload` on `channel`. Best-effort; pg's transactional NOTIFY is
|
|
21
|
+
* preferred (emit inside the row-insert tx) but fire-and-forget post-commit
|
|
22
|
+
* is acceptable given reconcile-on-restart is the correctness floor.
|
|
23
|
+
*/
|
|
24
|
+
notify(channel: string, payload: string): Promise<void>;
|
|
25
|
+
}
|
|
26
|
+
interface ProposalsPluginConfig {
|
|
27
|
+
/**
|
|
28
|
+
* Namespaces this plugin owns. Propose mutations outside this list are
|
|
29
|
+
* rejected at resolver entry. A proposed crossRef must target a type in an
|
|
30
|
+
* owned (or hot-added) namespace.
|
|
31
|
+
*/
|
|
32
|
+
ownedNamespaces: string[];
|
|
33
|
+
/**
|
|
34
|
+
* Multi-instance invalidation transport. Required: pass a no-op stub in
|
|
35
|
+
* single-instance deployments (e.g. `createNoopPubSub()`).
|
|
36
|
+
*/
|
|
37
|
+
pubsub: PubSub;
|
|
38
|
+
/**
|
|
39
|
+
* Capability name required to invoke propose_* mutations. Defaults to
|
|
40
|
+
* `'substrate.propose'`. Override only when a deployment needs to partition
|
|
41
|
+
* propose authority under a differently-named cap.
|
|
42
|
+
*/
|
|
43
|
+
proposeCapability?: string;
|
|
44
|
+
/**
|
|
45
|
+
* Lookup function the `propose_namespace` resolver uses to find the dataset
|
|
46
|
+
* for a hot-added namespace. The host (typically the substrate binary) wires
|
|
47
|
+
* this to its dataset registry. Returning `undefined` causes the resolver
|
|
48
|
+
* to throw `dataset_not_provisioned` — substrate selection is the host's
|
|
49
|
+
* responsibility, so the host MUST register the dataset before calling
|
|
50
|
+
* `propose_namespace`.
|
|
51
|
+
*/
|
|
52
|
+
datasetFor?: (ns: string) => Dataset | undefined;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Convenience stub for tests and single-instance deployments where multi-
|
|
56
|
+
* instance invalidation is not needed.
|
|
57
|
+
*/
|
|
58
|
+
export declare function createNoopPubSub(): PubSub;
|
|
59
|
+
//#endregion
|
|
60
|
+
//#region ../../plugins/substrate-proposals/src/errors.d.ts
|
|
61
|
+
type ProposalErrorCode = 'type_already_exists' | 'type_already_proposed' | 'field_already_exists' | 'destructive_change_required' | 'unknown_type' | 'unowned_namespace' | 'invalid_field_spec' | 'invalid_params' | 'invalid_namespace' | 'namespace_already_proposed' | 'namespace_retired' | 'dataset_not_provisioned' | 'engine_not_wired';
|
|
62
|
+
/**
|
|
63
|
+
* Structured error for proposal operations with stable error codes.
|
|
64
|
+
*
|
|
65
|
+
* Supports wire serialization via toJSON() for consistent error handling
|
|
66
|
+
* across HTTP boundaries.
|
|
67
|
+
*/
|
|
68
|
+
export declare class ProposalError extends ResolverFrameError {
|
|
69
|
+
readonly name = "ProposalError";
|
|
70
|
+
readonly code: ProposalErrorCode;
|
|
71
|
+
readonly details: Record<string, unknown>;
|
|
72
|
+
constructor(code: ProposalErrorCode, message: string, details?: Record<string, unknown>, options?: {
|
|
73
|
+
cause?: unknown;
|
|
74
|
+
});
|
|
75
|
+
toJSON(): {
|
|
76
|
+
error: ProposalErrorCode;
|
|
77
|
+
message: string;
|
|
78
|
+
details: Record<string, unknown>;
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
//#endregion
|
|
82
|
+
//#region ../../plugins/substrate-proposals/src/field-spec-json.d.ts
|
|
83
|
+
type ScalarType = 'string' | 'number' | 'boolean' | 'date' | 'json';
|
|
84
|
+
interface ScalarSpec {
|
|
85
|
+
kind: 'scalar';
|
|
86
|
+
type: ScalarType;
|
|
87
|
+
indexed?: boolean;
|
|
88
|
+
nullable?: boolean;
|
|
89
|
+
readonly?: boolean;
|
|
90
|
+
enum?: readonly string[];
|
|
91
|
+
aliases?: Record<string, string>;
|
|
92
|
+
strict?: boolean;
|
|
93
|
+
implies?: Record<string, Record<string, unknown>>;
|
|
94
|
+
}
|
|
95
|
+
interface CrossRefSpec {
|
|
96
|
+
kind: 'crossRef';
|
|
97
|
+
ns: string;
|
|
98
|
+
type: string;
|
|
99
|
+
onDelete?: OnDelete;
|
|
100
|
+
indexed?: boolean;
|
|
101
|
+
nullable?: boolean;
|
|
102
|
+
}
|
|
103
|
+
interface StructSpec {
|
|
104
|
+
kind: 'struct';
|
|
105
|
+
fields: Record<string, FieldSpecJson>;
|
|
106
|
+
nullable?: boolean;
|
|
107
|
+
}
|
|
108
|
+
interface ListSpec {
|
|
109
|
+
kind: 'list';
|
|
110
|
+
of: FieldSpecJson;
|
|
111
|
+
nullable?: boolean;
|
|
112
|
+
}
|
|
113
|
+
type FieldSpecJson = ScalarSpec | CrossRefSpec | StructSpec | ListSpec;
|
|
114
|
+
//#endregion
|
|
115
|
+
//#region ../../plugins/substrate-proposals/src/index.d.ts
|
|
116
|
+
export declare function createProposalsPlugin(config: ProposalsPluginConfig): CoordinatorPlugin;
|
|
117
|
+
//#endregion
|
|
118
|
+
export type { CrossRefSpec, FieldSpecJson, ListSpec, ProposalsPluginConfig, PubSub, ScalarSpec, StructSpec };
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
|
|
2
|
+
import { n as createNoopPubSub, r as ProposalError, t as createProposalsPlugin } from "./src-5hVbeiAu.js";
|
|
3
|
+
|
|
4
|
+
export { ProposalError, createNoopPubSub, createProposalsPlugin };
|
|
@@ -0,0 +1,451 @@
|
|
|
1
|
+
import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
|
|
2
|
+
import { Pt as defineBootstrap } from "./src-YXlv_RyL.js";
|
|
3
|
+
import { t as FakeEmbedder } from "./fake-DUVtKVSE.js";
|
|
4
|
+
import { t as createRecallPlugin } from "./plugin-BuQQEzL-.js";
|
|
5
|
+
import { z } from "zod";
|
|
6
|
+
|
|
7
|
+
//#region ../../packages/substrate-embedder/src/types.ts
|
|
8
|
+
var EmbedError = class extends Error {
|
|
9
|
+
code;
|
|
10
|
+
name = "EmbedError";
|
|
11
|
+
constructor(code, message, options) {
|
|
12
|
+
super(message, options);
|
|
13
|
+
this.code = code;
|
|
14
|
+
}
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
//#endregion
|
|
18
|
+
//#region ../../packages/substrate-embedder/src/ollama.ts
|
|
19
|
+
/**
|
|
20
|
+
* OllamaEmbedder — sends `POST {baseUrl}/api/embed` with the modern Ollama
|
|
21
|
+
* request shape `{ model, input, truncate }`. Returns the first row of the
|
|
22
|
+
* `embeddings` array as Float32Array. Validates dimension. AbortController-
|
|
23
|
+
* backed timeout. Throws `EmbedError` on every failure path so callers see a
|
|
24
|
+
* typed error with a discriminated code.
|
|
25
|
+
*
|
|
26
|
+
* Use `/api/embed`, not the legacy `/api/embeddings`: the legacy endpoint
|
|
27
|
+
* returns 200 with a degenerate vector for inputs past the context window
|
|
28
|
+
* instead of truncating. `/api/embed` also returns L2-normalised vectors.
|
|
29
|
+
*
|
|
30
|
+
* Every request carries `num_ctx` and `num_batch` set to the model's context
|
|
31
|
+
* length. Without them Ollama truncates at its default batch size (2048
|
|
32
|
+
* tokens), not at the model's context, and says nothing.
|
|
33
|
+
*/
|
|
34
|
+
var OllamaEmbedder = class {
|
|
35
|
+
modelId;
|
|
36
|
+
dimension;
|
|
37
|
+
baseUrl;
|
|
38
|
+
model;
|
|
39
|
+
timeoutMs;
|
|
40
|
+
contextLength;
|
|
41
|
+
constructor(opts) {
|
|
42
|
+
this.baseUrl = opts.baseUrl ?? "http://localhost:11434";
|
|
43
|
+
this.model = opts.model;
|
|
44
|
+
this.dimension = opts.dimension;
|
|
45
|
+
this.timeoutMs = opts.timeoutMs ?? 12e4;
|
|
46
|
+
this.modelId = `${opts.model}:${opts.dimension}`;
|
|
47
|
+
if (opts.contextLength !== void 0) this.contextLength = Promise.resolve(opts.contextLength);
|
|
48
|
+
}
|
|
49
|
+
async embed(text, opts) {
|
|
50
|
+
const timeoutMs = opts?.timeoutMs ?? this.timeoutMs;
|
|
51
|
+
const contextLength = await this.resolveContextLength(timeoutMs);
|
|
52
|
+
const url = `${this.baseUrl.replace(/\/+$/, "")}/api/embed`;
|
|
53
|
+
const controller = new AbortController();
|
|
54
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
55
|
+
let res;
|
|
56
|
+
try {
|
|
57
|
+
res = await fetch(url, {
|
|
58
|
+
method: "POST",
|
|
59
|
+
headers: { "content-type": "application/json" },
|
|
60
|
+
body: JSON.stringify({
|
|
61
|
+
model: this.model,
|
|
62
|
+
input: text,
|
|
63
|
+
truncate: true,
|
|
64
|
+
options: {
|
|
65
|
+
num_ctx: contextLength,
|
|
66
|
+
num_batch: contextLength
|
|
67
|
+
}
|
|
68
|
+
}),
|
|
69
|
+
signal: controller.signal
|
|
70
|
+
});
|
|
71
|
+
} catch (err) {
|
|
72
|
+
throw transportError(url, controller, timeoutMs, err);
|
|
73
|
+
} finally {
|
|
74
|
+
clearTimeout(timer);
|
|
75
|
+
}
|
|
76
|
+
if (!res.ok) {
|
|
77
|
+
const preview = await res.text().catch(() => "<unreadable>");
|
|
78
|
+
throw new EmbedError("unavailable", `OllamaEmbedder: POST ${url} returned ${res.status}: ${preview.slice(0, 200)}`);
|
|
79
|
+
}
|
|
80
|
+
let body;
|
|
81
|
+
try {
|
|
82
|
+
body = await res.json();
|
|
83
|
+
} catch (err) {
|
|
84
|
+
throw new EmbedError("invalid_response", `OllamaEmbedder: response body was not valid JSON: ${err instanceof Error ? err.message : String(err)}`, { cause: err });
|
|
85
|
+
}
|
|
86
|
+
const embedding = this.extractEmbedding(body);
|
|
87
|
+
if (embedding.length !== this.dimension) throw new EmbedError("invalid_response", `OllamaEmbedder: expected dimension ${this.dimension}, got ${embedding.length}`);
|
|
88
|
+
return Float32Array.from(embedding);
|
|
89
|
+
}
|
|
90
|
+
/** The model's context length, from the constructor or one `/api/show` call. */
|
|
91
|
+
resolveContextLength(timeoutMs) {
|
|
92
|
+
if (this.contextLength === void 0) {
|
|
93
|
+
this.contextLength = this.fetchContextLength(timeoutMs);
|
|
94
|
+
this.contextLength.catch(() => {
|
|
95
|
+
this.contextLength = void 0;
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
return this.contextLength;
|
|
99
|
+
}
|
|
100
|
+
async fetchContextLength(timeoutMs) {
|
|
101
|
+
const url = `${this.baseUrl.replace(/\/+$/, "")}/api/show`;
|
|
102
|
+
const controller = new AbortController();
|
|
103
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
104
|
+
let body;
|
|
105
|
+
try {
|
|
106
|
+
const res = await fetch(url, {
|
|
107
|
+
method: "POST",
|
|
108
|
+
headers: { "content-type": "application/json" },
|
|
109
|
+
body: JSON.stringify({ model: this.model }),
|
|
110
|
+
signal: controller.signal
|
|
111
|
+
});
|
|
112
|
+
if (!res.ok) {
|
|
113
|
+
const preview = await res.text().catch(() => "<unreadable>");
|
|
114
|
+
throw new EmbedError("unavailable", `OllamaEmbedder: POST ${url} returned ${res.status}: ${preview.slice(0, 200)}`);
|
|
115
|
+
}
|
|
116
|
+
body = await res.json();
|
|
117
|
+
} catch (err) {
|
|
118
|
+
if (err instanceof EmbedError) throw err;
|
|
119
|
+
throw transportError(url, controller, timeoutMs, err);
|
|
120
|
+
} finally {
|
|
121
|
+
clearTimeout(timer);
|
|
122
|
+
}
|
|
123
|
+
const info = body?.model_info;
|
|
124
|
+
if (info !== null && typeof info === "object") {
|
|
125
|
+
for (const [key, value] of Object.entries(info)) if (key.endsWith(".context_length") && typeof value === "number" && value > 0) return value;
|
|
126
|
+
}
|
|
127
|
+
throw new EmbedError("invalid_response", `OllamaEmbedder: ${url} reported no context length for '${this.model}'; set contextLength explicitly`);
|
|
128
|
+
}
|
|
129
|
+
extractEmbedding(body) {
|
|
130
|
+
if (body === null || typeof body !== "object") throw new EmbedError("invalid_response", "OllamaEmbedder: response body was not an object");
|
|
131
|
+
const embeddings = body.embeddings;
|
|
132
|
+
if (!Array.isArray(embeddings)) throw new EmbedError("invalid_response", "OllamaEmbedder: response missing 'embeddings' array");
|
|
133
|
+
if (embeddings.length === 0) throw new EmbedError("invalid_response", "OllamaEmbedder: response contained empty embeddings (model returned no vector — input may exceed the context window with truncation disabled)");
|
|
134
|
+
const first = embeddings[0];
|
|
135
|
+
if (!Array.isArray(first) || first.some((v) => typeof v !== "number")) throw new EmbedError("invalid_response", "OllamaEmbedder: response 'embeddings[0]' was not a numeric array");
|
|
136
|
+
return first;
|
|
137
|
+
}
|
|
138
|
+
};
|
|
139
|
+
/** `unavailable` for a failed request, naming the timeout when it was the abort. */
|
|
140
|
+
function transportError(url, controller, timeoutMs, err) {
|
|
141
|
+
const detail = controller.signal.aborted ? `timed out after ${timeoutMs}ms` : `failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
142
|
+
return new EmbedError("unavailable", `OllamaEmbedder: POST ${url} ${detail}`, { cause: err });
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
//#endregion
|
|
146
|
+
//#region ../../packages/substrate-embedder/src/onnx.ts
|
|
147
|
+
/**
|
|
148
|
+
* OnnxEmbedder — in-process embedder via fastembed (which uses
|
|
149
|
+
* onnxruntime-node + Hugging Face tokenizers under the hood).
|
|
150
|
+
*
|
|
151
|
+
* Runs the model in-process, avoiding the HTTP hop, serialisation and
|
|
152
|
+
* scheduler queue of OllamaEmbedder.
|
|
153
|
+
*
|
|
154
|
+
* `fastembed` is deliberately NOT declared — not as a dependency, and not as
|
|
155
|
+
* an optional peer: it pulls `onnxruntime-node` (~520MB), and GitHub Packages'
|
|
156
|
+
* packument drops `peerDependenciesMeta`, so pnpm would treat an optional peer
|
|
157
|
+
* as required and install it anyway. The import is deferred into `getEngine()`
|
|
158
|
+
* so this module loads, type-checks and constructs on a host without it; only
|
|
159
|
+
* a caller that actually embeds gets an error. The supported range lives in
|
|
160
|
+
* the error message below and in the README.
|
|
161
|
+
*
|
|
162
|
+
* Trade-offs:
|
|
163
|
+
* - Requires `pnpm add fastembed` alongside substrate-embedder.
|
|
164
|
+
* - First embed() blocks on a one-time model download, cached at `cacheDir`.
|
|
165
|
+
* - Native module (onnxruntime-node) — can't be inlined by the bundler; must
|
|
166
|
+
* be installed alongside the bundled substrate binary or kept external.
|
|
167
|
+
* - Models are the fastembed-curated set (see `MODEL_MAP`).
|
|
168
|
+
*/
|
|
169
|
+
/**
|
|
170
|
+
* Short name → fastembed `EmbeddingModel` value. The enum is a string enum, so
|
|
171
|
+
* these literals ARE its values; spelling them out is what lets the mapping be
|
|
172
|
+
* validated in the constructor without loading fastembed. Pinned by
|
|
173
|
+
* `test/onnx-undeclared-backend.test.ts`, which compares these literals to the
|
|
174
|
+
* real enum and fails if either side moves.
|
|
175
|
+
*/
|
|
176
|
+
const MODEL_MAP = {
|
|
177
|
+
"all-MiniLM-L6-v2": "fast-all-MiniLM-L6-v2",
|
|
178
|
+
"bge-base-en": "fast-bge-base-en",
|
|
179
|
+
"bge-base-en-v1.5": "fast-bge-base-en-v1.5",
|
|
180
|
+
"bge-small-en": "fast-bge-small-en",
|
|
181
|
+
"bge-small-en-v1.5": "fast-bge-small-en-v1.5",
|
|
182
|
+
"multilingual-e5-large": "fast-multilingual-e5-large"
|
|
183
|
+
};
|
|
184
|
+
/** fastembed's `ExecutionProvider.CPU`. Same string-enum reasoning as MODEL_MAP. */
|
|
185
|
+
const EXECUTION_PROVIDER_CPU = "cpu";
|
|
186
|
+
var OnnxEmbedder = class {
|
|
187
|
+
modelId;
|
|
188
|
+
dimension;
|
|
189
|
+
modelKey;
|
|
190
|
+
cacheDir;
|
|
191
|
+
maxLength;
|
|
192
|
+
engine = null;
|
|
193
|
+
initPromise = null;
|
|
194
|
+
constructor(opts) {
|
|
195
|
+
const mapped = MODEL_MAP[opts.model];
|
|
196
|
+
if (mapped === void 0) throw new Error(`OnnxEmbedder: unknown model '${opts.model}'. Supported: ${Object.keys(MODEL_MAP).join(", ")}`);
|
|
197
|
+
this.modelKey = mapped;
|
|
198
|
+
this.dimension = opts.dimension;
|
|
199
|
+
this.cacheDir = opts.cacheDir;
|
|
200
|
+
this.maxLength = opts.maxLength ?? 512;
|
|
201
|
+
this.modelId = `${opts.model}:${opts.dimension}`;
|
|
202
|
+
}
|
|
203
|
+
/**
|
|
204
|
+
* `opts.timeoutMs` bounds the wait, not the work: in-process inference
|
|
205
|
+
* cannot be cancelled, so a timed-out embed still finishes in the background.
|
|
206
|
+
*/
|
|
207
|
+
async embed(text, opts) {
|
|
208
|
+
const timeoutMs = opts?.timeoutMs;
|
|
209
|
+
if (timeoutMs === void 0) return this.embedNow(text);
|
|
210
|
+
let timer;
|
|
211
|
+
const timeout = new Promise((_, reject) => {
|
|
212
|
+
timer = setTimeout(() => reject(new EmbedError("unavailable", `OnnxEmbedder: timed out after ${timeoutMs}ms`)), timeoutMs);
|
|
213
|
+
});
|
|
214
|
+
try {
|
|
215
|
+
return await Promise.race([this.embedNow(text), timeout]);
|
|
216
|
+
} finally {
|
|
217
|
+
clearTimeout(timer);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
async embedNow(text) {
|
|
221
|
+
const generator = (await this.getEngine()).embed([text], 1);
|
|
222
|
+
let first;
|
|
223
|
+
for await (const batch of generator) {
|
|
224
|
+
first = batch[0];
|
|
225
|
+
break;
|
|
226
|
+
}
|
|
227
|
+
if (first === void 0) throw new EmbedError("invalid_response", "OnnxEmbedder: fastembed yielded no batches");
|
|
228
|
+
if (first.length !== this.dimension) throw new EmbedError("invalid_response", `OnnxEmbedder: expected ${this.dimension} dimensions, got ${first.length}`);
|
|
229
|
+
return new Float32Array(first);
|
|
230
|
+
}
|
|
231
|
+
/**
|
|
232
|
+
* Lazy-init the underlying engine. First call imports fastembed,
|
|
233
|
+
* and may then block on a model download; subsequent calls return the same
|
|
234
|
+
* engine. Concurrent first calls share one `initPromise` so neither the
|
|
235
|
+
* import nor the download is duplicated.
|
|
236
|
+
*/
|
|
237
|
+
async getEngine() {
|
|
238
|
+
if (this.engine !== null) return this.engine;
|
|
239
|
+
if (this.initPromise === null) this.initPromise = this.init().catch((err) => {
|
|
240
|
+
this.initPromise = null;
|
|
241
|
+
throw err;
|
|
242
|
+
});
|
|
243
|
+
this.engine = await this.initPromise;
|
|
244
|
+
return this.engine;
|
|
245
|
+
}
|
|
246
|
+
async init() {
|
|
247
|
+
const { FlagEmbedding } = await loadFastembed();
|
|
248
|
+
const initOpts = {
|
|
249
|
+
model: this.modelKey,
|
|
250
|
+
maxLength: this.maxLength,
|
|
251
|
+
executionProviders: [EXECUTION_PROVIDER_CPU]
|
|
252
|
+
};
|
|
253
|
+
if (this.cacheDir !== void 0) initOpts.cacheDir = this.cacheDir;
|
|
254
|
+
try {
|
|
255
|
+
return await FlagEmbedding.init(initOpts);
|
|
256
|
+
} catch (err) {
|
|
257
|
+
throw new EmbedError("unavailable", `OnnxEmbedder: failed to initialize ${this.modelId}: ${err instanceof Error ? err.message : String(err)}`, { cause: err });
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
};
|
|
261
|
+
/**
|
|
262
|
+
* Import fastembed, translating "never installed" into an actionable
|
|
263
|
+
* error rather than a bare MODULE_NOT_FOUND. Kept separate from `init` so the
|
|
264
|
+
* two failure modes stay distinguishable: absent backend vs. broken backend.
|
|
265
|
+
*/
|
|
266
|
+
async function loadFastembed() {
|
|
267
|
+
try {
|
|
268
|
+
return await import("fastembed");
|
|
269
|
+
} catch (err) {
|
|
270
|
+
throw new EmbedError("unavailable", "OnnxEmbedder requires 'fastembed' (^2.1.1), which is not installed. Install it alongside @mnmal-ai/substrate-embedder (`pnpm add fastembed`), or use the ollama embedder instead. It is deliberately undeclared because it pulls onnxruntime-node (~520MB) and most deployments never use it.", { cause: err });
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
//#endregion
|
|
275
|
+
//#region ../../plugins/substrate-recall/src/bootstrap.ts
|
|
276
|
+
/**
|
|
277
|
+
* Structured-config bootstrap for `@mnmal-ai/substrate-recall`.
|
|
278
|
+
*
|
|
279
|
+
* Substrate-tier variant: merges the operator JSON config with a typed bag of
|
|
280
|
+
* host-built deps (`RecallHostDeps`) into `createRecallPlugin`'s input.
|
|
281
|
+
*
|
|
282
|
+
* What's operator-overridable (via JSON config):
|
|
283
|
+
* - `embedder` discriminated union: off | fake | ollama | onnx
|
|
284
|
+
* - `{ kind: 'off' }` (default) — no embedder; filter operators, MCP
|
|
285
|
+
* tools, and route handlers that need it are omitted
|
|
286
|
+
* - `{ kind: 'fake', dimension }` — deterministic hash-based embedder for
|
|
287
|
+
* tests and local dev
|
|
288
|
+
* - `{ kind: 'ollama', baseUrl, model, dimension, timeoutMs?, contextLength? }`
|
|
289
|
+
* — Ollama HTTP embedder; `contextLength` defaults to
|
|
290
|
+
* the model's, read from Ollama
|
|
291
|
+
* - `{ kind: 'onnx', model, dimension, cacheDir?, maxLength? }` — local
|
|
292
|
+
* ONNX embedder
|
|
293
|
+
* - `hnswM` HNSW index build param `m` (default 16)
|
|
294
|
+
* - `hnswEfConstruction` HNSW index build param `ef_construction` (default 64)
|
|
295
|
+
* - `salience`, `expansion`, `crystallise` — see the schemas below
|
|
296
|
+
*
|
|
297
|
+
* Vector dimension is derived from the constructed embedder's `dimension` so
|
|
298
|
+
* fresh installs match what the embedder will write. Falls back to 768 when
|
|
299
|
+
* no embedder is built.
|
|
300
|
+
*
|
|
301
|
+
* Invoked by the binary's plugin-dir loader when discovering plugins under
|
|
302
|
+
* `SUBSTRATE_PLUGIN_DIR`.
|
|
303
|
+
*/
|
|
304
|
+
const EmbedderSchema = z.discriminatedUnion("kind", [
|
|
305
|
+
z.object({ kind: z.literal("off") }).strict(),
|
|
306
|
+
z.object({
|
|
307
|
+
kind: z.literal("fake"),
|
|
308
|
+
dimension: z.number().int().positive().default(8)
|
|
309
|
+
}).strict(),
|
|
310
|
+
z.object({
|
|
311
|
+
kind: z.literal("ollama"),
|
|
312
|
+
baseUrl: z.url(),
|
|
313
|
+
model: z.string().min(1),
|
|
314
|
+
dimension: z.number().int().positive(),
|
|
315
|
+
timeoutMs: z.number().int().positive().default(12e4),
|
|
316
|
+
contextLength: z.number().int().positive().optional()
|
|
317
|
+
}).strict(),
|
|
318
|
+
z.object({
|
|
319
|
+
kind: z.literal("onnx"),
|
|
320
|
+
model: z.string().min(1),
|
|
321
|
+
dimension: z.number().int().positive(),
|
|
322
|
+
cacheDir: z.string().min(1).optional(),
|
|
323
|
+
maxLength: z.number().int().positive().default(512)
|
|
324
|
+
}).strict()
|
|
325
|
+
]);
|
|
326
|
+
const SalienceSchema = z.object({
|
|
327
|
+
/**
|
|
328
|
+
* Blend weight. `0` (default) leaves ranking
|
|
329
|
+
* unchanged; the boost-on-recall still fires so the signal accumulates
|
|
330
|
+
* passively, ready for opt-in once an operator has data to evaluate.
|
|
331
|
+
*/
|
|
332
|
+
weight: z.number().min(0).default(0),
|
|
333
|
+
/** Half-life for boost-time exponential decay. Default 168h (7 days). */
|
|
334
|
+
halfLifeHours: z.number().positive().default(168),
|
|
335
|
+
/**
|
|
336
|
+
* Activation-spreading damping. Each opt-in
|
|
337
|
+
* crossRef neighbour of a boosted row gets `spreadDamping × bump`.
|
|
338
|
+
* Default 0.3 keeps the spread modest; 0 disables spreading even if
|
|
339
|
+
* `spread: true` is declared on fields.
|
|
340
|
+
*/
|
|
341
|
+
spreadDamping: z.number().min(0).max(1).default(.3)
|
|
342
|
+
}).strict();
|
|
343
|
+
const ExpansionSchema = z.object({
|
|
344
|
+
/**
|
|
345
|
+
* Plugin-level kill-switch. When false, the recall route
|
|
346
|
+
* ignores per-request `expand:true` and returns the knn-only result.
|
|
347
|
+
* Per-request `expand:false` (the default) always wins regardless.
|
|
348
|
+
*/
|
|
349
|
+
enabled: z.boolean().default(true),
|
|
350
|
+
/**
|
|
351
|
+
* Maximum neighbours pushed per seed row, applied PER DIRECTION
|
|
352
|
+
* (forward + backward each track separately). Effective max across both
|
|
353
|
+
* directions is up to 2× this value when backward is enabled.
|
|
354
|
+
*/
|
|
355
|
+
maxNeighboursPerSeed: z.number().int().positive().default(8),
|
|
356
|
+
/** Maximum total neighbours per request (whole-batch cap). */
|
|
357
|
+
maxTotalNeighbours: z.number().int().positive().default(50)
|
|
358
|
+
}).strict();
|
|
359
|
+
const CrystalliseSchema = z.object({
|
|
360
|
+
/** UMAP output dimensionality. 0 skips reduction (holomap convention). */
|
|
361
|
+
nComponents: z.number().int().nonnegative().default(10),
|
|
362
|
+
nNeighbors: z.number().int().positive().default(15),
|
|
363
|
+
/**
|
|
364
|
+
* HDBSCAN floor, and the live-member floor below which a cluster retires.
|
|
365
|
+
* One knob for both: a cluster too small for the clusterer to have
|
|
366
|
+
* proposed is too small to keep claiming rows.
|
|
367
|
+
*/
|
|
368
|
+
minClusterSize: z.number().int().positive().default(5),
|
|
369
|
+
/** Fixed so a re-run is reproducible. Changing it changes assignments. */
|
|
370
|
+
seed: z.number().int().default(42),
|
|
371
|
+
/**
|
|
372
|
+
* Cosine floor for accretion. A POLICY knob — tune it against this
|
|
373
|
+
* deployment's own measured baseline, not against a figure from another
|
|
374
|
+
* corpus or another backend.
|
|
375
|
+
*/
|
|
376
|
+
accretionThreshold: z.number().min(0).max(1).default(.6)
|
|
377
|
+
}).strict();
|
|
378
|
+
const RecallBootstrapConfigSchema = z.object({
|
|
379
|
+
embedder: EmbedderSchema.optional(),
|
|
380
|
+
queryEmbedTimeoutMs: z.number().int().positive().default(3e4),
|
|
381
|
+
hnswM: z.number().int().positive().default(16),
|
|
382
|
+
hnswEfConstruction: z.number().int().positive().default(64),
|
|
383
|
+
salience: SalienceSchema.default({
|
|
384
|
+
weight: 0,
|
|
385
|
+
halfLifeHours: 168,
|
|
386
|
+
spreadDamping: .3
|
|
387
|
+
}),
|
|
388
|
+
expansion: ExpansionSchema.default({
|
|
389
|
+
enabled: true,
|
|
390
|
+
maxNeighboursPerSeed: 8,
|
|
391
|
+
maxTotalNeighbours: 50
|
|
392
|
+
}),
|
|
393
|
+
crystallise: CrystalliseSchema.default({
|
|
394
|
+
nComponents: 10,
|
|
395
|
+
nNeighbors: 15,
|
|
396
|
+
minClusterSize: 5,
|
|
397
|
+
seed: 42,
|
|
398
|
+
accretionThreshold: .6
|
|
399
|
+
})
|
|
400
|
+
}).strict();
|
|
401
|
+
function buildEmbedder(config) {
|
|
402
|
+
if (config.kind === "off") return null;
|
|
403
|
+
if (config.kind === "fake") return new FakeEmbedder({ dimension: config.dimension });
|
|
404
|
+
if (config.kind === "onnx") {
|
|
405
|
+
const opts = {
|
|
406
|
+
model: config.model,
|
|
407
|
+
dimension: config.dimension,
|
|
408
|
+
maxLength: config.maxLength
|
|
409
|
+
};
|
|
410
|
+
if (config.cacheDir !== void 0) opts.cacheDir = config.cacheDir;
|
|
411
|
+
return new OnnxEmbedder(opts);
|
|
412
|
+
}
|
|
413
|
+
return new OllamaEmbedder({
|
|
414
|
+
baseUrl: config.baseUrl,
|
|
415
|
+
model: config.model,
|
|
416
|
+
dimension: config.dimension,
|
|
417
|
+
timeoutMs: config.timeoutMs,
|
|
418
|
+
...config.contextLength !== void 0 ? { contextLength: config.contextLength } : {}
|
|
419
|
+
});
|
|
420
|
+
}
|
|
421
|
+
const DEFAULT_DIMENSION_NO_EMBEDDER = 768;
|
|
422
|
+
const bootstrap = defineBootstrap.kernel({
|
|
423
|
+
schema: RecallBootstrapConfigSchema,
|
|
424
|
+
build: (config, deps) => {
|
|
425
|
+
const embedder = config.embedder !== void 0 ? buildEmbedder(config.embedder) : null;
|
|
426
|
+
return createRecallPlugin({
|
|
427
|
+
...embedder !== null ? { embedder } : {},
|
|
428
|
+
queryEmbedTimeoutMs: config.queryEmbedTimeoutMs,
|
|
429
|
+
...deps.lexicalStore !== void 0 ? { lexicalStore: deps.lexicalStore } : {},
|
|
430
|
+
store: deps.store,
|
|
431
|
+
...deps.stats !== void 0 ? { stats: deps.stats } : {},
|
|
432
|
+
...deps.resolveDataset !== void 0 ? { resolveDataset: deps.resolveDataset } : {},
|
|
433
|
+
...deps.edges !== void 0 ? { edges: deps.edges } : {},
|
|
434
|
+
...deps.checkReadPolicy !== void 0 ? { checkReadPolicy: deps.checkReadPolicy } : {},
|
|
435
|
+
...deps.checkWritePolicy !== void 0 ? { checkWritePolicy: deps.checkWritePolicy } : {},
|
|
436
|
+
...deps.clusters !== void 0 ? { clusters: deps.clusters } : {},
|
|
437
|
+
...deps.rowsMissing !== void 0 ? { rowsMissing: deps.rowsMissing } : {},
|
|
438
|
+
migration: {
|
|
439
|
+
dimension: embedder?.dimension ?? DEFAULT_DIMENSION_NO_EMBEDDER,
|
|
440
|
+
hnswM: config.hnswM,
|
|
441
|
+
hnswEfConstruction: config.hnswEfConstruction
|
|
442
|
+
},
|
|
443
|
+
salience: config.salience,
|
|
444
|
+
expansion: config.expansion,
|
|
445
|
+
crystallise: config.crystallise
|
|
446
|
+
});
|
|
447
|
+
}
|
|
448
|
+
});
|
|
449
|
+
|
|
450
|
+
//#endregion
|
|
451
|
+
export { bootstrap as default };
|