@mnmal-ai/substrate 1.0.0-alpha.9 → 1.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -13
- package/dist/boot-store-BMQUbHKv.js +4086 -0
- package/dist/{runtime-metrics-DwpDJ8vz.d.ts → bridge-DnToJedN.d.ts} +2 -87
- package/dist/cli-DModIbnN.js +3390 -0
- package/dist/client.browser.js +2507 -0
- package/dist/client.d.ts +222 -171
- package/dist/client.js +6 -7
- package/dist/cluster-worker.js +0 -1
- package/dist/{content-type-C7T28xNb.js → content-type-wLljot5M.js} +0 -1
- package/dist/{env-utils-CGNazUNt.js → env-utils-s25Z1YSA.js} +0 -1
- package/dist/{errors-D4vKuwz8.d.ts → errors-Cmkp1qxf.d.ts} +44 -4
- package/dist/{fake-DUVtKVSE.js → fake-DdXtb-m8.js} +0 -1
- package/dist/{index-D7mSOFlM.d.ts → index-BI4KqFiR.d.ts} +7 -11
- package/dist/msgpackr-Dq18g6Ds.js +16 -0
- package/dist/node-version-BtK_Nq6z.js +32 -0
- package/dist/pack.d.ts +75 -11
- package/dist/pack.js +4 -4
- package/dist/{plugin-DpncES1p.js → plugin-CEOhrEme.js} +78 -8
- package/dist/proposals-bootstrap.js +6 -5
- package/dist/proposals.config.schema.json +5 -0
- package/dist/proposals.d.ts +17 -5
- package/dist/proposals.default-config.json +2 -1
- package/dist/proposals.js +2 -3
- package/dist/recall-bootstrap.js +3 -4
- package/dist/recall.d.ts +11 -12
- package/dist/recall.js +2 -3
- package/dist/{repl-DeKzfvNV.js → repl-BvDZy-Ut.js} +52 -26
- package/dist/{row-SRhura12.d.ts → row-D0If4qif.d.ts} +91 -13
- package/dist/run-BT123RV1.js +2443 -0
- package/dist/{plugin-t2g2wX0b.d.ts → schema-aIkY8sR3.d.ts} +1293 -63
- package/dist/{script-C6DSa_Na.js → script-ByQ9pzhC.js} +54 -26
- package/dist/script-runner.js +78 -0
- package/dist/sdk.d.ts +236 -38
- package/dist/sdk.js +5 -110
- package/dist/semver-D_zdq0hi.js +17 -0
- package/dist/src-BMD2OE--.js +8289 -0
- package/dist/{src-BLgbLq2k.js → src-B_kHkbsr.js} +780 -330
- package/dist/{src-DjIuRjzF.js → src-CKSJ13p8.js} +10231 -4237
- package/dist/{src-C12EA0w9.js → src-CW7eRDss.js} +179 -96
- package/dist/{row-DZ0Fnpc6.js → src-DmiUWOLp.js} +2 -1
- package/dist/{src-DCy7Iy2l.js → src-DmjwFMu8.js} +900 -77
- package/dist/{src-DiNTSaFk.js → src-FXO5JPyk.js} +686 -856
- package/dist/substrate-admin.js +17 -782
- package/dist/substrate.js +4 -5302
- package/dist/testing.d.ts +473 -96
- package/dist/testing.js +19 -165
- package/dist/{ws-C72iVS5T.d.ts → ws-DW8ng02H.d.ts} +4 -5
- package/package.json +26 -26
- package/dist/dataset-CqrqTYPz.js +0 -1535
- package/dist/pg-edges-store-KH_FQCxx.js +0 -381
- package/dist/schema-g4FQLr0V.d.ts +0 -433
- package/dist/src-BD27zeGF.js +0 -2339
|
@@ -0,0 +1,3390 @@
|
|
|
1
|
+
import { _ as AUDIT_EVENT_TYPE, g as buildAuditRow, it as saveSweepState, m as runInviteSweep, mn as validateHardDeleteEnv, nt as loadSweepState, rt as runSweep, u as isStoreProfileName, v as INVITE_TYPE } from "./src-CKSJ13p8.js";
|
|
2
|
+
import { n as trimTrailingSlashes, s as makeAuthorizationProvider } from "./env-utils-s25Z1YSA.js";
|
|
3
|
+
import { L as encodeList, T as SYSTEM_NAMESPACE, f as coversNamespace, m as grantedIn, p as grantOf } from "./src-DmiUWOLp.js";
|
|
4
|
+
import { B as EXPR_AGGREGATES, H as EXPR_LIMITS, V as EXPR_FUNCTIONS, z as DATE_UNITS } from "./src-B_kHkbsr.js";
|
|
5
|
+
import { X as AdminCliError, _ as version, a as MS_PER_DAY, c as parseEnv, d as redactStoreSpec, h as startSentinelClient, m as runDeferredIndexBuilds, n as resolvePluginDir, o as parseBridgeCreds, p as reportUnguardedIndexBuilds, s as parseBridges, t as bootStore, v as openPgDataset } from "./boot-store-BMQUbHKv.js";
|
|
6
|
+
import { T as substrateFetch, a as SubstrateClient, d as NetworkError, i as tokenAudience, n as resolveAgentCapabilities, p as AdminBackfillError, r as noRedirectFetch, t as createClientFromEnv, u as AuthError } from "./src-FXO5JPyk.js";
|
|
7
|
+
import { n as runScript, t as DEFAULT_MAX_WRITES } from "./script-ByQ9pzhC.js";
|
|
8
|
+
import { D as tryWithPgAdvisoryLock } from "./src-BMD2OE--.js";
|
|
9
|
+
import { n as PgLexicalStore, r as PgEmbeddingsStore } from "./src-CW7eRDss.js";
|
|
10
|
+
import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
|
|
11
|
+
import { homedir, hostname, tmpdir } from "node:os";
|
|
12
|
+
import { basename, dirname, join, resolve } from "node:path";
|
|
13
|
+
import { createHash, generateKeyPairSync, randomBytes } from "node:crypto";
|
|
14
|
+
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
15
|
+
import { z } from "zod";
|
|
16
|
+
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
17
|
+
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
18
|
+
import { fork } from "node:child_process";
|
|
19
|
+
|
|
20
|
+
//#region ../../node_modules/.pnpm/@mnmal-ai+surface-kit@0.17.1_ink@8.0.0_@types+react@19.3.0_react@19.3.1__react@19.3.1/node_modules/@mnmal-ai/surface-kit/dist/cells.js
|
|
21
|
+
const graphemes = new Intl.Segmenter(void 0, { granularity: "grapheme" });
|
|
22
|
+
|
|
23
|
+
//#endregion
|
|
24
|
+
//#region ../../node_modules/.pnpm/@mnmal-ai+surface-core@0.17.1_ink@8.0.0_@types+react@19.3.0_react@19.3.1__react@19.3.1/node_modules/@mnmal-ai/surface-core/dist/overlay-5s8slC38.js
|
|
25
|
+
const SEGMENTER = new Intl.Segmenter(void 0, { granularity: "grapheme" });
|
|
26
|
+
const OSC = /(?:\u001b\]|\u009d)[^\u0007\u001b\u009c]*(?:\u0007|\u009c|\u001b\\)?/g;
|
|
27
|
+
const CSI = /(?:\u001b\[|\u009b)[0-?]*[ -/]*[@-~]?/g;
|
|
28
|
+
const ESC_PAIR = /\u001b[ -~]?/g;
|
|
29
|
+
const CONTROL = /[\u0000-\u0008\u000b-\u001f\u007f-\u009f]/g;
|
|
30
|
+
/** Strips ESC sequences and C0/C1 control characters; keeps `\n`, and a tab becomes a space. */
|
|
31
|
+
function sanitise(text) {
|
|
32
|
+
return text.replace(/\t/g, " ").replace(OSC, "").replace(CSI, "").replace(ESC_PAIR, "").replace(CONTROL, "");
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
//#endregion
|
|
36
|
+
//#region src/cli/ask/main.ts
|
|
37
|
+
const USAGE$1 = "usage: substrate ask [--ns <namespace>] [--project <name>] [--json] <question…>\n";
|
|
38
|
+
async function runAsk(argv, options = {}) {
|
|
39
|
+
const write = options.write ?? ((s) => process.stdout.write(s));
|
|
40
|
+
const env = options.env ?? process.env;
|
|
41
|
+
let ns = env.SUBSTRATE_NAMESPACE !== void 0 && env.SUBSTRATE_NAMESPACE !== "" ? env.SUBSTRATE_NAMESPACE : void 0;
|
|
42
|
+
let project = basename(options.cwd ?? process.cwd());
|
|
43
|
+
let json = false;
|
|
44
|
+
const words = [];
|
|
45
|
+
for (let i = 0; i < argv.length; i++) {
|
|
46
|
+
const a = argv[i];
|
|
47
|
+
if (a === "--") {
|
|
48
|
+
words.push(...argv.slice(i + 1));
|
|
49
|
+
break;
|
|
50
|
+
}
|
|
51
|
+
if (a === "--json") json = true;
|
|
52
|
+
else if (a === "--ns" || a === "--project") {
|
|
53
|
+
const v = argv[++i];
|
|
54
|
+
if (v === void 0 || v.startsWith("--")) {
|
|
55
|
+
write(USAGE$1);
|
|
56
|
+
return 2;
|
|
57
|
+
}
|
|
58
|
+
if (a === "--ns") ns = v;
|
|
59
|
+
else project = v;
|
|
60
|
+
} else words.push(a);
|
|
61
|
+
}
|
|
62
|
+
if ((env.SUBSTRATE_URL ?? "") === "") {
|
|
63
|
+
write("SUBSTRATE_URL is required (e.g. https://substrate.example.com:3100)\n");
|
|
64
|
+
return 2;
|
|
65
|
+
}
|
|
66
|
+
const question = words.join(" ").trim();
|
|
67
|
+
if (question === "") {
|
|
68
|
+
write(USAGE$1);
|
|
69
|
+
return 2;
|
|
70
|
+
}
|
|
71
|
+
if (ns === void 0) {
|
|
72
|
+
write("substrate ask: name the namespace with --ns or SUBSTRATE_NAMESPACE\n");
|
|
73
|
+
return 2;
|
|
74
|
+
}
|
|
75
|
+
let client;
|
|
76
|
+
try {
|
|
77
|
+
({client} = createClientFromEnv({
|
|
78
|
+
env,
|
|
79
|
+
allowAnonymous: true,
|
|
80
|
+
...options.fetchImpl !== void 0 ? { fetch: options.fetchImpl } : {}
|
|
81
|
+
}));
|
|
82
|
+
} catch (err) {
|
|
83
|
+
write(`substrate ask: agent identity: ${sanitise(messageOf$1(err))}\n`);
|
|
84
|
+
return 2;
|
|
85
|
+
}
|
|
86
|
+
let result;
|
|
87
|
+
try {
|
|
88
|
+
result = await client.nl.askJson({
|
|
89
|
+
namespace: ns,
|
|
90
|
+
question,
|
|
91
|
+
project
|
|
92
|
+
});
|
|
93
|
+
} catch (err) {
|
|
94
|
+
if (err instanceof NetworkError && err.status === 404 && err.code !== "nl_unknown_namespace") write("substrate ask: the server has no NL route (is @mnmal-ai/substrate-nl installed?)\n");
|
|
95
|
+
else if (err instanceof NetworkError && err.serverMessage !== void 0) write(`substrate ask: ${sanitise(err.serverMessage)}\n`);
|
|
96
|
+
else write(`substrate ask: ${sanitise(messageOf$1(err))}\n`);
|
|
97
|
+
return 1;
|
|
98
|
+
}
|
|
99
|
+
const outcome = result.outcome;
|
|
100
|
+
if (json) write(`${JSON.stringify(result, null, 2).replace(/[\u0080-\u009f]/g, (c) => `\\u${c.charCodeAt(0).toString(16).padStart(4, "0")}`)}\n`);
|
|
101
|
+
else write(render(outcome).split("\n").map(sanitise).join("\n"));
|
|
102
|
+
if (outcome === null) return 0;
|
|
103
|
+
if (outcome.kind === "refuse" || outcome.kind === "unknown") return 1;
|
|
104
|
+
if (outcome.kind === "rows" && Array.isArray(outcome.errors) && outcome.errors.length > 0) return 1;
|
|
105
|
+
return 0;
|
|
106
|
+
}
|
|
107
|
+
function messageOf$1(err) {
|
|
108
|
+
return err instanceof Error ? err.message : String(err);
|
|
109
|
+
}
|
|
110
|
+
function render(o) {
|
|
111
|
+
if (o === null) return "{}\n";
|
|
112
|
+
switch (o.kind) {
|
|
113
|
+
case "rows": {
|
|
114
|
+
const key = Object.keys(o.frame)[0] ?? "?";
|
|
115
|
+
const rows = Array.isArray(o.data) ? o.data : void 0;
|
|
116
|
+
const tail = o.count !== void 0 ? `count ${o.count}` : rows === void 0 ? "rejected" : `${rows.length} rows`;
|
|
117
|
+
const head = `▸ ${key} · ${o.source}${o.handoff !== void 0 ? ` (${o.handoff})` : ""} · ${tail}\n`;
|
|
118
|
+
const errs = (o.errors ?? []).map((e) => ` ✗ ${JSON.stringify(e)}\n`).join("");
|
|
119
|
+
const body = (rows ?? []).map((r) => ` ${String(r.shortId ?? r.id ?? "")} ${String(r.title ?? r.synopsis ?? r.name ?? "")}\n`).join("");
|
|
120
|
+
return head + errs + droppedText(o.dropped) + body;
|
|
121
|
+
}
|
|
122
|
+
case "recall": {
|
|
123
|
+
const errs = (o.errors ?? []).map((e) => ` ✗ ${JSON.stringify(e)}\n`).join("");
|
|
124
|
+
if (!Array.isArray(o.hits)) return `▸ recall · rejected\n${errs}`;
|
|
125
|
+
const hits = o.hits;
|
|
126
|
+
return `▸ recall · ${hits.length} hits\n${errs}${hits.map((h) => ` ${String(h.type)} ${String(h.id)} ${Number(h.score).toFixed(2)}\n`).join("")}`;
|
|
127
|
+
}
|
|
128
|
+
case "clarify": {
|
|
129
|
+
const opts = Array.isArray(o.options) ? o.options : [];
|
|
130
|
+
return `? ${o.reason}\n${opts.map((x, i) => ` ${i + 1}. ${x.label}\n`).join("")}`;
|
|
131
|
+
}
|
|
132
|
+
case "refuse": return `✗ ${o.reason}\n`;
|
|
133
|
+
case "unknown": return `? unknown: not answered, because part of the question was dropped\n${droppedText(o.dropped)}${planText(o.plan)}`;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
function droppedText(dropped) {
|
|
137
|
+
return (dropped ?? []).map((d) => ` dropped "${d.span}": ${d.reason}\n`).join("");
|
|
138
|
+
}
|
|
139
|
+
function planText(plan) {
|
|
140
|
+
return (plan ?? []).map((c) => ` plan ${c.path.join(".")} (${c.kind})\n`).join("");
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
//#endregion
|
|
144
|
+
//#region src/cli/backfill-field/main.ts
|
|
145
|
+
/**
|
|
146
|
+
* `substrate backfill-field <namespace> <Type> <field> --values <file.json> [--apply] [--overwrite] [--json]`
|
|
147
|
+
*
|
|
148
|
+
* Writes per-row values of one field (`{ "<row id>": value, ... }` in the
|
|
149
|
+
* file) through the admin route `POST /substrate/v1/_system/backfill-field`, with
|
|
150
|
+
* row metadata preserved. Dry run unless `--apply`; fills only empty values
|
|
151
|
+
* unless `--overwrite`. Chunking lives in `client.admin.backfill`.
|
|
152
|
+
*
|
|
153
|
+
* Env: SUBSTRATE_URL, SUBSTRATE_ADMIN_TOKEN (admin Bearer token). Exit 0 on success,
|
|
154
|
+
* 1 when the server refuses the values, 2 on usage or transport errors.
|
|
155
|
+
*/
|
|
156
|
+
async function runBackfillField(argv, options = {}) {
|
|
157
|
+
const write = options.write ?? ((text) => process.stdout.write(text));
|
|
158
|
+
const [namespace, type, field] = argv.filter((a, i) => !a.startsWith("--") && argv[i - 1] !== "--values");
|
|
159
|
+
const valuesAt = argv.indexOf("--values");
|
|
160
|
+
const file = valuesAt >= 0 ? argv[valuesAt + 1] : void 0;
|
|
161
|
+
if (namespace === void 0 || type === void 0 || field === void 0 || file === void 0) {
|
|
162
|
+
write("usage: substrate backfill-field <namespace> <Type> <field> --values <file.json> [--apply] [--overwrite] [--json]\n");
|
|
163
|
+
return 2;
|
|
164
|
+
}
|
|
165
|
+
const baseUrl = trimTrailingSlashes(options.baseUrl ?? process.env.SUBSTRATE_URL ?? "");
|
|
166
|
+
if (baseUrl === "") {
|
|
167
|
+
write("SUBSTRATE_URL is required (e.g. https://substrate.example.com:3100)\n");
|
|
168
|
+
return 2;
|
|
169
|
+
}
|
|
170
|
+
let values;
|
|
171
|
+
try {
|
|
172
|
+
values = JSON.parse((options.readFile ?? ((p) => readFileSync(p, "utf8")))(file));
|
|
173
|
+
} catch (err) {
|
|
174
|
+
write(`cannot read ${file}: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
175
|
+
return 2;
|
|
176
|
+
}
|
|
177
|
+
if (values === null || typeof values !== "object" || Array.isArray(values)) {
|
|
178
|
+
write(`${file} must hold an object of row id -> value\n`);
|
|
179
|
+
return 2;
|
|
180
|
+
}
|
|
181
|
+
const token = options.token ?? process.env.SUBSTRATE_ADMIN_TOKEN;
|
|
182
|
+
const apply = argv.includes("--apply");
|
|
183
|
+
const overwrite = argv.includes("--overwrite");
|
|
184
|
+
const entries = Object.entries(values);
|
|
185
|
+
const json = argv.includes("--json");
|
|
186
|
+
if (token !== void 0 && token.length > 0) try {
|
|
187
|
+
tokenAudience(baseUrl);
|
|
188
|
+
} catch (err) {
|
|
189
|
+
write(`${err instanceof Error ? err.message : String(err)}\n`);
|
|
190
|
+
return 2;
|
|
191
|
+
}
|
|
192
|
+
const client = new SubstrateClient({
|
|
193
|
+
endpoint: baseUrl,
|
|
194
|
+
...token !== void 0 && token.length > 0 ? { token } : {},
|
|
195
|
+
fetch: noRedirectFetch(options.fetchImpl)
|
|
196
|
+
});
|
|
197
|
+
let total;
|
|
198
|
+
try {
|
|
199
|
+
total = await client.admin.backfill({
|
|
200
|
+
namespace,
|
|
201
|
+
type,
|
|
202
|
+
field,
|
|
203
|
+
values,
|
|
204
|
+
apply,
|
|
205
|
+
overwrite
|
|
206
|
+
}, { onChunk: (c) => {
|
|
207
|
+
if (json) write(`${JSON.stringify(c.body)}\n`);
|
|
208
|
+
} });
|
|
209
|
+
} catch (err) {
|
|
210
|
+
const cause = err instanceof AdminBackfillError ? err.cause : err;
|
|
211
|
+
const refusal = cause instanceof NetworkError || cause instanceof AuthError ? cause : void 0;
|
|
212
|
+
const body = refusal?.body ?? {};
|
|
213
|
+
const transport = refusal instanceof NetworkError && refusal.status === void 0 ? refusal.cause instanceof Error ? refusal.cause.message : refusal.message : void 0;
|
|
214
|
+
if (json) write(`${JSON.stringify(refusal?.body ?? { message: transport ?? String(cause) })}\n`);
|
|
215
|
+
else if (refusal !== void 0 && refusal.status !== void 0) {
|
|
216
|
+
write(`HTTP ${refusal.status}: ${refusal.code ?? ""} ${refusal.serverMessage ?? ""}\n`);
|
|
217
|
+
for (const v of body.invalid ?? []) write(` ${v.id}: ${String(v.value)} (${v.reason})\n`);
|
|
218
|
+
} else write(`request failed: ${transport ?? (cause instanceof Error ? cause.message : String(cause))}\n`);
|
|
219
|
+
if (!json && err instanceof AdminBackfillError && apply) {
|
|
220
|
+
if (err.chunksDone > 0) write(`${err.chunksDone} of ${err.chunks} chunk(s) were written before the refusal\n`);
|
|
221
|
+
const status = refusal?.status;
|
|
222
|
+
if (status === void 0 || status < 400 || status >= 500) write("the failing chunk's own state is unknown — re-running is safe (written rows come back unchanged)\n");
|
|
223
|
+
}
|
|
224
|
+
return refusal?.status === 400 ? 1 : 2;
|
|
225
|
+
}
|
|
226
|
+
if (json) return 0;
|
|
227
|
+
write(`${namespace}/${type}.${field}: ${entries.length} requested — ${apply ? "wrote" : "would write (dry run; pass --apply)"} ${total.write}\n`);
|
|
228
|
+
for (const [k, n] of Object.entries(total.counts).sort((a, b) => b[1] - a[1])) write(` ${k} ${n}\n`);
|
|
229
|
+
if (total.kept > 0) write(`kept ${total.kept} existing value(s)${overwrite ? "" : " (pass --overwrite to replace)"}\n`);
|
|
230
|
+
if (total.unchanged > 0) write(`unchanged ${total.unchanged}\n`);
|
|
231
|
+
if (total.missing.length > 0) write(`missing ${total.missing.length} id(s)\n`);
|
|
232
|
+
return 0;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
//#endregion
|
|
236
|
+
//#region src/cli/check-enums/main.ts
|
|
237
|
+
/**
|
|
238
|
+
* `substrate check-enums` — report stored values that fall outside a field's
|
|
239
|
+
* declared `enum` vocabulary.
|
|
240
|
+
*
|
|
241
|
+
* Write-time validation does not cover rows that predate an enum declaration,
|
|
242
|
+
* nor fields absent from an update patch, so stored values can drift from
|
|
243
|
+
* the published vocabulary.
|
|
244
|
+
*
|
|
245
|
+
* Read-only by default and runs over HTTP, so it needs no database
|
|
246
|
+
* credential. `--record` writes `_system/EnumCheckVerdict` rows and needs a
|
|
247
|
+
* credential with `_system` mutate via SUBSTRATE_ENUM_CHECK_TOKEN.
|
|
248
|
+
*/
|
|
249
|
+
/**
|
|
250
|
+
* Walk a federated schema and collect every scalar field declaring an `enum`.
|
|
251
|
+
* Exported so the scan and the reporting can be tested without a server.
|
|
252
|
+
*/
|
|
253
|
+
function collectEnumFields(federated) {
|
|
254
|
+
const out = [];
|
|
255
|
+
for (const [namespace, schema] of Object.entries(federated)) for (const [type, entry] of Object.entries(schema.types ?? {})) for (const [field, def] of Object.entries(entry.fields ?? {})) if (def.kind === "scalar" && def.enum !== void 0 && def.enum.length > 0) out.push({
|
|
256
|
+
namespace,
|
|
257
|
+
type,
|
|
258
|
+
field,
|
|
259
|
+
vocabulary: def.enum,
|
|
260
|
+
...def.aliases !== void 0 ? { aliases: def.aliases } : {},
|
|
261
|
+
...def.strict === false ? { strict: false } : {}
|
|
262
|
+
});
|
|
263
|
+
return out;
|
|
264
|
+
}
|
|
265
|
+
/**
|
|
266
|
+
* Tally values outside the vocabulary. `null`/`undefined` are skipped rather
|
|
267
|
+
* than reported: a nullable enum legitimately holds null, and
|
|
268
|
+
* `assertEnumValues` skips them on write for the same reason — flagging them
|
|
269
|
+
* here would contradict the writer and bury the real offenders.
|
|
270
|
+
*/
|
|
271
|
+
function tallyOffenders(values, vocabulary, opts = {}) {
|
|
272
|
+
const allowed = new Set(vocabulary);
|
|
273
|
+
const normalisable = /* @__PURE__ */ new Set([...vocabulary.map((v) => v.toLowerCase()), ...Object.keys(opts.aliases ?? {}).map((a) => a.toLowerCase())]);
|
|
274
|
+
const counts = /* @__PURE__ */ new Map();
|
|
275
|
+
for (const value of values) {
|
|
276
|
+
if (value === null || value === void 0) continue;
|
|
277
|
+
const asString = typeof value === "string" ? value : String(value);
|
|
278
|
+
if (allowed.has(asString)) continue;
|
|
279
|
+
if (opts.strict === false && !normalisable.has(asString.toLowerCase())) continue;
|
|
280
|
+
counts.set(asString, (counts.get(asString) ?? 0) + 1);
|
|
281
|
+
}
|
|
282
|
+
return [...counts.entries()].map(([value, count]) => ({
|
|
283
|
+
value,
|
|
284
|
+
count
|
|
285
|
+
})).sort((a, b) => b.count - a.count || a.value.localeCompare(b.value));
|
|
286
|
+
}
|
|
287
|
+
/**
|
|
288
|
+
* Project a single field across one type, following `_pagination.nextCursor`.
|
|
289
|
+
*
|
|
290
|
+
* Projecting ONE field keeps the response small, and it is also the only
|
|
291
|
+
* projection guaranteed safe: any other field name could itself be undeclared,
|
|
292
|
+
* which would land in `_diagnostics.unknownProjectionFields` and come back null
|
|
293
|
+
* on every row.
|
|
294
|
+
*/
|
|
295
|
+
async function scanField(field, opts) {
|
|
296
|
+
const key = `${field.namespace}/${field.type}`;
|
|
297
|
+
const values = [];
|
|
298
|
+
let cursor;
|
|
299
|
+
let truncated = false;
|
|
300
|
+
for (;;) {
|
|
301
|
+
const params = { limit: opts.pageSize };
|
|
302
|
+
if (cursor !== void 0) params.cursor = cursor;
|
|
303
|
+
const res = await opts.fetchImpl(`${opts.baseUrl}/substrate/v1/query`, {
|
|
304
|
+
method: "POST",
|
|
305
|
+
headers: {
|
|
306
|
+
"content-type": "application/json",
|
|
307
|
+
...opts.authorization !== void 0 ? { authorization: opts.authorization } : {}
|
|
308
|
+
},
|
|
309
|
+
body: JSON.stringify({ [key]: {
|
|
310
|
+
params,
|
|
311
|
+
fields: { [field.field]: true }
|
|
312
|
+
} })
|
|
313
|
+
});
|
|
314
|
+
if (!res.ok) throw new Error(`query ${key} failed: HTTP ${res.status}`);
|
|
315
|
+
const envelope = await res.json();
|
|
316
|
+
const failure = envelope.errors?.[0];
|
|
317
|
+
if (failure !== void 0) throw new Error(`query ${key} rejected: ${failure.code ?? "error"} ${failure.message ?? ""}`);
|
|
318
|
+
const rows = envelope.data?.[key];
|
|
319
|
+
if (Array.isArray(rows)) for (const row of rows) values.push(row?.[field.field]);
|
|
320
|
+
if (values.length >= opts.maxRows) {
|
|
321
|
+
truncated = true;
|
|
322
|
+
break;
|
|
323
|
+
}
|
|
324
|
+
const page = envelope._pagination?.[key];
|
|
325
|
+
if (page?.hasMore !== true || page.nextCursor === void 0) break;
|
|
326
|
+
cursor = page.nextCursor;
|
|
327
|
+
}
|
|
328
|
+
return {
|
|
329
|
+
...field,
|
|
330
|
+
rowsScanned: values.length,
|
|
331
|
+
offenders: tallyOffenders(values, field.vocabulary, field),
|
|
332
|
+
truncated
|
|
333
|
+
};
|
|
334
|
+
}
|
|
335
|
+
/**
|
|
336
|
+
* Returns 0 when every enum-declared field agrees with its stored rows, 1 when
|
|
337
|
+
* any drift is found — usable as a CI or cron gate.
|
|
338
|
+
*/
|
|
339
|
+
async function runCheckEnums(argv = [], options = {}) {
|
|
340
|
+
const write = options.write ?? ((text) => process.stdout.write(text));
|
|
341
|
+
const asJson = argv.includes("--json");
|
|
342
|
+
const baseUrl = trimTrailingSlashes(options.baseUrl ?? process.env.SUBSTRATE_URL ?? "");
|
|
343
|
+
if (baseUrl === "") {
|
|
344
|
+
write("SUBSTRATE_URL is required (e.g. https://substrate.example.com:3100)\n");
|
|
345
|
+
return 2;
|
|
346
|
+
}
|
|
347
|
+
const baseFetch = options.fetchImpl ?? substrateFetch;
|
|
348
|
+
const fetchImpl = (url, init) => baseFetch(url, {
|
|
349
|
+
...init,
|
|
350
|
+
redirect: "error"
|
|
351
|
+
});
|
|
352
|
+
const pageSize = options.pageSize ?? 500;
|
|
353
|
+
const maxRows = options.maxRows ?? 5e4;
|
|
354
|
+
const token = options.token ?? process.env.SUBSTRATE_ENUM_CHECK_TOKEN;
|
|
355
|
+
const authorization = token !== void 0 && token.length > 0 ? `Bearer ${token}` : void 0;
|
|
356
|
+
if (authorization !== void 0) try {
|
|
357
|
+
tokenAudience(baseUrl);
|
|
358
|
+
} catch (err) {
|
|
359
|
+
write(`${err instanceof Error ? err.message : String(err)}\n`);
|
|
360
|
+
return 2;
|
|
361
|
+
}
|
|
362
|
+
const schemaRes = await fetchImpl(`${baseUrl}/substrate/v1/federated-schema`, { headers: authorization !== void 0 ? { authorization } : {} });
|
|
363
|
+
if (!schemaRes.ok) {
|
|
364
|
+
const hint = schemaRes.status === 401 ? " — the server needs a key: set SUBSTRATE_ENUM_CHECK_TOKEN to a Bearer token with read" : "";
|
|
365
|
+
write(`federated-schema failed: HTTP ${schemaRes.status}${hint}\n`);
|
|
366
|
+
return 2;
|
|
367
|
+
}
|
|
368
|
+
const fields = collectEnumFields(await schemaRes.json());
|
|
369
|
+
const reports = [];
|
|
370
|
+
for (const field of fields) reports.push(await scanField(field, {
|
|
371
|
+
baseUrl,
|
|
372
|
+
fetchImpl,
|
|
373
|
+
pageSize,
|
|
374
|
+
maxRows,
|
|
375
|
+
authorization
|
|
376
|
+
}));
|
|
377
|
+
const drifted = reports.filter((r) => r.offenders.length > 0);
|
|
378
|
+
const result = {
|
|
379
|
+
fieldsChecked: fields.length,
|
|
380
|
+
reports,
|
|
381
|
+
drifted
|
|
382
|
+
};
|
|
383
|
+
if (argv.includes("--record")) {
|
|
384
|
+
const recordCode = await recordVerdicts(baseUrl, fetchImpl, reports);
|
|
385
|
+
if (recordCode !== 0) return recordCode;
|
|
386
|
+
}
|
|
387
|
+
if (asJson) {
|
|
388
|
+
write(`${JSON.stringify(result, null, 2)}\n`);
|
|
389
|
+
return drifted.length > 0 ? 1 : 0;
|
|
390
|
+
}
|
|
391
|
+
write(`Checked ${fields.length} enum-declared field(s).\n`);
|
|
392
|
+
for (const report of drifted) {
|
|
393
|
+
write(`\n DRIFT ${report.namespace}/${report.type}.${report.field}\n`);
|
|
394
|
+
write(` declared: ${report.vocabulary.join(", ")}\n`);
|
|
395
|
+
for (const { value, count } of report.offenders) write(` stored: '${value}' x${count} (not declared)\n`);
|
|
396
|
+
if (report.truncated) write(` NOTE: stopped at ${report.rowsScanned} rows — counts are a floor.\n`);
|
|
397
|
+
}
|
|
398
|
+
for (const report of reports) if (report.offenders.length === 0 && report.truncated) write(`\n PARTIAL ${report.namespace}/${report.type}.${report.field} — clean over the first ${report.rowsScanned} rows, table not exhausted.\n`);
|
|
399
|
+
if (drifted.length === 0) write("\nNo drift found.\n");
|
|
400
|
+
return drifted.length > 0 ? 1 : 0;
|
|
401
|
+
}
|
|
402
|
+
/**
|
|
403
|
+
* sha256 of the vocabulary JSON — a vocab edit invalidates the verdict
|
|
404
|
+
* structurally. Aliases and strictness change what counts as an offender, so
|
|
405
|
+
* they join the hash when declared; a plain enum hashes exactly as before.
|
|
406
|
+
*/
|
|
407
|
+
function vocabHash(vocabulary, opts = {}) {
|
|
408
|
+
const subject = opts.aliases === void 0 && opts.strict !== false ? [...vocabulary] : {
|
|
409
|
+
vocabulary: [...vocabulary],
|
|
410
|
+
aliases: opts.aliases ?? {},
|
|
411
|
+
strict: opts.strict !== false
|
|
412
|
+
};
|
|
413
|
+
return createHash("sha256").update(JSON.stringify(subject)).digest("hex");
|
|
414
|
+
}
|
|
415
|
+
/** Deterministic verdict-row id: one live verdict per (namespace, type, field). */
|
|
416
|
+
function verdictId(report) {
|
|
417
|
+
return `${report.namespace}/${report.type}/${report.field}`;
|
|
418
|
+
}
|
|
419
|
+
/**
|
|
420
|
+
* Write one `_system/EnumCheckVerdict` row per scanned field.
|
|
421
|
+
*
|
|
422
|
+
* Upsert by deterministic id: `updateEnumCheckVerdict` when the row exists,
|
|
423
|
+
* `createEnumCheckVerdict` when it does not (distinguished by a get, not by
|
|
424
|
+
* catching — a failed create must stay a failure, not a silent update).
|
|
425
|
+
* A run that found drift still records: the verdict carries the outlier
|
|
426
|
+
* count, and "checked with drift" is a different fact from "unchecked".
|
|
427
|
+
*
|
|
428
|
+
* Returns a process exit code: 3 when the credential is absent or refused
|
|
429
|
+
* (recording needs `_system` mutate; the scan itself already succeeded, so
|
|
430
|
+
* this is neither a 1 (drift) nor a 2 (usage/transport)).
|
|
431
|
+
*/
|
|
432
|
+
async function recordVerdicts(baseUrl, fetchImpl, reports, opts = {}) {
|
|
433
|
+
const token = opts.token ?? process.env.SUBSTRATE_ENUM_CHECK_TOKEN;
|
|
434
|
+
if (token === void 0 || token.length === 0) {
|
|
435
|
+
process.stderr.write("check-enums --record needs SUBSTRATE_ENUM_CHECK_TOKEN (Bearer token with _system mutate) — recording a verdict is a claim about the deployment, and anonymous callers must not mint one.\n");
|
|
436
|
+
return 3;
|
|
437
|
+
}
|
|
438
|
+
try {
|
|
439
|
+
tokenAudience(baseUrl);
|
|
440
|
+
} catch (err) {
|
|
441
|
+
process.stderr.write(`${err instanceof Error ? err.message : String(err)}\n`);
|
|
442
|
+
return 2;
|
|
443
|
+
}
|
|
444
|
+
const headers = {
|
|
445
|
+
"content-type": "application/json",
|
|
446
|
+
authorization: `Bearer ${token}`
|
|
447
|
+
};
|
|
448
|
+
const checkedAt = (opts.now ?? (() => (/* @__PURE__ */ new Date()).toISOString()))();
|
|
449
|
+
for (const report of reports) {
|
|
450
|
+
const id = verdictId(report);
|
|
451
|
+
const params = {
|
|
452
|
+
id,
|
|
453
|
+
namespace: report.namespace,
|
|
454
|
+
type: report.type,
|
|
455
|
+
field: report.field,
|
|
456
|
+
vocabHash: vocabHash(report.vocabulary, report),
|
|
457
|
+
checkedAt,
|
|
458
|
+
rowsScanned: report.rowsScanned,
|
|
459
|
+
outlierCount: report.offenders.reduce((n, o) => n + o.count, 0),
|
|
460
|
+
truncated: report.truncated,
|
|
461
|
+
verdictKey: id
|
|
462
|
+
};
|
|
463
|
+
const getRes = await fetchImpl(`${baseUrl}/substrate/v1/query`, {
|
|
464
|
+
method: "POST",
|
|
465
|
+
headers,
|
|
466
|
+
body: JSON.stringify({ "_system/EnumCheckVerdict": {
|
|
467
|
+
params: { id },
|
|
468
|
+
fields: { id: true }
|
|
469
|
+
} })
|
|
470
|
+
});
|
|
471
|
+
if (!getRes.ok) {
|
|
472
|
+
process.stderr.write(`verdict lookup for ${id} failed: HTTP ${getRes.status}\n`);
|
|
473
|
+
return 3;
|
|
474
|
+
}
|
|
475
|
+
const envelope = await getRes.json();
|
|
476
|
+
if (envelope.errors?.[0] !== void 0) {
|
|
477
|
+
process.stderr.write(`verdict lookup for ${id} refused: ${envelope.errors[0].code ?? "error"} ${envelope.errors[0].message ?? ""}\n`);
|
|
478
|
+
return 3;
|
|
479
|
+
}
|
|
480
|
+
const existing = envelope.data?.["_system/EnumCheckVerdict"];
|
|
481
|
+
const mutation = Array.isArray(existing) && existing.length > 0 ? "update" : "create";
|
|
482
|
+
const mutRes = await fetchImpl(`${baseUrl}/substrate/v1/mutate`, {
|
|
483
|
+
method: "POST",
|
|
484
|
+
headers,
|
|
485
|
+
body: JSON.stringify({ [`_system/${mutation}EnumCheckVerdict`]: {
|
|
486
|
+
params,
|
|
487
|
+
fields: { id: true }
|
|
488
|
+
} })
|
|
489
|
+
});
|
|
490
|
+
if (!mutRes.ok) {
|
|
491
|
+
process.stderr.write(`verdict write for ${id} failed: HTTP ${mutRes.status}\n`);
|
|
492
|
+
return 3;
|
|
493
|
+
}
|
|
494
|
+
const mutFailure = (await mutRes.json()).errors?.[0];
|
|
495
|
+
if (mutFailure !== void 0) {
|
|
496
|
+
process.stderr.write(`verdict write for ${id} refused: ${mutFailure.code ?? "error"} ${mutFailure.message ?? ""}\n`);
|
|
497
|
+
return 3;
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
return 0;
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
//#endregion
|
|
504
|
+
//#region src/sweep-cycle.ts
|
|
505
|
+
var SweepStageError = class extends Error {
|
|
506
|
+
name = "SweepStageError";
|
|
507
|
+
stage;
|
|
508
|
+
constructor(stage, cause) {
|
|
509
|
+
super(cause instanceof Error ? cause.message : String(cause), { cause });
|
|
510
|
+
this.stage = stage;
|
|
511
|
+
}
|
|
512
|
+
};
|
|
513
|
+
async function runSweepCycle(sweepFn, reapFn, purgeFn = null, embeddingsGcFn = null) {
|
|
514
|
+
let nextState;
|
|
515
|
+
try {
|
|
516
|
+
nextState = await sweepFn();
|
|
517
|
+
} catch (err) {
|
|
518
|
+
throw new SweepStageError("edge-sweep", err);
|
|
519
|
+
}
|
|
520
|
+
let reap;
|
|
521
|
+
if (reapFn !== null) try {
|
|
522
|
+
reap = await reapFn();
|
|
523
|
+
} catch (err) {
|
|
524
|
+
throw new SweepStageError("idempotency-reap", err);
|
|
525
|
+
}
|
|
526
|
+
let tombstonePurge;
|
|
527
|
+
if (purgeFn !== null) try {
|
|
528
|
+
tombstonePurge = await purgeFn();
|
|
529
|
+
} catch (err) {
|
|
530
|
+
throw new SweepStageError("tombstone-purge", err);
|
|
531
|
+
}
|
|
532
|
+
let embeddingsGc;
|
|
533
|
+
if (embeddingsGcFn !== null) try {
|
|
534
|
+
embeddingsGc = await embeddingsGcFn();
|
|
535
|
+
} catch (err) {
|
|
536
|
+
throw new SweepStageError("embeddings-gc", err);
|
|
537
|
+
}
|
|
538
|
+
const result = { state: nextState };
|
|
539
|
+
if (reap !== void 0) result.reap = reap;
|
|
540
|
+
if (tombstonePurge !== void 0) result.tombstonePurge = tombstonePurge;
|
|
541
|
+
if (embeddingsGc !== void 0) result.embeddingsGc = embeddingsGc;
|
|
542
|
+
return result;
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
//#endregion
|
|
546
|
+
//#region src/serve/background.ts
|
|
547
|
+
/**
|
|
548
|
+
* serve/background.ts — background timers: sweep, invite reaper, sentinel
|
|
549
|
+
* watcher.
|
|
550
|
+
*/
|
|
551
|
+
/** Wall-clock one sweep tick may spend walking edge batches before it yields. */
|
|
552
|
+
const SWEEP_TICK_BUDGET_MS = 3e4;
|
|
553
|
+
const SWEEP_LOCK_KEY = "substrate:sweep";
|
|
554
|
+
function startBackground(args) {
|
|
555
|
+
const { config, sweepDeps, idempotencyStore, dataset, edges, storeSql, stores, composedDatasets, sentinelClientConfig, bridgeRegistry, engine } = args;
|
|
556
|
+
const { recallStore, lexicalStore } = stores;
|
|
557
|
+
const tombstoneRetentionMs = config.tombstoneRetentionDays * MS_PER_DAY;
|
|
558
|
+
const pgDatasetForPurge = storeSql !== null ? dataset : null;
|
|
559
|
+
const pgEdgesForPurge = storeSql !== null ? edges : null;
|
|
560
|
+
const cursorDataset = composedDatasets.get(SYSTEM_NAMESPACE);
|
|
561
|
+
let sweepState = { lastSweptCursor: null };
|
|
562
|
+
const sweepFn = async () => {
|
|
563
|
+
const state = cursorDataset === void 0 ? sweepState : await loadSweepState(cursorDataset);
|
|
564
|
+
const next = await runSweep(sweepDeps, state, void 0, SWEEP_TICK_BUDGET_MS);
|
|
565
|
+
if (cursorDataset !== void 0) await saveSweepState(cursorDataset, next);
|
|
566
|
+
return next;
|
|
567
|
+
};
|
|
568
|
+
let sweepRunning = false;
|
|
569
|
+
const sweepTimer = setInterval(() => {
|
|
570
|
+
if (sweepRunning) return;
|
|
571
|
+
sweepRunning = true;
|
|
572
|
+
const reapFn = idempotencyStore !== null ? () => idempotencyStore.reapExpired() : null;
|
|
573
|
+
const purgeFn = pgDatasetForPurge !== null && pgEdgesForPurge !== null ? async () => {
|
|
574
|
+
const olderThan = new Date(Date.now() - tombstoneRetentionMs);
|
|
575
|
+
const rowsResult = await pgDatasetForPurge.purgeTombstonedRows({ olderThan });
|
|
576
|
+
const edgesResult = await pgEdgesForPurge.purgeTombstonedEdges({ olderThan });
|
|
577
|
+
return {
|
|
578
|
+
rows: rowsResult.deleted,
|
|
579
|
+
edges: edgesResult.deleted
|
|
580
|
+
};
|
|
581
|
+
} : null;
|
|
582
|
+
const pgRecall = recallStore instanceof PgEmbeddingsStore ? recallStore : null;
|
|
583
|
+
const pgLexical = lexicalStore instanceof PgLexicalStore ? lexicalStore : null;
|
|
584
|
+
const embeddingsGcFn = pgRecall !== null || pgLexical !== null ? async () => (await pgRecall?.purgeOrphans() ?? 0) + (await pgLexical?.purgeOrphans() ?? 0) : null;
|
|
585
|
+
const cycle = () => runSweepCycle(sweepFn, reapFn, purgeFn, embeddingsGcFn);
|
|
586
|
+
(storeSql === null ? cycle() : tryWithPgAdvisoryLock(storeSql, SWEEP_LOCK_KEY, cycle).then((r) => {
|
|
587
|
+
if (r.acquired) return r.value;
|
|
588
|
+
process.stdout.write(`sweep.skipped ${JSON.stringify({ reason: "lock_held" })}\n`);
|
|
589
|
+
return null;
|
|
590
|
+
})).then((result) => {
|
|
591
|
+
if (result === null) return;
|
|
592
|
+
const { state, reap, tombstonePurge, embeddingsGc } = result;
|
|
593
|
+
sweepState = state;
|
|
594
|
+
if (reap !== void 0) process.stdout.write(`plugin.idempotency_reap ${JSON.stringify(reap)}\n`);
|
|
595
|
+
if (tombstonePurge !== void 0 && (tombstonePurge.rows > 0 || tombstonePurge.edges > 0)) process.stdout.write(`tombstone_purge ${JSON.stringify(tombstonePurge)}\n`);
|
|
596
|
+
if (embeddingsGc !== void 0 && embeddingsGc > 0) process.stdout.write(`embeddings_gc ${JSON.stringify({ orphans: embeddingsGc })}\n`);
|
|
597
|
+
}).catch((err) => {
|
|
598
|
+
if (err instanceof SweepStageError) process.stderr.write(`sweep cycle failed [${err.stage}]: ${err.message}\n`);
|
|
599
|
+
else process.stderr.write(`sweep cycle failed: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
600
|
+
}).finally(() => {
|
|
601
|
+
sweepRunning = false;
|
|
602
|
+
});
|
|
603
|
+
}, config.sweepIntervalMs);
|
|
604
|
+
sweepTimer.unref();
|
|
605
|
+
let inviteSweepTimer = null;
|
|
606
|
+
if (config.auth !== null) {
|
|
607
|
+
const inviteDataset = composedDatasets.get(SYSTEM_NAMESPACE);
|
|
608
|
+
if (inviteDataset === void 0) process.stderr.write(`invite sweep disabled: '${SYSTEM_NAMESPACE}' dataset missing from registry\n`);
|
|
609
|
+
else {
|
|
610
|
+
inviteSweepTimer = setInterval(() => {
|
|
611
|
+
runInviteSweep(inviteDataset).then((result) => {
|
|
612
|
+
process.stdout.write(`plugin.invite_sweep ${JSON.stringify(result)}\n`);
|
|
613
|
+
}).catch((err) => {
|
|
614
|
+
process.stderr.write(`invite sweep cycle failed: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
615
|
+
});
|
|
616
|
+
}, config.inviteSweepIntervalMs);
|
|
617
|
+
inviteSweepTimer.unref();
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
let sentinelClient = null;
|
|
621
|
+
if (sentinelClientConfig !== null) {
|
|
622
|
+
sentinelClient = startSentinelClient({
|
|
623
|
+
sentinelUrl: sentinelClientConfig.url,
|
|
624
|
+
bridges: bridgeRegistry,
|
|
625
|
+
engine,
|
|
626
|
+
creds: config.bridgeCreds,
|
|
627
|
+
auth: makeAuthorizationProvider(sentinelClientConfig.cred, { audience: new URL(sentinelClientConfig.url.replace(/^ws/, "http")).origin }),
|
|
628
|
+
metric: (name, payload) => {
|
|
629
|
+
process.stdout.write(`sentinel-client.${name} ${JSON.stringify(payload ?? null)}\n`);
|
|
630
|
+
},
|
|
631
|
+
reconnectBaseMs: sentinelClientConfig.reconnectBaseMs,
|
|
632
|
+
reconnectMaxMs: sentinelClientConfig.reconnectMaxMs
|
|
633
|
+
});
|
|
634
|
+
process.stdout.write(`sentinel-client: watching ${sentinelClientConfig.url}\n`);
|
|
635
|
+
}
|
|
636
|
+
return {
|
|
637
|
+
sweepTimer,
|
|
638
|
+
inviteSweepTimer,
|
|
639
|
+
sentinelClient
|
|
640
|
+
};
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
//#endregion
|
|
644
|
+
//#region src/serve/banners.ts
|
|
645
|
+
/**
|
|
646
|
+
* The "listening at" line, with the scheme matching whether TLS is on.
|
|
647
|
+
* Operators copy a base URL from here, and `substrate federate enroll --origin`
|
|
648
|
+
* validates the scheme.
|
|
649
|
+
*/
|
|
650
|
+
function listeningBanner(address, port, tlsEnabled) {
|
|
651
|
+
return `@mnmal-ai/substrate listening at ${tlsEnabled ? "https" : "http"}://${address}:${port}\n`;
|
|
652
|
+
}
|
|
653
|
+
/**
|
|
654
|
+
* The "you have a server that can do nothing" banner, for a plugin-less boot
|
|
655
|
+
* that mounts only `_system` admin + `/health`.
|
|
656
|
+
*
|
|
657
|
+
* Unset and set-but-empty get different text because they have different
|
|
658
|
+
* fixes: point SUBSTRATE_PLUGIN_DIR somewhere, or install packages under
|
|
659
|
+
* `<dir>/node_modules`.
|
|
660
|
+
*/
|
|
661
|
+
function noPluginsBanner(pluginDir) {
|
|
662
|
+
return [
|
|
663
|
+
"",
|
|
664
|
+
"┌─ no plugins ──────────────────────────────────────────────────────┐",
|
|
665
|
+
...pluginDir === null ? [
|
|
666
|
+
"│ SUBSTRATE_PLUGIN_DIR is not set, so no plugins were loaded. │",
|
|
667
|
+
"│ Serving `_system` admin + /health only — no user types. │",
|
|
668
|
+
"│ Point it at a directory holding plugins, e.g.: │",
|
|
669
|
+
"│ SUBSTRATE_PLUGIN_DIR=/srv/substrate (with plugins under node_modules) │"
|
|
670
|
+
] : [
|
|
671
|
+
"│ No plugins found — serving `_system` admin + /health only. │",
|
|
672
|
+
"│ Looked under <SUBSTRATE_PLUGIN_DIR>/node_modules for packages │",
|
|
673
|
+
"│ declaring a `substrate` manifest, and found none. If you have not │",
|
|
674
|
+
"│ installed any there yet, that is the next step. │"
|
|
675
|
+
],
|
|
676
|
+
"└───────────────────────────────────────────────────────────────────┘",
|
|
677
|
+
""
|
|
678
|
+
].join("\n");
|
|
679
|
+
}
|
|
680
|
+
/**
|
|
681
|
+
* Loud banner emitted at boot whenever the substrate is running on memory
|
|
682
|
+
* mode, so operators can't accidentally believe their data is durable.
|
|
683
|
+
*
|
|
684
|
+
* `byDefault` is `true` when `SUBSTRATE_DATABASE_URL` was unset and we fell back to
|
|
685
|
+
* memory; `false` when the operator chose memory explicitly.
|
|
686
|
+
*/
|
|
687
|
+
function memoryStoreBanner(byDefault) {
|
|
688
|
+
return [
|
|
689
|
+
"",
|
|
690
|
+
"┌─ memory mode ─────────────────────────────────────────────────────┐",
|
|
691
|
+
`│ ${(byDefault ? "SUBSTRATE_DATABASE_URL not set — defaulting to in-memory store." : "SUBSTRATE_DATABASE_URL=memory — running with in-memory store.").padEnd(65)} │`,
|
|
692
|
+
"│ Data is NOT persisted; all writes vanish on restart. │",
|
|
693
|
+
"│ For durable storage, set SUBSTRATE_DATABASE_URL to a postgres:// URL, e.g.: │",
|
|
694
|
+
"│ SUBSTRATE_DATABASE_URL='postgres://substrate:substrate@localhost:5432/substrate' │",
|
|
695
|
+
"└───────────────────────────────────────────────────────────────────┘",
|
|
696
|
+
""
|
|
697
|
+
].join("\n");
|
|
698
|
+
}
|
|
699
|
+
/**
|
|
700
|
+
* The knob block — every defaulted knob, its effective value, and whether
|
|
701
|
+
* that value came from the operator or from the default. Sorted by env var
|
|
702
|
+
* and column-aligned so two hosts' blocks can be `diff`ed.
|
|
703
|
+
*
|
|
704
|
+
* `—` is an effective value of null: off, or deferred to a layer further down
|
|
705
|
+
* (Fastify's 1 MiB body cap, the engine's 24h idempotency TTL). Every knob
|
|
706
|
+
* appears whether set or not.
|
|
707
|
+
*/
|
|
708
|
+
function knobsBanner(knobs) {
|
|
709
|
+
const rows = Object.values(knobs).sort((a, b) => a.env < b.env ? -1 : a.env > b.env ? 1 : 0);
|
|
710
|
+
if (rows.length === 0) return "";
|
|
711
|
+
const width = Math.max(...rows.map((r) => r.env.length));
|
|
712
|
+
return [
|
|
713
|
+
"knobs (effective value, source):",
|
|
714
|
+
...rows.map((r) => {
|
|
715
|
+
const value = r.value === null ? "—" : String(r.value);
|
|
716
|
+
return ` ${r.env.padEnd(width)} ${value.padEnd(14)} ${r.source}`;
|
|
717
|
+
}),
|
|
718
|
+
""
|
|
719
|
+
].join("\n");
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
//#endregion
|
|
723
|
+
//#region src/serve/boot-report.ts
|
|
724
|
+
function reportBoot(args) {
|
|
725
|
+
const { config, info, storeSql, substrateNamespaceDataset, coordinators, kernel, pluginDir } = args;
|
|
726
|
+
process.stdout.write(listeningBanner(info.address, info.port, config.tls !== null));
|
|
727
|
+
process.stdout.write(`store: ${redactStoreSpec(config.storeSpec)}\n`);
|
|
728
|
+
if (config.storeSpec === "memory") process.stdout.write(memoryStoreBanner(config.memoryByDefault));
|
|
729
|
+
process.stdout.write(substrateNamespaceDataset !== null ? `_system: pg-backed\n` : `_system: memory-backed (keys ephemeral)\n`);
|
|
730
|
+
if (coordinators.length > 0) process.stdout.write(`coordinators: ${coordinators.map((p) => p.name).join(", ")}\n`);
|
|
731
|
+
if (kernel.length > 0) process.stdout.write(`kernel: ${kernel.map((p) => p.name).join(", ")}\n`);
|
|
732
|
+
if (pluginDir !== null) process.stdout.write(`plugin-dir: ${pluginDir}\n`);
|
|
733
|
+
if (coordinators.length === 0 && kernel.length === 0) process.stdout.write(noPluginsBanner(pluginDir));
|
|
734
|
+
if (config.bridges.length > 0) process.stdout.write(`bridges: ${config.bridges.map((b) => b.ns).join(", ")}\n`);
|
|
735
|
+
if (config.auth !== null) {
|
|
736
|
+
const fileKeys = config.auth.keys.length;
|
|
737
|
+
const source = fileKeys === 0 ? "database-registered keys only" : `${fileKeys} file key(s)`;
|
|
738
|
+
process.stdout.write(`auth: enabled, mode=${config.auth.mode ?? "required"} (${source})\n`);
|
|
739
|
+
const reprime = config.keyReprimeIntervalMs;
|
|
740
|
+
const reprimeSource = ` (${config.knobs.keyReprimeIntervalMs?.source ?? "default"})`;
|
|
741
|
+
process.stdout.write(reprime === null ? `key cache: reprime off${reprimeSource}\n` : `key cache: reprime every ${reprime}ms${reprimeSource}\n`);
|
|
742
|
+
} else process.stdout.write("auth: DISABLED — all requests are anonymous\n");
|
|
743
|
+
if (config.tls !== null) process.stdout.write(`tls: enabled${config.tls.ca !== void 0 ? " (with CA chain)" : ""}\n`);
|
|
744
|
+
if (config.idempotency.disabled) process.stdout.write(`idempotency: disabled\n`);
|
|
745
|
+
else {
|
|
746
|
+
const backend = storeSql !== null ? "pg-backed" : "memory-backed";
|
|
747
|
+
const ttl = config.idempotency.defaultTtlMs === null ? "default 24h" : `${config.idempotency.defaultTtlMs}ms`;
|
|
748
|
+
process.stdout.write(`idempotency: enabled (${backend}, ttl=${ttl})\n`);
|
|
749
|
+
}
|
|
750
|
+
if (storeSql !== null) process.stdout.write(`tombstone purge: enabled (retention=${config.tombstoneRetentionDays}d, cadence=${config.sweepIntervalMs}ms)\n`);
|
|
751
|
+
process.stdout.write(knobsBanner(config.knobs));
|
|
752
|
+
}
|
|
753
|
+
|
|
754
|
+
//#endregion
|
|
755
|
+
//#region src/serve/shutdown.ts
|
|
756
|
+
/** The longest shutdown waits for the background index builds to stop. */
|
|
757
|
+
const INDEX_BUILDS_STOP_MS = 2e3;
|
|
758
|
+
/**
|
|
759
|
+
* Re-entrancy guard: systemd retries SIGTERM under load, and a double-fire
|
|
760
|
+
* here would clear already-cleared timers and call `stop()` twice. The flag
|
|
761
|
+
* flips on first entry and stays set; subsequent signals just log.
|
|
762
|
+
*/
|
|
763
|
+
function installShutdown(args) {
|
|
764
|
+
const { config, sweepTimer, inviteSweepTimer, sentinelClient, disposeBridges, stop, closeStore, stopIndexBuilds } = args;
|
|
765
|
+
let shuttingDown = false;
|
|
766
|
+
const shutdown = (signal) => {
|
|
767
|
+
if (shuttingDown) {
|
|
768
|
+
process.stdout.write(`received ${signal} again — already shutting down\n`);
|
|
769
|
+
return;
|
|
770
|
+
}
|
|
771
|
+
shuttingDown = true;
|
|
772
|
+
process.stdout.write(`\nreceived ${signal} — shutting down (grace ${config.shutdownGraceMs}ms)\n`);
|
|
773
|
+
setTimeout(() => {
|
|
774
|
+
process.stderr.write("shutdown exceeded its grace; exiting with work still pending\n");
|
|
775
|
+
process.exit(1);
|
|
776
|
+
}, config.shutdownGraceMs + 1e3).unref();
|
|
777
|
+
clearInterval(sweepTimer);
|
|
778
|
+
if (inviteSweepTimer !== null) clearInterval(inviteSweepTimer);
|
|
779
|
+
const buildsBound = Math.min(INDEX_BUILDS_STOP_MS, config.shutdownGraceMs / 2);
|
|
780
|
+
const stopBuilds = new Promise((resolve) => {
|
|
781
|
+
if (stopIndexBuilds === void 0) return resolve();
|
|
782
|
+
const timer = setTimeout(() => {
|
|
783
|
+
process.stderr.write(`index builds did not stop within ${buildsBound}ms; abandoning them\n`);
|
|
784
|
+
resolve();
|
|
785
|
+
}, buildsBound);
|
|
786
|
+
timer.unref();
|
|
787
|
+
stopIndexBuilds().catch(() => void 0).then(() => {
|
|
788
|
+
clearTimeout(timer);
|
|
789
|
+
resolve();
|
|
790
|
+
});
|
|
791
|
+
});
|
|
792
|
+
(sentinelClient !== null ? sentinelClient.stop().catch((err) => {
|
|
793
|
+
process.stderr.write(`sentinel-client stop failed: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
794
|
+
}) : Promise.resolve()).then(() => {
|
|
795
|
+
disposeBridges();
|
|
796
|
+
return stop({
|
|
797
|
+
grace: config.shutdownGraceMs,
|
|
798
|
+
abortInFlight: true
|
|
799
|
+
});
|
|
800
|
+
}).then(async () => {
|
|
801
|
+
await stopBuilds;
|
|
802
|
+
if (closeStore) await closeStore();
|
|
803
|
+
}).then(() => {
|
|
804
|
+
process.exit(0);
|
|
805
|
+
}).catch((err) => {
|
|
806
|
+
process.stderr.write(`shutdown error: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
807
|
+
process.exit(1);
|
|
808
|
+
});
|
|
809
|
+
};
|
|
810
|
+
process.on("SIGTERM", () => shutdown("SIGTERM"));
|
|
811
|
+
process.on("SIGINT", () => shutdown("SIGINT"));
|
|
812
|
+
}
|
|
813
|
+
|
|
814
|
+
//#endregion
|
|
815
|
+
//#region src/main.ts
|
|
816
|
+
/**
|
|
817
|
+
* @mnmal-ai/substrate — §5.12 binary entry point.
|
|
818
|
+
*
|
|
819
|
+
* Wires SubstrateServer + Dataset + EdgesStore + optional coordinator plugins
|
|
820
|
+
* + optional bridges + optional JWT auth into a running process. The entry
|
|
821
|
+
* surface is env-var driven:
|
|
822
|
+
*
|
|
823
|
+
* SUBSTRATE_DATABASE_URL memory | postgres://... (default: memory)
|
|
824
|
+
* SUBSTRATE_PLUGIN_DIR abs path to plugin install root (optional). The
|
|
825
|
+
* manifest-driven loader scans
|
|
826
|
+
* <dir>/node_modules/{*,@*\/*} for packages
|
|
827
|
+
* declaring a `substrate` manifest, merges each
|
|
828
|
+
* plugin's default-config.json with the colocated
|
|
829
|
+
* operator override at <dir>/<basename>.config.json,
|
|
830
|
+
* and bootstraps substrate plugins (alphabetical)
|
|
831
|
+
* before coordinators (alphabetical). Unset = no
|
|
832
|
+
* plugins; binary boots with `_system` admin
|
|
833
|
+
* namespace + /health only.
|
|
834
|
+
* SUBSTRATE_BRIDGES JSON array of { ns, name, url } bridge defs (optional)
|
|
835
|
+
* SUBSTRATE_BRIDGE_CREDENTIALS JSON map bridgeName -> { keyPath, kid, origin }.
|
|
836
|
+
* REQUIRED for every entry in SUBSTRATE_BRIDGES — a
|
|
837
|
+
* bridge without one refuses to boot rather than
|
|
838
|
+
* connecting unauthenticated.
|
|
839
|
+
* SUBSTRATE_SENTINEL_CREDENTIAL JSON { keyPath, kid } the sentinel-client signs its
|
|
840
|
+
* frame-stream requests with. REQUIRED whenever
|
|
841
|
+
* SUBSTRATE_SENTINEL_URL is set.
|
|
842
|
+
* SUBSTRATE_AUTH_KEY_REGISTRY path to JSON key file (optional — auth opt-in)
|
|
843
|
+
* SUBSTRATE_ADMIN_CAPABILITY JWT claim value granting admin (optional)
|
|
844
|
+
* SUBSTRATE_JWT_LEEWAY_SECONDS clock-skew tolerance for JWT exp/nbf (optional;
|
|
845
|
+
* default 0 = strict). 30 absorbs typical NTP drift.
|
|
846
|
+
* SUBSTRATE_KEY_REFRESH_INTERVAL_MS periodic full re-prime of the public-key
|
|
847
|
+
* cache in ms (default 30000; `off` disables). Bounds
|
|
848
|
+
* revocation staleness. Auth-gated.
|
|
849
|
+
* SUBSTRATE_TLS_KEY_PATH path to TLS private key PEM (optional — TLS opt-in)
|
|
850
|
+
* SUBSTRATE_TLS_CERT_PATH path to TLS certificate PEM (optional — TLS opt-in)
|
|
851
|
+
* SUBSTRATE_TLS_CA_PATH path to CA chain PEM (optional)
|
|
852
|
+
* SUBSTRATE_BODY_LIMIT_BYTES max request body size in bytes (optional). Unset =
|
|
853
|
+
* Fastify's 1 MiB default. Raise for bulk batched
|
|
854
|
+
* mutation frames that exceed 1 MiB.
|
|
855
|
+
* SUBSTRATE_STATEMENT_TIMEOUT_MS statement timeout for walk reads in ms (default 30000);
|
|
856
|
+
* a cancelled read is refused `query_timeout`
|
|
857
|
+
* SUBSTRATE_TYPE_TABLE_BUDGET type tables above which the storage report warns (default 500);
|
|
858
|
+
* the deprecated SUBSTRATE_PARTITION_BUDGET is read while it is unset
|
|
859
|
+
* SUBSTRATE_SWEEP_INTERVAL_MS sweep cadence in ms (default 300000)
|
|
860
|
+
* SUBSTRATE_INVITE_SWEEP_INTERVAL_MS invite reaper cadence in ms (default 3600000 = 1h)
|
|
861
|
+
* SUBSTRATE_TOMBSTONE_RETENTION_DAYS retention window for tombstoned rows /
|
|
862
|
+
* edges before they're hard-deleted by the
|
|
863
|
+
* sweep loop. Default 30. Pg-backed only — memory
|
|
864
|
+
* mode tombstones vanish on restart anyway.
|
|
865
|
+
* SUBSTRATE_IDEMPOTENCY_TTL_MS override §5.13 dedup TTL (default: engine 24h)
|
|
866
|
+
* SUBSTRATE_IDEMPOTENCY_DISABLED truthy (1/true) disables mutation dedup
|
|
867
|
+
* SUBSTRATE_TRUSTED_PROXIES CSV of exact peer-IP strings whose x-forwarded-for
|
|
868
|
+
* header is honoured by `/substrate/v1/_system/register`.
|
|
869
|
+
* Exact IP strings only — no CIDR. Callers needing
|
|
870
|
+
* CIDR should terminate XFF parsing at their reverse
|
|
871
|
+
* proxy. See §13 for the scope limit.
|
|
872
|
+
* SUBSTRATE_CORS_ORIGINS comma-separated list of allowed CORS origins for
|
|
873
|
+
* browser clients, or '*' to allow any origin.
|
|
874
|
+
* Unset (default) = no CORS plugin registered —
|
|
875
|
+
* agent-only deployments don't need it.
|
|
876
|
+
* Example: SUBSTRATE_CORS_ORIGINS=https://app.example.com
|
|
877
|
+
* Example: SUBSTRATE_CORS_ORIGINS=* (allow any origin;
|
|
878
|
+
* pairs with credentials:false — anonymous only)
|
|
879
|
+
* PORT / HOST listen args (default 3100; 0.0.0.0 with auth, 127.0.0.1 without)
|
|
880
|
+
*
|
|
881
|
+
* Env parsing is the pure `parseEnv(env)`; `runServe()` sequences the I/O.
|
|
882
|
+
*/
|
|
883
|
+
/**
|
|
884
|
+
* Boot the Substrate HTTP server. Reads env, wires substrate + coordinators,
|
|
885
|
+
* starts Fastify, registers signal handlers for graceful shutdown. Invoked
|
|
886
|
+
* through the `substrate serve` subcommand.
|
|
887
|
+
*/
|
|
888
|
+
async function runServe() {
|
|
889
|
+
validateHardDeleteEnv();
|
|
890
|
+
const config = parseEnv(process.env);
|
|
891
|
+
const pluginDir = resolvePluginDir();
|
|
892
|
+
const { engine, datasets: composedDatasets, sql: storeSql, server, dataset, edges, substrateNamespaceDataset, closeStore, kernel, coordinators, recallStore, lexicalStore, idempotencyStore, sentinelClientConfig, bridgeRegistry, disposeBridges, sweepDeps, scheduler, stop } = await bootStore({
|
|
893
|
+
config,
|
|
894
|
+
pluginDir
|
|
895
|
+
});
|
|
896
|
+
if (config.indexDiskPath === null) await reportUnguardedIndexBuilds(engine, composedDatasets);
|
|
897
|
+
const info = await server.start({
|
|
898
|
+
host: config.host,
|
|
899
|
+
port: config.port
|
|
900
|
+
});
|
|
901
|
+
const indexBuilds = new AbortController();
|
|
902
|
+
const deferredIndexes = runDeferredIndexBuilds(engine, composedDatasets, config.indexRetryIntervalMs, indexBuilds.signal);
|
|
903
|
+
if (scheduler !== null) scheduler.start();
|
|
904
|
+
reportBoot({
|
|
905
|
+
config,
|
|
906
|
+
info,
|
|
907
|
+
storeSql,
|
|
908
|
+
substrateNamespaceDataset,
|
|
909
|
+
coordinators,
|
|
910
|
+
kernel,
|
|
911
|
+
pluginDir
|
|
912
|
+
});
|
|
913
|
+
const { sweepTimer, inviteSweepTimer, sentinelClient } = startBackground({
|
|
914
|
+
config,
|
|
915
|
+
sweepDeps,
|
|
916
|
+
idempotencyStore,
|
|
917
|
+
dataset,
|
|
918
|
+
edges,
|
|
919
|
+
storeSql,
|
|
920
|
+
stores: {
|
|
921
|
+
recallStore,
|
|
922
|
+
lexicalStore
|
|
923
|
+
},
|
|
924
|
+
composedDatasets,
|
|
925
|
+
sentinelClientConfig,
|
|
926
|
+
bridgeRegistry,
|
|
927
|
+
engine
|
|
928
|
+
});
|
|
929
|
+
installShutdown({
|
|
930
|
+
config,
|
|
931
|
+
sweepTimer,
|
|
932
|
+
inviteSweepTimer,
|
|
933
|
+
sentinelClient,
|
|
934
|
+
disposeBridges,
|
|
935
|
+
stop,
|
|
936
|
+
closeStore,
|
|
937
|
+
stopIndexBuilds: () => {
|
|
938
|
+
indexBuilds.abort();
|
|
939
|
+
return deferredIndexes;
|
|
940
|
+
}
|
|
941
|
+
});
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
//#endregion
|
|
945
|
+
//#region src/cli/federate/http.ts
|
|
946
|
+
/**
|
|
947
|
+
* Minimal JSON-over-HTTP client for the `substrate federate` verbs.
|
|
948
|
+
*
|
|
949
|
+
* A non-2xx status is a RESULT, not an error: callers map peer error codes to
|
|
950
|
+
* specific operator-facing messages, which they cannot do from an exception.
|
|
951
|
+
* Only transport failures and timeouts reject.
|
|
952
|
+
*
|
|
953
|
+
* There is deliberately no way to skip TLS verification. Every caller of this
|
|
954
|
+
* module puts a secret on the wire — `join` sends a single-use invite code an
|
|
955
|
+
* active MITM could steal and redeem as itself, and `check` sends a live
|
|
956
|
+
* bearer JWT. A self-signed lab CA belongs in NODE_EXTRA_CA_CERTS, which Node
|
|
957
|
+
* honours natively and which still verifies the chain.
|
|
958
|
+
*/
|
|
959
|
+
const DEFAULT_TIMEOUT_MS = 1e4;
|
|
960
|
+
var HttpJsonError = class extends Error {
|
|
961
|
+
name = "HttpJsonError";
|
|
962
|
+
};
|
|
963
|
+
async function httpJson(url, opts = {}) {
|
|
964
|
+
const headers = {
|
|
965
|
+
accept: "application/json",
|
|
966
|
+
...opts.headers
|
|
967
|
+
};
|
|
968
|
+
const init = {
|
|
969
|
+
method: opts.method ?? "GET",
|
|
970
|
+
headers,
|
|
971
|
+
signal: AbortSignal.timeout(opts.timeoutMs ?? DEFAULT_TIMEOUT_MS)
|
|
972
|
+
};
|
|
973
|
+
if (opts.body !== void 0) {
|
|
974
|
+
headers["content-type"] = "application/json";
|
|
975
|
+
init.body = JSON.stringify(opts.body);
|
|
976
|
+
}
|
|
977
|
+
try {
|
|
978
|
+
tokenAudience(url);
|
|
979
|
+
} catch (err) {
|
|
980
|
+
throw new HttpJsonError(err instanceof Error ? err.message : String(err));
|
|
981
|
+
}
|
|
982
|
+
init.redirect = "error";
|
|
983
|
+
let res;
|
|
984
|
+
try {
|
|
985
|
+
res = await substrateFetch(url, init);
|
|
986
|
+
} catch (err) {
|
|
987
|
+
throw new HttpJsonError(`${url}: ${err instanceof Error ? err.message : String(err)}`);
|
|
988
|
+
}
|
|
989
|
+
const raw = await res.text();
|
|
990
|
+
let body;
|
|
991
|
+
try {
|
|
992
|
+
body = raw.length > 0 ? JSON.parse(raw) : void 0;
|
|
993
|
+
} catch {
|
|
994
|
+
body = void 0;
|
|
995
|
+
}
|
|
996
|
+
return {
|
|
997
|
+
status: res.status,
|
|
998
|
+
body,
|
|
999
|
+
raw
|
|
1000
|
+
};
|
|
1001
|
+
}
|
|
1002
|
+
|
|
1003
|
+
//#endregion
|
|
1004
|
+
//#region src/cli/federate/check.ts
|
|
1005
|
+
/**
|
|
1006
|
+
* `substrate federate check` — diagnose a configured bridge, naming the fault.
|
|
1007
|
+
*
|
|
1008
|
+
* All six failure modes below present to a caller identically, as
|
|
1009
|
+
* `_completeness: { complete: false, unreachable: ["peer-b"] }`; this command
|
|
1010
|
+
* tells them apart.
|
|
1011
|
+
*
|
|
1012
|
+
* Steps run in dependency order and stop at the first failure — a scope
|
|
1013
|
+
* verdict computed against an unreachable peer would be noise. That applies to
|
|
1014
|
+
* config/key file/reachable/auth, which are properties of the coordinator, not
|
|
1015
|
+
* of any one namespace. Scope and namespace are properties of a GRANT, and one
|
|
1016
|
+
* coordinator can carry several — `formatJoinOutput` emits one `SUBSTRATE_BRIDGES`
|
|
1017
|
+
* entry per namespace under a single coordinator name — so those two steps run
|
|
1018
|
+
* once per configured namespace rather than stopping at the first.
|
|
1019
|
+
*/
|
|
1020
|
+
/**
|
|
1021
|
+
* Step names for the per-namespace pair. A single namespace (the common case)
|
|
1022
|
+
* keeps the plain `scope`/`namespace` labels; two or more get the namespace
|
|
1023
|
+
* folded into the label so an operator can see which one failed without
|
|
1024
|
+
* cross-referencing detail text.
|
|
1025
|
+
*/
|
|
1026
|
+
function namespaceStepNames(namespaces) {
|
|
1027
|
+
if (namespaces.length <= 1) return ["scope", "namespace"];
|
|
1028
|
+
return namespaces.flatMap((ns) => [`scope:${ns}`, `namespace:${ns}`]);
|
|
1029
|
+
}
|
|
1030
|
+
async function runCheck(name, env, deps = {}) {
|
|
1031
|
+
const call = deps.httpJson ?? httpJson;
|
|
1032
|
+
const mintToken = deps.tokenFor ?? ((cred) => makeAuthorizationProvider(cred)());
|
|
1033
|
+
const specs = parseBridges(env.SUBSTRATE_BRIDGES);
|
|
1034
|
+
const creds = parseBridgeCreds(env.SUBSTRATE_BRIDGE_CREDENTIALS);
|
|
1035
|
+
const byName = /* @__PURE__ */ new Map();
|
|
1036
|
+
for (const spec of specs) {
|
|
1037
|
+
const existing = byName.get(spec.name);
|
|
1038
|
+
if (existing === void 0) byName.set(spec.name, [spec]);
|
|
1039
|
+
else existing.push(spec);
|
|
1040
|
+
}
|
|
1041
|
+
const targets = name === void 0 ? [...byName.entries()].map(([n, list]) => ({
|
|
1042
|
+
name: n,
|
|
1043
|
+
specs: list
|
|
1044
|
+
})) : [{
|
|
1045
|
+
name,
|
|
1046
|
+
specs: byName.get(name) ?? []
|
|
1047
|
+
}];
|
|
1048
|
+
if (targets.length === 0) throw new AdminCliError("no bridges configured — SUBSTRATE_BRIDGES is empty. Run `substrate federate join` first, or pass --env-file pointing at the env file your instance actually reads.", 2);
|
|
1049
|
+
const results = [];
|
|
1050
|
+
for (const target of targets) results.push(await checkOne(target.name, target.specs, creds, call, mintToken));
|
|
1051
|
+
return results;
|
|
1052
|
+
}
|
|
1053
|
+
async function checkOne(bridgeName, specs, creds, call, mintToken) {
|
|
1054
|
+
const spec = specs[0];
|
|
1055
|
+
const steps = [
|
|
1056
|
+
"config",
|
|
1057
|
+
"key file",
|
|
1058
|
+
"reachable",
|
|
1059
|
+
"auth",
|
|
1060
|
+
...namespaceStepNames(specs.map((s) => s.ns))
|
|
1061
|
+
].map((n) => ({
|
|
1062
|
+
name: n,
|
|
1063
|
+
status: "skipped",
|
|
1064
|
+
detail: ""
|
|
1065
|
+
}));
|
|
1066
|
+
const setStep = (index, status, detail) => {
|
|
1067
|
+
const step = steps[index];
|
|
1068
|
+
if (step !== void 0) {
|
|
1069
|
+
step.status = status;
|
|
1070
|
+
step.detail = detail;
|
|
1071
|
+
}
|
|
1072
|
+
};
|
|
1073
|
+
const pass = (index, detail) => setStep(index, "ok", detail);
|
|
1074
|
+
const fail = (index, detail) => {
|
|
1075
|
+
setStep(index, "fail", detail);
|
|
1076
|
+
return {
|
|
1077
|
+
bridge: bridgeName,
|
|
1078
|
+
ok: false,
|
|
1079
|
+
steps
|
|
1080
|
+
};
|
|
1081
|
+
};
|
|
1082
|
+
if (spec === void 0) return fail(0, `bridge '${bridgeName}' is not configured in SUBSTRATE_BRIDGES`);
|
|
1083
|
+
const cred = creds.get(bridgeName);
|
|
1084
|
+
if (cred === void 0) return fail(0, `bridge '${bridgeName}' has no entry in SUBSTRATE_BRIDGE_CREDENTIALS`);
|
|
1085
|
+
let specOrigin;
|
|
1086
|
+
try {
|
|
1087
|
+
specOrigin = new URL(spec.url).origin;
|
|
1088
|
+
} catch {
|
|
1089
|
+
return fail(0, `bridge url '${spec.url}' is malformed`);
|
|
1090
|
+
}
|
|
1091
|
+
if (specOrigin !== cred.origin) return fail(0, `bridge url origin '${specOrigin}' does not match the credential origin '${cred.origin}'`);
|
|
1092
|
+
pass(0, `SUBSTRATE_BRIDGES + CREDS agree on ${specOrigin}`);
|
|
1093
|
+
if (!existsSync(cred.keyPath)) return fail(1, `key file '${cred.keyPath}' does not exist`);
|
|
1094
|
+
let authorization;
|
|
1095
|
+
try {
|
|
1096
|
+
authorization = mintToken(cred);
|
|
1097
|
+
} catch (err) {
|
|
1098
|
+
return fail(1, `key file '${cred.keyPath}' could not be used to mint a token: ${err instanceof Error ? err.message : String(err)}`);
|
|
1099
|
+
}
|
|
1100
|
+
pass(1, cred.keyPath);
|
|
1101
|
+
try {
|
|
1102
|
+
const health = await call(`${cred.origin}/health`);
|
|
1103
|
+
if (health.status !== 200) return fail(2, `${cred.origin}/health returned HTTP ${health.status}`);
|
|
1104
|
+
} catch (err) {
|
|
1105
|
+
return fail(2, `${cred.origin} — ${err instanceof Error ? err.message : String(err)}`);
|
|
1106
|
+
}
|
|
1107
|
+
pass(2, `${cred.origin}/health`);
|
|
1108
|
+
let whoami;
|
|
1109
|
+
try {
|
|
1110
|
+
const res = await call(`${cred.origin}/substrate/v1/whoami`, { headers: { authorization } });
|
|
1111
|
+
if (res.status === 401) return fail(3, `peer does not recognise kid '${cred.kid}' — register it there, or it was revoked`);
|
|
1112
|
+
if (res.status === 404) return fail(3, `${cred.origin} does not expose /substrate/v1/whoami — the peer is likely running a release that predates it, or one still serving the older \`/substrate/v1/_system/whoami\`. Upgrade the peer, or verify auth by hand in the meantime.`);
|
|
1113
|
+
if (res.status !== 200) return fail(3, `whoami returned HTTP ${res.status}`);
|
|
1114
|
+
whoami = res.body ?? {};
|
|
1115
|
+
} catch (err) {
|
|
1116
|
+
return fail(3, `whoami failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
1117
|
+
}
|
|
1118
|
+
if (whoami.authEnabled !== true) return fail(3, "peer reports auth DISABLED — your credential is not being verified. Anything that \"works\" against it proves nothing");
|
|
1119
|
+
if (whoami.identity === null || whoami.identity === void 0) return fail(3, `the peer verified nothing for kid '${cred.kid}' — auth is enabled there, but this request arrived without a credential. Check SUBSTRATE_BRIDGE_CREDENTIALS / the key path rather than the peer`);
|
|
1120
|
+
const granted = grantOf(whoami.identity);
|
|
1121
|
+
const summary = granted === null ? "peer reported no grant" : Object.entries(granted).map(([ns, caps]) => `${ns}:${caps.length > 0 ? caps.join("+") : "(nothing)"}`).join(", ");
|
|
1122
|
+
pass(3, `peer accepted kid '${cred.kid}' — grant: ${summary}`);
|
|
1123
|
+
let anyNsFailed = false;
|
|
1124
|
+
for (let i = 0; i < specs.length; i++) {
|
|
1125
|
+
const nsSpec = specs[i];
|
|
1126
|
+
if (nsSpec === void 0) continue;
|
|
1127
|
+
const scopeIdx = 4 + i * 2;
|
|
1128
|
+
const namespaceIdx = 5 + i * 2;
|
|
1129
|
+
if (!coversNamespace(granted, nsSpec.ns)) {
|
|
1130
|
+
setStep(scopeIdx, "fail", `peer grants ${summary} — this bridge needs '${nsSpec.ns}'`);
|
|
1131
|
+
anyNsFailed = true;
|
|
1132
|
+
continue;
|
|
1133
|
+
}
|
|
1134
|
+
const here = grantedIn(granted, nsSpec.ns);
|
|
1135
|
+
const writable = here.includes("mutate") || here.includes("admin");
|
|
1136
|
+
setStep(scopeIdx, "ok", `peer grants '${nsSpec.ns}': ${here.length === 0 ? "(nothing)" : here.join(", ")}` + (writable ? "" : " — read-only, writes through this bridge will be refused"));
|
|
1137
|
+
try {
|
|
1138
|
+
const schema = await call(`${cred.origin}/substrate/v1/schema/${nsSpec.ns}`);
|
|
1139
|
+
if (schema.status !== 200) {
|
|
1140
|
+
setStep(namespaceIdx, "fail", `peer is reachable and authorised but does not serve namespace '${nsSpec.ns}' (HTTP ${schema.status})`);
|
|
1141
|
+
anyNsFailed = true;
|
|
1142
|
+
continue;
|
|
1143
|
+
}
|
|
1144
|
+
} catch (err) {
|
|
1145
|
+
setStep(namespaceIdx, "fail", `schema lookup failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
1146
|
+
anyNsFailed = true;
|
|
1147
|
+
continue;
|
|
1148
|
+
}
|
|
1149
|
+
setStep(namespaceIdx, "ok", `peer serves '${nsSpec.ns}'`);
|
|
1150
|
+
}
|
|
1151
|
+
return {
|
|
1152
|
+
bridge: bridgeName,
|
|
1153
|
+
ok: !anyNsFailed,
|
|
1154
|
+
steps
|
|
1155
|
+
};
|
|
1156
|
+
}
|
|
1157
|
+
function formatCheckOutput(results) {
|
|
1158
|
+
const lines = [];
|
|
1159
|
+
for (const result of results) {
|
|
1160
|
+
lines.push("", `${result.bridge}`);
|
|
1161
|
+
for (const step of result.steps) {
|
|
1162
|
+
const mark = step.status === "ok" ? "OK " : step.status === "fail" ? "FAIL" : "-- ";
|
|
1163
|
+
lines.push(` ${mark} ${step.name.padEnd(10)} ${step.detail}`);
|
|
1164
|
+
}
|
|
1165
|
+
}
|
|
1166
|
+
lines.push("");
|
|
1167
|
+
return lines.join("\n");
|
|
1168
|
+
}
|
|
1169
|
+
|
|
1170
|
+
//#endregion
|
|
1171
|
+
//#region src/cli/federate/duration.ts
|
|
1172
|
+
/**
|
|
1173
|
+
* `--ttl` parsing for `substrate federate enroll`.
|
|
1174
|
+
*
|
|
1175
|
+
* The 30-day ceiling matches MAX_TTL_MS in the `_system` invites route, so a
|
|
1176
|
+
* CLI-minted invite can never outlive what the HTTP route would allow.
|
|
1177
|
+
*/
|
|
1178
|
+
const MAX_TTL_MS = 2592e6;
|
|
1179
|
+
const DURATION = /^(\d+)(s|m|h|d)$/;
|
|
1180
|
+
const UNIT_MS = {
|
|
1181
|
+
s: 1e3,
|
|
1182
|
+
m: 6e4,
|
|
1183
|
+
h: 36e5,
|
|
1184
|
+
d: 864e5
|
|
1185
|
+
};
|
|
1186
|
+
function parseDuration(raw) {
|
|
1187
|
+
const match = DURATION.exec(raw.trim());
|
|
1188
|
+
if (match === null) throw new Error(`invalid duration '${raw}' — expected a value like 30s, 15m, 24h, 7d`);
|
|
1189
|
+
const ms = Number(match[1]) * UNIT_MS[match[2]];
|
|
1190
|
+
if (ms <= 0) throw new Error(`invalid duration '${raw}' — must be greater than zero`);
|
|
1191
|
+
if (ms > 2592e6) throw new Error(`invalid duration '${raw}' — must not exceed 30 days`);
|
|
1192
|
+
return ms;
|
|
1193
|
+
}
|
|
1194
|
+
|
|
1195
|
+
//#endregion
|
|
1196
|
+
//#region src/cli/federate/probe.ts
|
|
1197
|
+
/**
|
|
1198
|
+
* Auth-posture probe used by `substrate federate enroll`.
|
|
1199
|
+
*
|
|
1200
|
+
* A token minted against a callee with auth disabled yields a credential that
|
|
1201
|
+
* proves nothing, with no signal to the operator — so enroll refuses that
|
|
1202
|
+
* case. Anything OTHER than a clear "auth is off" answer reads as
|
|
1203
|
+
* `unreachable`, which enroll treats as a warning: a host legitimately may not
|
|
1204
|
+
* reach its own public URL (split-horizon DNS), and that is not evidence about
|
|
1205
|
+
* auth posture. Guessing either way from an ambiguous response would break one
|
|
1206
|
+
* of the two cases.
|
|
1207
|
+
*/
|
|
1208
|
+
const PROBE_TIMEOUT_MS = 5e3;
|
|
1209
|
+
async function probeAuthPosture(origin, opts = {}) {
|
|
1210
|
+
const call = opts.httpJson ?? httpJson;
|
|
1211
|
+
let res;
|
|
1212
|
+
try {
|
|
1213
|
+
res = await call(`${origin}/substrate/v1/whoami`, { timeoutMs: PROBE_TIMEOUT_MS });
|
|
1214
|
+
} catch {
|
|
1215
|
+
return "unreachable";
|
|
1216
|
+
}
|
|
1217
|
+
if (res.status < 200 || res.status >= 300) return "unreachable";
|
|
1218
|
+
const body = res.body;
|
|
1219
|
+
if (body === null || typeof body !== "object") return "unreachable";
|
|
1220
|
+
const authEnabled = body.authEnabled;
|
|
1221
|
+
if (authEnabled === true) return "enabled";
|
|
1222
|
+
if (authEnabled === false) return "disabled";
|
|
1223
|
+
return "unreachable";
|
|
1224
|
+
}
|
|
1225
|
+
|
|
1226
|
+
//#endregion
|
|
1227
|
+
//#region src/cli/federate/token.ts
|
|
1228
|
+
/**
|
|
1229
|
+
* The `substrate federate` join-token envelope.
|
|
1230
|
+
*
|
|
1231
|
+
* substrate-join-v1.<base64url(JSON)> payload: { origin, code }
|
|
1232
|
+
*
|
|
1233
|
+
* Origin and code only. Namespaces and capabilities are deliberately absent:
|
|
1234
|
+
* redemption returns them from the invite row, which is authoritative, and a
|
|
1235
|
+
* second copy in the token would be the one the operator reads while not
|
|
1236
|
+
* being the one that binds.
|
|
1237
|
+
*
|
|
1238
|
+
* The versioned prefix makes a pasted token self-identifying, greppable by
|
|
1239
|
+
* secret scanners, and safe to change later. `.` cannot collide with the
|
|
1240
|
+
* base64url alphabet (which contains `-` and `_`).
|
|
1241
|
+
*/
|
|
1242
|
+
const TOKEN_PREFIX = "substrate-join-v1.";
|
|
1243
|
+
/** base64url alphabet, unpadded. Checked explicitly because Buffer's
|
|
1244
|
+
* 'base64url' decoder is lenient and silently drops invalid bytes. */
|
|
1245
|
+
const BASE64URL = /^[A-Za-z0-9_-]+$/;
|
|
1246
|
+
var JoinTokenError = class extends Error {
|
|
1247
|
+
name = "JoinTokenError";
|
|
1248
|
+
exitCode;
|
|
1249
|
+
constructor(message, exitCode = 2) {
|
|
1250
|
+
super(message);
|
|
1251
|
+
this.exitCode = exitCode;
|
|
1252
|
+
}
|
|
1253
|
+
};
|
|
1254
|
+
function encodeJoinToken(payload) {
|
|
1255
|
+
const json = JSON.stringify({
|
|
1256
|
+
origin: payload.origin,
|
|
1257
|
+
code: payload.code
|
|
1258
|
+
});
|
|
1259
|
+
return TOKEN_PREFIX + Buffer.from(json, "utf8").toString("base64url");
|
|
1260
|
+
}
|
|
1261
|
+
function decodeJoinToken(token) {
|
|
1262
|
+
const trimmed = token.trim();
|
|
1263
|
+
if (!trimmed.startsWith(TOKEN_PREFIX)) throw new JoinTokenError(`not a substrate join token — expected a value starting with '${TOKEN_PREFIX}'`);
|
|
1264
|
+
const encoded = trimmed.slice(18);
|
|
1265
|
+
if (encoded.length === 0) throw new JoinTokenError("join token has an empty payload");
|
|
1266
|
+
if (!BASE64URL.test(encoded)) throw new JoinTokenError("join token payload is not valid base64url");
|
|
1267
|
+
const json = Buffer.from(encoded, "base64url").toString("utf8");
|
|
1268
|
+
let parsed;
|
|
1269
|
+
try {
|
|
1270
|
+
parsed = JSON.parse(json);
|
|
1271
|
+
} catch {
|
|
1272
|
+
throw new JoinTokenError("join token payload is not valid JSON");
|
|
1273
|
+
}
|
|
1274
|
+
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) throw new JoinTokenError("join token payload must be a JSON object");
|
|
1275
|
+
const p = parsed;
|
|
1276
|
+
if (typeof p.code !== "string" || p.code.length === 0) throw new JoinTokenError("join token payload is missing a non-empty 'code'");
|
|
1277
|
+
if (typeof p.origin !== "string" || p.origin.length === 0) throw new JoinTokenError("join token payload is missing a non-empty 'origin'");
|
|
1278
|
+
let url;
|
|
1279
|
+
try {
|
|
1280
|
+
url = new URL(p.origin);
|
|
1281
|
+
} catch {
|
|
1282
|
+
throw new JoinTokenError(`join token origin '${p.origin}' is not a valid URL`);
|
|
1283
|
+
}
|
|
1284
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") throw new JoinTokenError(`join token origin '${p.origin}' must be http(s)`);
|
|
1285
|
+
return {
|
|
1286
|
+
origin: url.origin,
|
|
1287
|
+
code: p.code
|
|
1288
|
+
};
|
|
1289
|
+
}
|
|
1290
|
+
|
|
1291
|
+
//#endregion
|
|
1292
|
+
//#region src/cli/federate/enroll.ts
|
|
1293
|
+
/**
|
|
1294
|
+
* `substrate federate enroll` — mint a single-use enrollment token on the callee.
|
|
1295
|
+
*
|
|
1296
|
+
* Transport is direct-to-pg, mirroring `substrate-admin register`: no admin JWT is
|
|
1297
|
+
* needed, and it works on an instance that was never bootstrapped. That is
|
|
1298
|
+
* what dissolves the chicken-and-egg — the operator with shell on the callee
|
|
1299
|
+
* is exactly the operator with authority to admit a peer.
|
|
1300
|
+
*
|
|
1301
|
+
* The `invite_created` audit row is emitted EXPLICITLY. The HTTP route gets it
|
|
1302
|
+
* from an autoCRUD mutation interceptor; going direct-to-pg bypasses the
|
|
1303
|
+
* engine, so nothing emits it implicitly. Same hand-rolled emit `runRegister`
|
|
1304
|
+
* does for `key_registered`.
|
|
1305
|
+
*/
|
|
1306
|
+
/** Actor stamped on audit rows written by the `substrate federate` verbs. */
|
|
1307
|
+
const CLI_ACTOR_KID = "substrate-federate-cli";
|
|
1308
|
+
const defaultProbe = (origin) => probeAuthPosture(origin);
|
|
1309
|
+
async function runEnroll(args, env, openDataset = openPgDataset, probe = defaultProbe) {
|
|
1310
|
+
const posture = await probe(args.origin);
|
|
1311
|
+
if (posture === "disabled") throw new AdminCliError(`${args.origin} reports auth is DISABLED — a credential issued against it would verify nothing. Set SUBSTRATE_AUTH_MODE on the callee, restart it, then retry.`, 7);
|
|
1312
|
+
const probeWarning = posture === "unreachable" ? `could not reach ${args.origin} to confirm auth is enabled — minting anyway. Verify the callee shows \`auth: enabled\` in its boot banner.` : null;
|
|
1313
|
+
const code = randomBytes(32).toString("base64url");
|
|
1314
|
+
const now = /* @__PURE__ */ new Date();
|
|
1315
|
+
const expiresAt = new Date(now.getTime() + args.ttlMs);
|
|
1316
|
+
const { dataset, close } = await openDataset(env);
|
|
1317
|
+
try {
|
|
1318
|
+
await dataset.withTransaction(async (tx) => {
|
|
1319
|
+
await tx.write(INVITE_TYPE, [{
|
|
1320
|
+
id: code,
|
|
1321
|
+
namespaces: encodeList(args.namespaces),
|
|
1322
|
+
capabilities: encodeList(args.capabilities),
|
|
1323
|
+
expiresAt: expiresAt.toISOString(),
|
|
1324
|
+
consumedAt: null,
|
|
1325
|
+
consumedBy: null
|
|
1326
|
+
}], {
|
|
1327
|
+
identity: {
|
|
1328
|
+
kind: "system",
|
|
1329
|
+
id: CLI_ACTOR_KID
|
|
1330
|
+
},
|
|
1331
|
+
now
|
|
1332
|
+
});
|
|
1333
|
+
await tx.write(AUDIT_EVENT_TYPE, [buildAuditRow({
|
|
1334
|
+
actorKid: CLI_ACTOR_KID,
|
|
1335
|
+
kind: "invite_created",
|
|
1336
|
+
targetKid: code,
|
|
1337
|
+
details: {
|
|
1338
|
+
origin: args.origin,
|
|
1339
|
+
namespaces: args.namespaces.join(","),
|
|
1340
|
+
capabilities: args.capabilities.join(","),
|
|
1341
|
+
expiresAt: expiresAt.toISOString()
|
|
1342
|
+
},
|
|
1343
|
+
now
|
|
1344
|
+
})], {
|
|
1345
|
+
identity: {
|
|
1346
|
+
kind: "system",
|
|
1347
|
+
id: CLI_ACTOR_KID
|
|
1348
|
+
},
|
|
1349
|
+
now
|
|
1350
|
+
});
|
|
1351
|
+
});
|
|
1352
|
+
} finally {
|
|
1353
|
+
await close();
|
|
1354
|
+
}
|
|
1355
|
+
return {
|
|
1356
|
+
token: encodeJoinToken({
|
|
1357
|
+
origin: args.origin,
|
|
1358
|
+
code
|
|
1359
|
+
}),
|
|
1360
|
+
origin: args.origin,
|
|
1361
|
+
expiresAt: expiresAt.toISOString(),
|
|
1362
|
+
namespaces: args.namespaces,
|
|
1363
|
+
capabilities: args.capabilities,
|
|
1364
|
+
probeWarning
|
|
1365
|
+
};
|
|
1366
|
+
}
|
|
1367
|
+
function formatEnrollOutput(result) {
|
|
1368
|
+
const lines = [];
|
|
1369
|
+
if (result.probeWarning !== null) lines.push(`warning: ${result.probeWarning}`, "");
|
|
1370
|
+
lines.push("Enrollment token minted.", ` origin: ${result.origin}`, ` namespaces: ${result.namespaces.join(", ")}`, ` capabilities: ${result.capabilities.join(", ")}`, ` expires: ${result.expiresAt}`, "", ` token: ${result.token}`, "", "Run this ON THE CALLER and paste the token at the (silent) prompt.", "Single use — it stops working once redeemed.", "", " read -rs SUBSTRATE_JOIN_TOKEN && SUBSTRATE_JOIN_TOKEN=\"$SUBSTRATE_JOIN_TOKEN\" substrate federate join", "");
|
|
1371
|
+
return lines.join("\n");
|
|
1372
|
+
}
|
|
1373
|
+
|
|
1374
|
+
//#endregion
|
|
1375
|
+
//#region src/cli/federate/env-file.ts
|
|
1376
|
+
/**
|
|
1377
|
+
* Minimal `KEY=VALUE` reader for `substrate federate check --env-file`.
|
|
1378
|
+
*
|
|
1379
|
+
* The bridge config lives in the runtime's `.env` (or a systemd
|
|
1380
|
+
* EnvironmentFile), not in the operator's shell. Without this the operator has
|
|
1381
|
+
* to remember `set -a && source .env && set +a` before every check.
|
|
1382
|
+
*
|
|
1383
|
+
* Deliberately not a full dotenv implementation: no interpolation, no
|
|
1384
|
+
* multi-line values, no `export ` prefix handling. systemd's EnvironmentFile
|
|
1385
|
+
* does not do those either, so matching its subset is the correct target.
|
|
1386
|
+
*/
|
|
1387
|
+
function loadEnvFile(path) {
|
|
1388
|
+
let text;
|
|
1389
|
+
try {
|
|
1390
|
+
text = readFileSync(path, "utf8");
|
|
1391
|
+
} catch (err) {
|
|
1392
|
+
throw new Error(`cannot read env file '${path}': ${err instanceof Error ? err.message : String(err)}`);
|
|
1393
|
+
}
|
|
1394
|
+
const out = {};
|
|
1395
|
+
for (const rawLine of text.split("\n")) {
|
|
1396
|
+
const line = rawLine.trim();
|
|
1397
|
+
if (line.length === 0 || line.startsWith("#")) continue;
|
|
1398
|
+
const eq = line.indexOf("=");
|
|
1399
|
+
if (eq <= 0) continue;
|
|
1400
|
+
const key = line.slice(0, eq).trim();
|
|
1401
|
+
let value = line.slice(eq + 1).trim();
|
|
1402
|
+
if (value.startsWith("'") && value.endsWith("'") && value.length >= 2 || value.startsWith("\"") && value.endsWith("\"") && value.length >= 2) value = value.slice(1, -1);
|
|
1403
|
+
out[key] = value;
|
|
1404
|
+
}
|
|
1405
|
+
return out;
|
|
1406
|
+
}
|
|
1407
|
+
|
|
1408
|
+
//#endregion
|
|
1409
|
+
//#region src/cli/federate/join.ts
|
|
1410
|
+
/**
|
|
1411
|
+
* `substrate federate join` — redeem an enrollment token on the caller.
|
|
1412
|
+
*
|
|
1413
|
+
* The ordering below IS the safety property:
|
|
1414
|
+
*
|
|
1415
|
+
* check key-out clobber → generate keypair in memory → redeem → write key
|
|
1416
|
+
*
|
|
1417
|
+
* Checking the output path after redemption would burn a single-use invite
|
|
1418
|
+
* with nowhere to put the resulting key. And a generated-but-unregistered key
|
|
1419
|
+
* is strictly worse than no key: the tooling then signs with something the
|
|
1420
|
+
* peer rejects, so every read fails auth rather than degrading cleanly. Any
|
|
1421
|
+
* failure here leaves the filesystem untouched.
|
|
1422
|
+
*/
|
|
1423
|
+
function asStringArray(value) {
|
|
1424
|
+
if (!Array.isArray(value)) return [];
|
|
1425
|
+
return value.filter((v) => typeof v === "string");
|
|
1426
|
+
}
|
|
1427
|
+
/** Map the callee's response into a specific operator-facing message. */
|
|
1428
|
+
function registerFailure(status, body, origin) {
|
|
1429
|
+
const code = typeof body === "object" && body !== null ? body.code : void 0;
|
|
1430
|
+
if (code === "invite_not_found") return new AdminCliError(`token not recognised at ${origin} — it was already swept, or it was minted against a different instance`, 3);
|
|
1431
|
+
if (code === "invite_already_consumed") return new AdminCliError("this token has already been redeemed — tokens are single use. Mint a fresh one on the callee with `substrate federate enroll`.", 4);
|
|
1432
|
+
if (code === "invite_expired") return new AdminCliError("this token has expired. Mint a fresh one on the callee with `substrate federate enroll`.", 5);
|
|
1433
|
+
if (code === "rate_limited") {
|
|
1434
|
+
const retryMs = typeof body === "object" && body !== null ? body.retryAfterMs : void 0;
|
|
1435
|
+
const seconds = typeof retryMs === "number" ? Math.max(1, Math.ceil(retryMs / 1e3)) : 60;
|
|
1436
|
+
return new AdminCliError(`${origin} is rate-limiting registrations — retry in ${seconds}s`, 6);
|
|
1437
|
+
}
|
|
1438
|
+
const message = typeof body === "object" && body !== null ? String(body.message ?? "") : "";
|
|
1439
|
+
return new AdminCliError(`${origin} refused the registration (HTTP ${status})${message.length > 0 ? `: ${message}` : ""}`, 3);
|
|
1440
|
+
}
|
|
1441
|
+
async function runJoin(args, deps = {}) {
|
|
1442
|
+
const call = deps.httpJson ?? httpJson;
|
|
1443
|
+
const host = (deps.hostname ?? hostname)();
|
|
1444
|
+
const home = (deps.homeDir ?? homedir)();
|
|
1445
|
+
const payload = decodeJoinToken(args.token);
|
|
1446
|
+
const name = args.name ?? new URL(payload.origin).hostname;
|
|
1447
|
+
const keyPath = args.keyOut ?? join(home, ".config/substrate/keys", `${name}.key`);
|
|
1448
|
+
if (existsSync(keyPath)) throw new AdminCliError(`key path '${keyPath}' already exists — move it aside and retry, or pass --key-out. Nothing has been redeemed; your token is still valid.`, 2);
|
|
1449
|
+
const { privateKey, publicKey } = generateKeyPairSync("ed25519", {
|
|
1450
|
+
publicKeyEncoding: {
|
|
1451
|
+
format: "pem",
|
|
1452
|
+
type: "spki"
|
|
1453
|
+
},
|
|
1454
|
+
privateKeyEncoding: {
|
|
1455
|
+
format: "pem",
|
|
1456
|
+
type: "pkcs8"
|
|
1457
|
+
}
|
|
1458
|
+
});
|
|
1459
|
+
let res;
|
|
1460
|
+
try {
|
|
1461
|
+
res = await call(`${payload.origin}/substrate/v1/_system/register`, {
|
|
1462
|
+
method: "POST",
|
|
1463
|
+
body: {
|
|
1464
|
+
code: payload.code,
|
|
1465
|
+
publicKey: publicKey.toString(),
|
|
1466
|
+
keyName: `bridge from ${host}`,
|
|
1467
|
+
algorithm: "EdDSA"
|
|
1468
|
+
}
|
|
1469
|
+
});
|
|
1470
|
+
} catch (err) {
|
|
1471
|
+
throw new AdminCliError(`cannot reach ${payload.origin}: ${err instanceof Error ? err.message : String(err)}. That origin came from the token itself, so check it is the URL this machine can reach — a wrong origin at mint time looks identical to a peer being down.`, 3);
|
|
1472
|
+
}
|
|
1473
|
+
if (res.status !== 201) throw registerFailure(res.status, res.body, payload.origin);
|
|
1474
|
+
const body = res.body ?? {};
|
|
1475
|
+
const kid = typeof body.kid === "string" ? body.kid : "";
|
|
1476
|
+
if (kid.length === 0) throw new AdminCliError(`${payload.origin} accepted the registration but returned no kid — nothing was written locally`, 3);
|
|
1477
|
+
try {
|
|
1478
|
+
mkdirSync(dirname(keyPath), {
|
|
1479
|
+
recursive: true,
|
|
1480
|
+
mode: 448
|
|
1481
|
+
});
|
|
1482
|
+
writeFileSync(keyPath, privateKey.toString(), { mode: 384 });
|
|
1483
|
+
} catch (err) {
|
|
1484
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1485
|
+
throw new AdminCliError(`redeemed at ${payload.origin} (kid ${kid}) but failed to write the private key to '${keyPath}': ${msg}. The invite token IS consumed — it cannot be redeemed again. Mint a fresh one on the callee with \`substrate federate enroll\`, then retry \`join\` with a path this process can actually create and write.`, 8);
|
|
1486
|
+
}
|
|
1487
|
+
return {
|
|
1488
|
+
origin: payload.origin,
|
|
1489
|
+
name,
|
|
1490
|
+
kid,
|
|
1491
|
+
namespaces: asStringArray(body.namespaces),
|
|
1492
|
+
capabilities: asStringArray(body.capabilities),
|
|
1493
|
+
keyPath
|
|
1494
|
+
};
|
|
1495
|
+
}
|
|
1496
|
+
function formatJoinOutput(result) {
|
|
1497
|
+
const bridges = result.namespaces.map((ns) => ({
|
|
1498
|
+
ns,
|
|
1499
|
+
name: result.name,
|
|
1500
|
+
url: result.origin
|
|
1501
|
+
}));
|
|
1502
|
+
const creds = { [result.name]: {
|
|
1503
|
+
keyPath: result.keyPath,
|
|
1504
|
+
kid: result.kid,
|
|
1505
|
+
origin: result.origin
|
|
1506
|
+
} };
|
|
1507
|
+
return [
|
|
1508
|
+
"",
|
|
1509
|
+
` redeemed at ${result.origin}`,
|
|
1510
|
+
` kid: ${result.kid}`,
|
|
1511
|
+
` namespaces: ${result.namespaces.join(", ")} capabilities: ${result.capabilities.join(", ")}`,
|
|
1512
|
+
` key: ${result.keyPath} (0600)`,
|
|
1513
|
+
"",
|
|
1514
|
+
"Add to your .env, then restart:",
|
|
1515
|
+
"",
|
|
1516
|
+
`SUBSTRATE_BRIDGES='${JSON.stringify(bridges)}'`,
|
|
1517
|
+
`SUBSTRATE_BRIDGE_CREDENTIALS='${JSON.stringify(creds)}'`,
|
|
1518
|
+
"",
|
|
1519
|
+
`Then verify with: substrate federate check ${result.name}`,
|
|
1520
|
+
""
|
|
1521
|
+
].join("\n");
|
|
1522
|
+
}
|
|
1523
|
+
|
|
1524
|
+
//#endregion
|
|
1525
|
+
//#region src/cli/federate/main.ts
|
|
1526
|
+
/**
|
|
1527
|
+
* Mirrors `packages/substrate-compose/src/compose.ts`'s `Capability` union — the
|
|
1528
|
+
* only spellings any route's `gate()` call ever recognises. Anything else
|
|
1529
|
+
* mints a credential that authenticates but never satisfies a capability
|
|
1530
|
+
* check, diagnosable only via `substrate federate check` after the fact.
|
|
1531
|
+
*/
|
|
1532
|
+
const KNOWN_CAPABILITIES = [
|
|
1533
|
+
"read",
|
|
1534
|
+
"mutate",
|
|
1535
|
+
"admin"
|
|
1536
|
+
];
|
|
1537
|
+
const FEDERATE_USAGE = [
|
|
1538
|
+
"Usage: substrate federate <verb> [args...]",
|
|
1539
|
+
"",
|
|
1540
|
+
"Verbs:",
|
|
1541
|
+
" enroll --origin <url> --ns <csv> [--caps <csv>] [--ttl 15m] [--allow-admin]",
|
|
1542
|
+
" ON THE CALLEE. Mints a single-use enrollment token. Needs SUBSTRATE_DATABASE_URL.",
|
|
1543
|
+
" --caps admin requires --allow-admin — the credential leaves this machine.",
|
|
1544
|
+
"",
|
|
1545
|
+
" join [--name <n>] [--key-out <path>]",
|
|
1546
|
+
" ON THE CALLER. Redeems the token in SUBSTRATE_JOIN_TOKEN and prints the config to paste.",
|
|
1547
|
+
" A <token> positional still works but is visible in ps and shell history.",
|
|
1548
|
+
"",
|
|
1549
|
+
" check [<name>] [--env-file <path>]",
|
|
1550
|
+
" ON THE CALLER. Diagnoses a configured bridge end to end.",
|
|
1551
|
+
"",
|
|
1552
|
+
" TLS is always verified. For a private CA, set NODE_EXTRA_CA_CERTS."
|
|
1553
|
+
].join("\n");
|
|
1554
|
+
function splitCsv(csv) {
|
|
1555
|
+
return csv.split(",").map((s) => s.trim()).filter((s) => s.length > 0);
|
|
1556
|
+
}
|
|
1557
|
+
/** Shared single-value flag reader — rejects a missing value or a following flag. */
|
|
1558
|
+
function takeValue(rest, index, verb, flag) {
|
|
1559
|
+
const value = rest[index + 1];
|
|
1560
|
+
if (value === void 0 || value.length === 0 || value.startsWith("--")) throw new AdminCliError(`${verb}: ${flag} requires a value`, 1);
|
|
1561
|
+
return value;
|
|
1562
|
+
}
|
|
1563
|
+
function parseEnrollArgs(argv) {
|
|
1564
|
+
const rest = argv.slice(1);
|
|
1565
|
+
let origin;
|
|
1566
|
+
let nsCsv;
|
|
1567
|
+
let capsCsv;
|
|
1568
|
+
let ttlRaw;
|
|
1569
|
+
let allowAdmin = false;
|
|
1570
|
+
for (let i = 0; i < rest.length; i++) {
|
|
1571
|
+
const tok = rest[i];
|
|
1572
|
+
if (tok === void 0) continue;
|
|
1573
|
+
if (tok === "--origin") {
|
|
1574
|
+
if (origin !== void 0) throw new AdminCliError("enroll: --origin specified more than once", 1);
|
|
1575
|
+
origin = takeValue(rest, i, "enroll", "--origin");
|
|
1576
|
+
i += 1;
|
|
1577
|
+
} else if (tok === "--ns") {
|
|
1578
|
+
if (nsCsv !== void 0) throw new AdminCliError("enroll: --ns specified more than once", 1);
|
|
1579
|
+
nsCsv = takeValue(rest, i, "enroll", "--ns");
|
|
1580
|
+
i += 1;
|
|
1581
|
+
} else if (tok === "--caps") {
|
|
1582
|
+
if (capsCsv !== void 0) throw new AdminCliError("enroll: --caps specified more than once", 1);
|
|
1583
|
+
capsCsv = takeValue(rest, i, "enroll", "--caps");
|
|
1584
|
+
i += 1;
|
|
1585
|
+
} else if (tok === "--ttl") {
|
|
1586
|
+
if (ttlRaw !== void 0) throw new AdminCliError("enroll: --ttl specified more than once", 1);
|
|
1587
|
+
ttlRaw = takeValue(rest, i, "enroll", "--ttl");
|
|
1588
|
+
i += 1;
|
|
1589
|
+
} else if (tok === "--allow-admin") {
|
|
1590
|
+
if (allowAdmin) throw new AdminCliError("enroll: --allow-admin specified more than once", 1);
|
|
1591
|
+
allowAdmin = true;
|
|
1592
|
+
} else if (tok.startsWith("--")) throw new AdminCliError(`enroll: unknown flag ${tok}`, 1);
|
|
1593
|
+
else throw new AdminCliError(`enroll: unexpected positional arg '${tok}'`, 1);
|
|
1594
|
+
}
|
|
1595
|
+
if (origin === void 0) throw new AdminCliError("enroll requires --origin <url> (the callee base URL)", 1);
|
|
1596
|
+
let normalisedOrigin;
|
|
1597
|
+
try {
|
|
1598
|
+
const url = new URL(origin);
|
|
1599
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") throw new AdminCliError(`enroll: --origin '${origin}' must be http(s)`, 1);
|
|
1600
|
+
normalisedOrigin = url.origin;
|
|
1601
|
+
} catch (err) {
|
|
1602
|
+
if (err instanceof AdminCliError) throw err;
|
|
1603
|
+
throw new AdminCliError(`enroll: --origin '${origin}' is not a valid URL`, 1);
|
|
1604
|
+
}
|
|
1605
|
+
if (nsCsv === void 0) throw new AdminCliError("enroll requires --ns <csv> naming the namespaces this peer may query", 1);
|
|
1606
|
+
const namespaces = splitCsv(nsCsv);
|
|
1607
|
+
if (namespaces.length === 0) throw new AdminCliError("enroll: --ns must name at least one namespace", 1);
|
|
1608
|
+
if (namespaces.includes("*")) throw new AdminCliError("enroll: --ns does not accept the wildcard '*' — name the namespaces explicitly. A wildcard grant cannot produce a usable SUBSTRATE_BRIDGES entry, because there is no local alias for `*`, and a federation credential should never get wildcard access by default.", 1);
|
|
1609
|
+
const capabilities = capsCsv === void 0 ? ["read"] : splitCsv(capsCsv);
|
|
1610
|
+
if (capabilities.length === 0) throw new AdminCliError("enroll: --caps must name at least one capability", 1);
|
|
1611
|
+
const unknownCaps = capabilities.filter((c) => !KNOWN_CAPABILITIES.includes(c));
|
|
1612
|
+
if (unknownCaps.length > 0) throw new AdminCliError(`enroll: unknown --caps value '${unknownCaps.join("', '")}' — valid values are ${KNOWN_CAPABILITIES.join(", ")}. A misspelled capability mints a credential that authenticates but satisfies no capability check — diagnosable only after the fact via \`substrate federate check\`.`, 1);
|
|
1613
|
+
if (capabilities.includes("admin") && !allowAdmin) throw new AdminCliError("enroll: --caps admin requires --allow-admin — this credential is redeemable remotely by whoever holds the token, so minting an admin-scoped one needs to be deliberate, not a `--caps` typo away from `mutate`.", 1);
|
|
1614
|
+
let ttlMs;
|
|
1615
|
+
try {
|
|
1616
|
+
ttlMs = parseDuration(ttlRaw ?? "15m");
|
|
1617
|
+
} catch (err) {
|
|
1618
|
+
throw new AdminCliError(`enroll: ${err instanceof Error ? err.message : String(err)}`, 1);
|
|
1619
|
+
}
|
|
1620
|
+
return {
|
|
1621
|
+
origin: normalisedOrigin,
|
|
1622
|
+
namespaces,
|
|
1623
|
+
capabilities,
|
|
1624
|
+
ttlMs
|
|
1625
|
+
};
|
|
1626
|
+
}
|
|
1627
|
+
/**
|
|
1628
|
+
* The token comes from `SUBSTRATE_JOIN_TOKEN`, or from a positional, which the
|
|
1629
|
+
* caller warns about: argv is readable by every user on the box via `ps`.
|
|
1630
|
+
*/
|
|
1631
|
+
function parseJoinArgs(argv, env = {}) {
|
|
1632
|
+
const rest = argv.slice(1);
|
|
1633
|
+
let token;
|
|
1634
|
+
let name;
|
|
1635
|
+
let keyOut;
|
|
1636
|
+
for (let i = 0; i < rest.length; i++) {
|
|
1637
|
+
const tok = rest[i];
|
|
1638
|
+
if (tok === void 0) continue;
|
|
1639
|
+
if (tok === "--name") {
|
|
1640
|
+
if (name !== void 0) throw new AdminCliError("join: --name specified more than once", 1);
|
|
1641
|
+
name = takeValue(rest, i, "join", "--name");
|
|
1642
|
+
i += 1;
|
|
1643
|
+
} else if (tok === "--key-out") {
|
|
1644
|
+
if (keyOut !== void 0) throw new AdminCliError("join: --key-out specified more than once", 1);
|
|
1645
|
+
keyOut = takeValue(rest, i, "join", "--key-out");
|
|
1646
|
+
i += 1;
|
|
1647
|
+
} else if (tok.startsWith("--")) throw new AdminCliError(`join: unknown flag ${tok}`, 1);
|
|
1648
|
+
else {
|
|
1649
|
+
if (token !== void 0) throw new AdminCliError(`join: unexpected positional arg '${tok}'`, 1);
|
|
1650
|
+
token = tok;
|
|
1651
|
+
}
|
|
1652
|
+
}
|
|
1653
|
+
const envToken = env.SUBSTRATE_JOIN_TOKEN;
|
|
1654
|
+
if (envToken !== void 0 && envToken.length > 0) {
|
|
1655
|
+
if (token !== void 0) throw new AdminCliError("join: token given both in SUBSTRATE_JOIN_TOKEN and on argv", 1);
|
|
1656
|
+
token = envToken;
|
|
1657
|
+
}
|
|
1658
|
+
if (token === void 0 || token.length === 0) throw new AdminCliError("join requires a token: set SUBSTRATE_JOIN_TOKEN", 1);
|
|
1659
|
+
return {
|
|
1660
|
+
token,
|
|
1661
|
+
name,
|
|
1662
|
+
keyOut
|
|
1663
|
+
};
|
|
1664
|
+
}
|
|
1665
|
+
function parseCheckArgs(argv) {
|
|
1666
|
+
const rest = argv.slice(1);
|
|
1667
|
+
let name;
|
|
1668
|
+
let envFile;
|
|
1669
|
+
for (let i = 0; i < rest.length; i++) {
|
|
1670
|
+
const tok = rest[i];
|
|
1671
|
+
if (tok === void 0) continue;
|
|
1672
|
+
if (tok === "--env-file") {
|
|
1673
|
+
if (envFile !== void 0) throw new AdminCliError("check: --env-file specified more than once", 1);
|
|
1674
|
+
envFile = takeValue(rest, i, "check", "--env-file");
|
|
1675
|
+
i += 1;
|
|
1676
|
+
} else if (tok.startsWith("--")) throw new AdminCliError(`check: unknown flag ${tok}`, 1);
|
|
1677
|
+
else {
|
|
1678
|
+
if (name !== void 0) throw new AdminCliError(`check: unexpected positional arg '${tok}'`, 1);
|
|
1679
|
+
name = tok;
|
|
1680
|
+
}
|
|
1681
|
+
}
|
|
1682
|
+
return {
|
|
1683
|
+
name,
|
|
1684
|
+
envFile
|
|
1685
|
+
};
|
|
1686
|
+
}
|
|
1687
|
+
async function runFederate(argv, env, out, errOut) {
|
|
1688
|
+
const verb = argv[0];
|
|
1689
|
+
if (verb === void 0 || verb === "--help" || verb === "-h") {
|
|
1690
|
+
(verb === void 0 ? errOut : out)(`${FEDERATE_USAGE}\n`);
|
|
1691
|
+
return verb === void 0 ? 1 : 0;
|
|
1692
|
+
}
|
|
1693
|
+
try {
|
|
1694
|
+
switch (verb) {
|
|
1695
|
+
case "enroll": {
|
|
1696
|
+
const args = parseEnrollArgs(argv);
|
|
1697
|
+
const result = await runEnroll(args, env);
|
|
1698
|
+
out(formatEnrollOutput(result));
|
|
1699
|
+
return 0;
|
|
1700
|
+
}
|
|
1701
|
+
case "join": {
|
|
1702
|
+
const args = parseJoinArgs(argv, env);
|
|
1703
|
+
if (env.SUBSTRATE_JOIN_TOKEN === void 0 || env.SUBSTRATE_JOIN_TOKEN.length === 0) errOut("warning: a token on the command line is visible in ps and shell history; pass it in SUBSTRATE_JOIN_TOKEN instead\n");
|
|
1704
|
+
const result = await runJoin(args);
|
|
1705
|
+
out(formatJoinOutput(result));
|
|
1706
|
+
return 0;
|
|
1707
|
+
}
|
|
1708
|
+
case "check": {
|
|
1709
|
+
const args = parseCheckArgs(argv);
|
|
1710
|
+
const effectiveEnv = args.envFile === void 0 ? env : {
|
|
1711
|
+
...env,
|
|
1712
|
+
...loadEnvFile(args.envFile)
|
|
1713
|
+
};
|
|
1714
|
+
const results = await runCheck(args.name, effectiveEnv);
|
|
1715
|
+
out(formatCheckOutput(results));
|
|
1716
|
+
return results.every((r) => r.ok) ? 0 : 1;
|
|
1717
|
+
}
|
|
1718
|
+
default:
|
|
1719
|
+
errOut(`substrate federate: unknown verb '${verb}'\n\n${FEDERATE_USAGE}\n`);
|
|
1720
|
+
return 1;
|
|
1721
|
+
}
|
|
1722
|
+
} catch (err) {
|
|
1723
|
+
const exitCode = err instanceof AdminCliError || err instanceof JoinTokenError ? err.exitCode : 1;
|
|
1724
|
+
errOut(`${err instanceof Error ? err.message : String(err)}\n`);
|
|
1725
|
+
return exitCode;
|
|
1726
|
+
}
|
|
1727
|
+
}
|
|
1728
|
+
|
|
1729
|
+
//#endregion
|
|
1730
|
+
//#region src/cli/init/sizing.ts
|
|
1731
|
+
/** What an unanswered question assumes: a small personal store. */
|
|
1732
|
+
const DEFAULT_SIZING = {
|
|
1733
|
+
rowsPerDay: 1e3,
|
|
1734
|
+
retentionDays: 365,
|
|
1735
|
+
readTargetMs: 250,
|
|
1736
|
+
writeHeavy: false
|
|
1737
|
+
};
|
|
1738
|
+
/** The profile for the expected row count; `writeHeavy` never changes the name. */
|
|
1739
|
+
function profileFromAnswers(a) {
|
|
1740
|
+
const rows = a.rowsPerDay * a.retentionDays;
|
|
1741
|
+
if (rows >= 5e7 || a.readTargetMs <= 50 && rows >= 5e6) return "scale";
|
|
1742
|
+
if (rows >= 1e6) return "growth";
|
|
1743
|
+
return "personal";
|
|
1744
|
+
}
|
|
1745
|
+
/** A sizing flag or answer: a positive finite number. */
|
|
1746
|
+
function parsePositive(flag, value) {
|
|
1747
|
+
const n = Number(value);
|
|
1748
|
+
if (value.trim() === "" || !Number.isFinite(n) || n <= 0) throw new Error(`--${flag} must be a positive number, got: ${value}`);
|
|
1749
|
+
return n;
|
|
1750
|
+
}
|
|
1751
|
+
|
|
1752
|
+
//#endregion
|
|
1753
|
+
//#region src/cli/init/config.ts
|
|
1754
|
+
const EMBEDDER_MODES = [
|
|
1755
|
+
"bge-base",
|
|
1756
|
+
"minilm",
|
|
1757
|
+
"ollama",
|
|
1758
|
+
"none"
|
|
1759
|
+
];
|
|
1760
|
+
const KNOWN_FLAGS = /* @__PURE__ */ new Set([
|
|
1761
|
+
"yes",
|
|
1762
|
+
"force",
|
|
1763
|
+
"plugins",
|
|
1764
|
+
"store",
|
|
1765
|
+
"port",
|
|
1766
|
+
"pg-port",
|
|
1767
|
+
"embedder",
|
|
1768
|
+
"substrate-url",
|
|
1769
|
+
"tls",
|
|
1770
|
+
"tls-cert-path",
|
|
1771
|
+
"tls-key-path",
|
|
1772
|
+
"rows-per-day",
|
|
1773
|
+
"retention-days",
|
|
1774
|
+
"read-target-ms",
|
|
1775
|
+
"write-heavy",
|
|
1776
|
+
"profile"
|
|
1777
|
+
]);
|
|
1778
|
+
function parsePort(flag, value) {
|
|
1779
|
+
const n = Number(value);
|
|
1780
|
+
if (!/^\d+$/.test(value) || n < 1 || n > 65535) throw new Error(`--${flag} must be a positive integer port (1-65535), got: ${value}`);
|
|
1781
|
+
return n;
|
|
1782
|
+
}
|
|
1783
|
+
function parseArgs(argv) {
|
|
1784
|
+
const config = {
|
|
1785
|
+
targetDir: void 0,
|
|
1786
|
+
store: void 0,
|
|
1787
|
+
port: void 0,
|
|
1788
|
+
pgPort: void 0,
|
|
1789
|
+
plugins: void 0,
|
|
1790
|
+
embedder: void 0,
|
|
1791
|
+
substrateUrl: void 0,
|
|
1792
|
+
tls: false,
|
|
1793
|
+
tlsCertPath: void 0,
|
|
1794
|
+
tlsKeyPath: void 0,
|
|
1795
|
+
rowsPerDay: void 0,
|
|
1796
|
+
retentionDays: void 0,
|
|
1797
|
+
readTargetMs: void 0,
|
|
1798
|
+
writeHeavy: false,
|
|
1799
|
+
profile: void 0,
|
|
1800
|
+
yes: false,
|
|
1801
|
+
force: false
|
|
1802
|
+
};
|
|
1803
|
+
for (const arg of argv) {
|
|
1804
|
+
if (!arg.startsWith("--")) {
|
|
1805
|
+
if (config.targetDir === void 0) config.targetDir = arg;
|
|
1806
|
+
continue;
|
|
1807
|
+
}
|
|
1808
|
+
const eqIndex = arg.indexOf("=");
|
|
1809
|
+
const flag = eqIndex === -1 ? arg.slice(2) : arg.slice(2, eqIndex);
|
|
1810
|
+
const value = eqIndex === -1 ? "" : arg.slice(eqIndex + 1);
|
|
1811
|
+
if (!KNOWN_FLAGS.has(flag)) throw new Error(`unknown flag --${flag}`);
|
|
1812
|
+
switch (flag) {
|
|
1813
|
+
case "yes":
|
|
1814
|
+
config.yes = true;
|
|
1815
|
+
break;
|
|
1816
|
+
case "force":
|
|
1817
|
+
config.force = true;
|
|
1818
|
+
break;
|
|
1819
|
+
case "plugins":
|
|
1820
|
+
config.plugins = value.split(",").map((p) => p.trim()).filter(Boolean);
|
|
1821
|
+
break;
|
|
1822
|
+
case "store":
|
|
1823
|
+
if (value !== "memory" && value !== "postgres") throw new Error(`--store must be memory|postgres, got: ${value}`);
|
|
1824
|
+
config.store = value;
|
|
1825
|
+
break;
|
|
1826
|
+
case "port":
|
|
1827
|
+
config.port = parsePort("port", value);
|
|
1828
|
+
break;
|
|
1829
|
+
case "pg-port":
|
|
1830
|
+
config.pgPort = parsePort("pg-port", value);
|
|
1831
|
+
break;
|
|
1832
|
+
case "tls":
|
|
1833
|
+
config.tls = true;
|
|
1834
|
+
break;
|
|
1835
|
+
case "tls-cert-path":
|
|
1836
|
+
config.tlsCertPath = value;
|
|
1837
|
+
break;
|
|
1838
|
+
case "tls-key-path":
|
|
1839
|
+
config.tlsKeyPath = value;
|
|
1840
|
+
break;
|
|
1841
|
+
case "embedder":
|
|
1842
|
+
if (!isEmbedderMode(value)) throw new Error(`--embedder must be ${EMBEDDER_MODES.join("|")}, got: ${value}`);
|
|
1843
|
+
config.embedder = value;
|
|
1844
|
+
break;
|
|
1845
|
+
case "rows-per-day":
|
|
1846
|
+
config.rowsPerDay = parsePositive(flag, value);
|
|
1847
|
+
break;
|
|
1848
|
+
case "retention-days":
|
|
1849
|
+
config.retentionDays = parsePositive(flag, value);
|
|
1850
|
+
break;
|
|
1851
|
+
case "read-target-ms":
|
|
1852
|
+
config.readTargetMs = parsePositive(flag, value);
|
|
1853
|
+
break;
|
|
1854
|
+
case "write-heavy":
|
|
1855
|
+
if (value !== "" && value !== "true" && value !== "false") throw new Error(`--write-heavy must be true|false, got: ${value}`);
|
|
1856
|
+
config.writeHeavy = value !== "false";
|
|
1857
|
+
break;
|
|
1858
|
+
case "profile":
|
|
1859
|
+
if (!isStoreProfileName(value)) throw new Error(`--profile must be personal|growth|scale, got: ${value}`);
|
|
1860
|
+
config.profile = value;
|
|
1861
|
+
break;
|
|
1862
|
+
case "substrate-url": config.substrateUrl = value;
|
|
1863
|
+
}
|
|
1864
|
+
}
|
|
1865
|
+
if (config.tls && (config.tlsCertPath === void 0 || config.tlsKeyPath === void 0)) throw new Error("--tls requires both --tls-cert-path and --tls-key-path");
|
|
1866
|
+
return config;
|
|
1867
|
+
}
|
|
1868
|
+
function isEmbedderMode(value) {
|
|
1869
|
+
return EMBEDDER_MODES.includes(value);
|
|
1870
|
+
}
|
|
1871
|
+
|
|
1872
|
+
//#endregion
|
|
1873
|
+
//#region src/cli/init/plugins.ts
|
|
1874
|
+
/**
|
|
1875
|
+
* The cortext line publishes on its own versions and declares substrate as a
|
|
1876
|
+
* peer range. Raise the floor when a cortext release drops support for the
|
|
1877
|
+
* substrate major this binary is on. The floor names a prerelease because a
|
|
1878
|
+
* caret range admits no prerelease of a version it does not itself name.
|
|
1879
|
+
*/
|
|
1880
|
+
const CORTEXT_RANGE = "^1.0.0-alpha.0";
|
|
1881
|
+
const KNOWN_PLUGINS = [
|
|
1882
|
+
{
|
|
1883
|
+
name: "cortext",
|
|
1884
|
+
npmName: "@mnmal-ai/cortext",
|
|
1885
|
+
range: CORTEXT_RANGE,
|
|
1886
|
+
description: "Agent coordination context store (Todos, Decisions, SessionLogs, AgentRules, Memory)."
|
|
1887
|
+
},
|
|
1888
|
+
{
|
|
1889
|
+
name: "substrate-nl",
|
|
1890
|
+
npmName: "@mnmal-ai/substrate-nl",
|
|
1891
|
+
description: "Natural-language questions answered as frames."
|
|
1892
|
+
},
|
|
1893
|
+
{
|
|
1894
|
+
name: "substrate-sentinel",
|
|
1895
|
+
npmName: "@mnmal-ai/substrate-sentinel",
|
|
1896
|
+
description: "Coordinator registry for dynamic bridge discovery."
|
|
1897
|
+
}
|
|
1898
|
+
];
|
|
1899
|
+
function defaultPluginsForPersona(persona) {
|
|
1900
|
+
switch (persona) {
|
|
1901
|
+
case "agent-context": return ["cortext"];
|
|
1902
|
+
case "minimal": return [];
|
|
1903
|
+
case "full": return KNOWN_PLUGINS.map((p) => p.name);
|
|
1904
|
+
}
|
|
1905
|
+
}
|
|
1906
|
+
function pluginNpmName(name) {
|
|
1907
|
+
const known = KNOWN_PLUGINS.find((p) => p.name === name);
|
|
1908
|
+
if (known !== void 0) return known.npmName;
|
|
1909
|
+
return name.startsWith("@") ? name : `@mnmal-ai/${name}`;
|
|
1910
|
+
}
|
|
1911
|
+
function pluginRange(name, substrateRange) {
|
|
1912
|
+
return KNOWN_PLUGINS.find((p) => p.name === name)?.range ?? substrateRange;
|
|
1913
|
+
}
|
|
1914
|
+
|
|
1915
|
+
//#endregion
|
|
1916
|
+
//#region src/cli/init/templates.ts
|
|
1917
|
+
function envBinaryTemplate(ctx) {
|
|
1918
|
+
const lines = [
|
|
1919
|
+
"# Substrate runtime — environment config.",
|
|
1920
|
+
"# Generated by `substrate init`. Edit values, then `substrate serve` (or wrap in systemd).",
|
|
1921
|
+
""
|
|
1922
|
+
];
|
|
1923
|
+
if (ctx.store === "postgres") lines.push(`SUBSTRATE_DATABASE_URL=postgres://substrate:${ctx.pgPassword}@127.0.0.1:${ctx.pgPort}/${ctx.dbName}`, "# Set to the database volume mount path when the database runs on this host, to check free disk before index builds.", "SUBSTRATE_INDEX_DISK_PATH=off");
|
|
1924
|
+
else lines.push("SUBSTRATE_DATABASE_URL=memory");
|
|
1925
|
+
lines.push("");
|
|
1926
|
+
lines.push("# Sizing profile: how sensitive the index advisor is. Seeds the store once; change it live afterwards.");
|
|
1927
|
+
lines.push(`SUBSTRATE_STORE_PROFILE=${ctx.storeProfile}`);
|
|
1928
|
+
if (ctx.writeHeavy) {
|
|
1929
|
+
lines.push("# Write-heavy store: the advisor flags unused indexes at half the write rate.");
|
|
1930
|
+
lines.push("SUBSTRATE_STORE_WRITE_HEAVY=1");
|
|
1931
|
+
}
|
|
1932
|
+
lines.push("");
|
|
1933
|
+
lines.push("# Plugin install root — manifest-driven loader scans node_modules under this dir.");
|
|
1934
|
+
lines.push("SUBSTRATE_PLUGIN_DIR=.");
|
|
1935
|
+
lines.push("");
|
|
1936
|
+
lines.push(`PORT=${ctx.port}`);
|
|
1937
|
+
lines.push("");
|
|
1938
|
+
lines.push("# Who this deployment writes as while auth is off. Every row it writes");
|
|
1939
|
+
lines.push("# carries this in createdBy. Turn auth on and callers are attributed");
|
|
1940
|
+
lines.push("# individually instead, at which point this is ignored.");
|
|
1941
|
+
lines.push("SUBSTRATE_SYSTEM_ACTOR=local-dev");
|
|
1942
|
+
lines.push("");
|
|
1943
|
+
if (ctx.tls) {
|
|
1944
|
+
lines.push("# TLS termination — paths to existing cert + key on disk.");
|
|
1945
|
+
lines.push(`SUBSTRATE_TLS_CERT_PATH=${ctx.tlsCertPath ?? ""}`);
|
|
1946
|
+
lines.push(`SUBSTRATE_TLS_KEY_PATH=${ctx.tlsKeyPath ?? ""}`);
|
|
1947
|
+
lines.push("");
|
|
1948
|
+
}
|
|
1949
|
+
while (lines.length > 0 && lines[lines.length - 1] === "") lines.pop();
|
|
1950
|
+
return `${lines.join("\n")}\n`;
|
|
1951
|
+
}
|
|
1952
|
+
/**
|
|
1953
|
+
* The in-process models, run by fastembed. Each name is a key of
|
|
1954
|
+
* `OnnxEmbedder`'s model map, and the dimension is the model's own.
|
|
1955
|
+
*/
|
|
1956
|
+
const ONNX_MODELS = {
|
|
1957
|
+
minilm: {
|
|
1958
|
+
model: "all-MiniLM-L6-v2",
|
|
1959
|
+
dimension: 384
|
|
1960
|
+
},
|
|
1961
|
+
"bge-base": {
|
|
1962
|
+
model: "bge-base-en-v1.5",
|
|
1963
|
+
dimension: 768
|
|
1964
|
+
}
|
|
1965
|
+
};
|
|
1966
|
+
/** The onnx embedder's backend. Undeclared by substrate-embedder, so the runtime installs it. */
|
|
1967
|
+
const FASTEMBED_RANGE = "^2.1.1";
|
|
1968
|
+
/**
|
|
1969
|
+
* `<runtime>/substrate-recall.config.json`, or null for `--embedder=none`. The
|
|
1970
|
+
* recall plugin reads its embedder from this file (merged by the manifest
|
|
1971
|
+
* loader), not from env.
|
|
1972
|
+
*
|
|
1973
|
+
* `cacheDir` is absolute because fastembed's own default is `local_cache`
|
|
1974
|
+
* under the working directory, which moves with whoever starts the server.
|
|
1975
|
+
*/
|
|
1976
|
+
function recallConfigTemplate(embedder, runtimeDir) {
|
|
1977
|
+
if (embedder === "none") return null;
|
|
1978
|
+
const config = embedder === "ollama" ? { embedder: {
|
|
1979
|
+
kind: "ollama",
|
|
1980
|
+
baseUrl: "http://localhost:11434",
|
|
1981
|
+
model: "hf.co/CompendiumLabs/bge-base-en-v1.5-gguf:F16",
|
|
1982
|
+
dimension: 768
|
|
1983
|
+
} } : { embedder: {
|
|
1984
|
+
kind: "onnx",
|
|
1985
|
+
...ONNX_MODELS[embedder],
|
|
1986
|
+
cacheDir: join(runtimeDir, "models")
|
|
1987
|
+
} };
|
|
1988
|
+
return `${JSON.stringify(config, null, 2)}\n`;
|
|
1989
|
+
}
|
|
1990
|
+
function envDockerTemplate(ctx) {
|
|
1991
|
+
if (ctx.store === "memory") return "";
|
|
1992
|
+
return [
|
|
1993
|
+
"# Postgres compose config — consumed by docker-compose.yml.",
|
|
1994
|
+
`POSTGRES_PASSWORD=${ctx.pgPassword}`,
|
|
1995
|
+
"POSTGRES_USER=substrate",
|
|
1996
|
+
`SUBSTRATE_DATABASE_NAME=${ctx.dbName}`,
|
|
1997
|
+
""
|
|
1998
|
+
].join("\n");
|
|
1999
|
+
}
|
|
2000
|
+
function dockerComposeTemplate(ctx) {
|
|
2001
|
+
if (ctx.store === "memory") return "";
|
|
2002
|
+
return `# Generated by \`substrate init\`. Postgres + backup sidecar.
|
|
2003
|
+
name: ${ctx.composeName}
|
|
2004
|
+
|
|
2005
|
+
services:
|
|
2006
|
+
postgres:
|
|
2007
|
+
image: pgvector/pgvector:pg18
|
|
2008
|
+
restart: unless-stopped
|
|
2009
|
+
environment:
|
|
2010
|
+
POSTGRES_USER: \${POSTGRES_USER:-substrate}
|
|
2011
|
+
POSTGRES_PASSWORD: \${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set in .env.docker, so pass --env-file .env.docker}
|
|
2012
|
+
POSTGRES_DB: \${SUBSTRATE_DATABASE_NAME:-${ctx.dbName}}
|
|
2013
|
+
ports:
|
|
2014
|
+
- "127.0.0.1:${ctx.pgPort}:5432"
|
|
2015
|
+
volumes:
|
|
2016
|
+
- ./data/pg:/var/lib/postgresql
|
|
2017
|
+
- ./postgres-init:/docker-entrypoint-initdb.d:ro
|
|
2018
|
+
healthcheck:
|
|
2019
|
+
test: ["CMD-SHELL", "pg_isready -U \${POSTGRES_USER:-substrate} -d \${SUBSTRATE_DATABASE_NAME:-${ctx.dbName}}"]
|
|
2020
|
+
interval: 10s
|
|
2021
|
+
timeout: 5s
|
|
2022
|
+
retries: 6
|
|
2023
|
+
start_period: 30s
|
|
2024
|
+
|
|
2025
|
+
# Backup sidecar — COMMENTED OUT: no backups run until you opt
|
|
2026
|
+
# in. pg_dump on a loop (every 24h, retain 14 days — tunable via
|
|
2027
|
+
# BACKUP_INTERVAL_HOURS / BACKUP_RETENTION_DAYS). To enable:
|
|
2028
|
+
# 1. mkdir backups && touch backups/.substrate-backup-target (the marker the
|
|
2029
|
+
# loop requires — without it the loop refuses to write),
|
|
2030
|
+
# 2. uncomment the block below and run docker compose --env-file .env.docker up -d again.
|
|
2031
|
+
# Each dump is a gzipped plain SQL file: restore with gunzip -c <file> | psql.
|
|
2032
|
+
#
|
|
2033
|
+
# postgres-backup:
|
|
2034
|
+
# image: pgvector/pgvector:pg18
|
|
2035
|
+
# restart: unless-stopped
|
|
2036
|
+
# depends_on:
|
|
2037
|
+
# postgres:
|
|
2038
|
+
# condition: service_healthy
|
|
2039
|
+
# user: "\${HOST_UID:-1000}:\${HOST_GID:-1000}"
|
|
2040
|
+
# environment:
|
|
2041
|
+
# PGHOST: postgres
|
|
2042
|
+
# PGUSER: \${POSTGRES_USER:-substrate}
|
|
2043
|
+
# PGPASSWORD: \${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set in .env.docker}
|
|
2044
|
+
# PGDATABASE: \${SUBSTRATE_DATABASE_NAME:-${ctx.dbName}}
|
|
2045
|
+
# BACKUP_INTERVAL_HOURS: \${BACKUP_INTERVAL_HOURS:-24}
|
|
2046
|
+
# BACKUP_RETENTION_DAYS: \${BACKUP_RETENTION_DAYS:-14}
|
|
2047
|
+
# volumes:
|
|
2048
|
+
# - ./backups:/backups
|
|
2049
|
+
# entrypoint: ["/bin/sh", "-c"]
|
|
2050
|
+
# command:
|
|
2051
|
+
# - |
|
|
2052
|
+
# [ -f /backups/.substrate-backup-target ] || { echo "backups/.substrate-backup-target missing, refusing to write" >&2; exit 1; }
|
|
2053
|
+
# while :; do
|
|
2054
|
+
# out=/backups/$$PGDATABASE-$$(date -u +%Y%m%dT%H%M%SZ).sql.gz
|
|
2055
|
+
# if pg_dump -Z 6 -f "$$out.partial"; then mv "$$out.partial" "$$out"; echo "backup ok $$out"
|
|
2056
|
+
# else rm -f "$$out.partial"; echo "backup FAILED" >&2; fi
|
|
2057
|
+
# find /backups -maxdepth 1 -name '*.sql.gz' -mtime +$$BACKUP_RETENTION_DAYS -delete
|
|
2058
|
+
# sleep $$((BACKUP_INTERVAL_HOURS * 3600))
|
|
2059
|
+
# done
|
|
2060
|
+
`;
|
|
2061
|
+
}
|
|
2062
|
+
function runtimePackageJsonTemplate(ctx) {
|
|
2063
|
+
const range = ctx.substrateVersion.includes("-") ? `^${ctx.substrateVersion}` : `^${ctx.substrateVersion.split(".")[0]}.0.0`;
|
|
2064
|
+
const deps = { "@mnmal-ai/substrate": range };
|
|
2065
|
+
for (const p of ctx.plugins) deps[pluginNpmName(p)] = pluginRange(p, range);
|
|
2066
|
+
if (ctx.embedder === "minilm" || ctx.embedder === "bge-base") deps.fastembed = FASTEMBED_RANGE;
|
|
2067
|
+
return `${JSON.stringify({
|
|
2068
|
+
name: "substrate-runtime",
|
|
2069
|
+
private: true,
|
|
2070
|
+
dependencies: deps
|
|
2071
|
+
}, null, 2)}\n`;
|
|
2072
|
+
}
|
|
2073
|
+
/**
|
|
2074
|
+
* pnpm 11 fails the install outright on a dependency build script nobody has
|
|
2075
|
+
* approved. These two ship prebuilt binaries and their scripts only fetch
|
|
2076
|
+
* extras, but an unanswered script is an error, not a skip.
|
|
2077
|
+
*/
|
|
2078
|
+
function pnpmWorkspaceTemplate() {
|
|
2079
|
+
return [
|
|
2080
|
+
"packages: []",
|
|
2081
|
+
"allowBuilds:",
|
|
2082
|
+
" msgpackr-extract: true",
|
|
2083
|
+
" onnxruntime-node: true",
|
|
2084
|
+
""
|
|
2085
|
+
].join("\n");
|
|
2086
|
+
}
|
|
2087
|
+
function pgInitSqlTemplate(ctx) {
|
|
2088
|
+
if (ctx.store === "memory") return "";
|
|
2089
|
+
return `-- Idempotent create of the application database.
|
|
2090
|
+
SELECT 'CREATE DATABASE ${ctx.dbName} OWNER substrate'
|
|
2091
|
+
WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = '${ctx.dbName}')
|
|
2092
|
+
\\gexec
|
|
2093
|
+
`;
|
|
2094
|
+
}
|
|
2095
|
+
function readmeTemplate(ctx) {
|
|
2096
|
+
const lines = [
|
|
2097
|
+
"# Substrate runtime",
|
|
2098
|
+
"",
|
|
2099
|
+
"Generated by `substrate init`.",
|
|
2100
|
+
"",
|
|
2101
|
+
"## Quickstart",
|
|
2102
|
+
""
|
|
2103
|
+
];
|
|
2104
|
+
if (ctx.store === "postgres") {
|
|
2105
|
+
lines.push("```bash");
|
|
2106
|
+
lines.push("# 1. Start postgres.");
|
|
2107
|
+
lines.push("docker compose --env-file .env.docker up -d");
|
|
2108
|
+
lines.push("");
|
|
2109
|
+
lines.push("# 2. Install substrate binary + plugins.");
|
|
2110
|
+
lines.push("pnpm install");
|
|
2111
|
+
lines.push("");
|
|
2112
|
+
lines.push("# 3. Run substrate.");
|
|
2113
|
+
lines.push("set -a && source .env && set +a && ./node_modules/.bin/substrate serve");
|
|
2114
|
+
lines.push("```");
|
|
2115
|
+
} else {
|
|
2116
|
+
lines.push("```bash");
|
|
2117
|
+
lines.push("# Memory mode — data vanishes on restart.");
|
|
2118
|
+
lines.push("pnpm install");
|
|
2119
|
+
lines.push("set -a && source .env && set +a && ./node_modules/.bin/substrate serve");
|
|
2120
|
+
lines.push("```");
|
|
2121
|
+
}
|
|
2122
|
+
if (ctx.store === "postgres") {
|
|
2123
|
+
lines.push("");
|
|
2124
|
+
lines.push("## Backups");
|
|
2125
|
+
lines.push("");
|
|
2126
|
+
lines.push("None are configured. The scaffolded `docker-compose.yml` runs postgres only — the `postgres-backup` sidecar is present but commented out, so this instance has no backup story until you opt in. Uncomment it per the comments in the compose file.");
|
|
2127
|
+
}
|
|
2128
|
+
if (ctx.embedder !== "none") {
|
|
2129
|
+
lines.push("");
|
|
2130
|
+
lines.push("## Semantic recall");
|
|
2131
|
+
lines.push("");
|
|
2132
|
+
lines.push(...recallReadmeLines(ctx.embedder));
|
|
2133
|
+
}
|
|
2134
|
+
lines.push("");
|
|
2135
|
+
lines.push("## Reaching this instance");
|
|
2136
|
+
lines.push("");
|
|
2137
|
+
const curlFlags = `${ctx.substrateUrl.startsWith("https") ? "-sfk" : "-sf"} --http2-prior-knowledge`;
|
|
2138
|
+
lines.push(`Health check: \`curl ${curlFlags} ${ctx.substrateUrl}/health\``);
|
|
2139
|
+
lines.push("");
|
|
2140
|
+
lines.push(`Clients (MCP/CLI — e.g. \`.mcp.json\`) connect via \`SUBSTRATE_URL=${ctx.substrateUrl}\`. That's a **client** setting: the server reads \`PORT\`${ctx.tls ? " + the TLS paths" : ""}, not \`SUBSTRATE_URL\`.`);
|
|
2141
|
+
return `${lines.join("\n")}\n`;
|
|
2142
|
+
}
|
|
2143
|
+
function recallReadmeLines(embedder) {
|
|
2144
|
+
const lines = embedder === "ollama" ? ["`substrate-recall.config.json` points at Ollama on `localhost:11434` with bge-base-en-v1.5. Pull the model first: `ollama pull hf.co/CompendiumLabs/bge-base-en-v1.5-gguf:F16`."] : [`\`substrate-recall.config.json\` runs \`${ONNX_MODELS[embedder].model}\` in-process through \`fastembed\`, which \`pnpm install\` adds (about 540MB, most of it ONNX Runtime). The first write downloads the model into \`models/\`.`];
|
|
2145
|
+
lines.push("", "Recall also ranks by keyword, so a small model loses little. Changing model later means emptying the stored vectors and embedding every row again, through the admin route `POST /substrate/v1/embed/reindex`.");
|
|
2146
|
+
return lines;
|
|
2147
|
+
}
|
|
2148
|
+
|
|
2149
|
+
//#endregion
|
|
2150
|
+
//#region src/cli/init/main.ts
|
|
2151
|
+
function normalizeComposeName(targetDir) {
|
|
2152
|
+
return `substrate-${basename(targetDir).toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "") || "runtime"}`;
|
|
2153
|
+
}
|
|
2154
|
+
/** Both env files carry the postgres password. chmod too: `mode` is ignored when --force overwrites. */
|
|
2155
|
+
function writeSecret(path, content) {
|
|
2156
|
+
writeFileSync(path, content, { mode: 384 });
|
|
2157
|
+
chmodSync(path, 384);
|
|
2158
|
+
}
|
|
2159
|
+
async function runInit(argv) {
|
|
2160
|
+
let config;
|
|
2161
|
+
try {
|
|
2162
|
+
config = parseArgs(argv);
|
|
2163
|
+
} catch (err) {
|
|
2164
|
+
process.stderr.write(`substrate init: ${err.message}\n`);
|
|
2165
|
+
return 1;
|
|
2166
|
+
}
|
|
2167
|
+
if (!config.yes) try {
|
|
2168
|
+
const { runRepl } = await import("./repl-BvDZy-Ut.js");
|
|
2169
|
+
config = await runRepl(config);
|
|
2170
|
+
} catch (err) {
|
|
2171
|
+
if (err.message?.includes("User force closed")) return 130;
|
|
2172
|
+
throw err;
|
|
2173
|
+
}
|
|
2174
|
+
if (config.targetDir === void 0) {
|
|
2175
|
+
process.stderr.write("substrate init: --yes requires a target directory positional arg\n");
|
|
2176
|
+
return 1;
|
|
2177
|
+
}
|
|
2178
|
+
const target = resolve(config.targetDir);
|
|
2179
|
+
if (existsSync(target) && !config.force) {
|
|
2180
|
+
process.stderr.write(`substrate init: ${target} already exists — pass --force to scaffold anyway\n`);
|
|
2181
|
+
return 1;
|
|
2182
|
+
}
|
|
2183
|
+
const pgData = join(target, "data", "pg");
|
|
2184
|
+
if (existsSync(pgData) && readdirSync(pgData).length > 0) {
|
|
2185
|
+
process.stderr.write(`substrate init: ${pgData} already holds a postgres cluster — refusing to overwrite its credentials. Remove it first to start fresh.\n`);
|
|
2186
|
+
return 1;
|
|
2187
|
+
}
|
|
2188
|
+
mkdirSync(target, { recursive: true });
|
|
2189
|
+
const substrateVersion = typeof version === "string" && version.length > 0 ? version : "0.0.0";
|
|
2190
|
+
const ctx = {
|
|
2191
|
+
store: config.store ?? "postgres",
|
|
2192
|
+
port: config.port ?? 3100,
|
|
2193
|
+
pgPort: config.pgPort ?? 5433,
|
|
2194
|
+
plugins: [...config.plugins ?? []],
|
|
2195
|
+
embedder: config.embedder ?? "bge-base",
|
|
2196
|
+
substrateUrl: config.substrateUrl ?? `${config.tls ? "https" : "http"}://localhost:${config.port ?? 3100}`,
|
|
2197
|
+
tls: config.tls,
|
|
2198
|
+
tlsCertPath: config.tlsCertPath,
|
|
2199
|
+
tlsKeyPath: config.tlsKeyPath,
|
|
2200
|
+
pgPassword: randomBytes(32).toString("hex"),
|
|
2201
|
+
dbName: "substrate_prod",
|
|
2202
|
+
composeName: normalizeComposeName(target),
|
|
2203
|
+
substrateVersion,
|
|
2204
|
+
storeProfile: config.profile ?? profileFromAnswers({
|
|
2205
|
+
rowsPerDay: config.rowsPerDay ?? DEFAULT_SIZING.rowsPerDay,
|
|
2206
|
+
retentionDays: config.retentionDays ?? DEFAULT_SIZING.retentionDays,
|
|
2207
|
+
readTargetMs: config.readTargetMs ?? DEFAULT_SIZING.readTargetMs,
|
|
2208
|
+
writeHeavy: config.writeHeavy
|
|
2209
|
+
}),
|
|
2210
|
+
writeHeavy: config.writeHeavy
|
|
2211
|
+
};
|
|
2212
|
+
writeSecret(join(target, ".env"), envBinaryTemplate(ctx));
|
|
2213
|
+
writeFileSync(join(target, "README.md"), readmeTemplate(ctx));
|
|
2214
|
+
writeFileSync(join(target, "pnpm-workspace.yaml"), pnpmWorkspaceTemplate());
|
|
2215
|
+
writeFileSync(join(target, "package.json"), runtimePackageJsonTemplate(ctx));
|
|
2216
|
+
const recallConfig = recallConfigTemplate(ctx.embedder, target);
|
|
2217
|
+
if (recallConfig !== null) writeFileSync(join(target, "substrate-recall.config.json"), recallConfig);
|
|
2218
|
+
if (ctx.store === "postgres") {
|
|
2219
|
+
writeSecret(join(target, ".env.docker"), envDockerTemplate(ctx));
|
|
2220
|
+
writeFileSync(join(target, "docker-compose.yml"), dockerComposeTemplate(ctx));
|
|
2221
|
+
mkdirSync(join(target, "postgres-init"), { recursive: true });
|
|
2222
|
+
writeFileSync(join(target, "postgres-init", "01-create-database.sql"), pgInitSqlTemplate(ctx));
|
|
2223
|
+
mkdirSync(join(target, "data", "pg"), { recursive: true });
|
|
2224
|
+
}
|
|
2225
|
+
process.stdout.write(`substrate init: wrote runtime to ${target}\n`);
|
|
2226
|
+
if (ctx.store === "postgres") {
|
|
2227
|
+
process.stdout.write("\nNext steps:\n");
|
|
2228
|
+
process.stdout.write(` cd ${config.targetDir}\n`);
|
|
2229
|
+
process.stdout.write(" docker compose --env-file .env.docker up -d\n");
|
|
2230
|
+
process.stdout.write(" pnpm install\n");
|
|
2231
|
+
process.stdout.write(" set -a && source .env && set +a && ./node_modules/.bin/substrate serve\n");
|
|
2232
|
+
process.stdout.write(" No backups are configured — see README \"Backups\" to opt in.\n");
|
|
2233
|
+
} else {
|
|
2234
|
+
process.stdout.write("\nNext steps:\n");
|
|
2235
|
+
process.stdout.write(` cd ${config.targetDir}\n`);
|
|
2236
|
+
process.stdout.write(" pnpm install\n");
|
|
2237
|
+
process.stdout.write(" set -a && source .env && set +a && ./node_modules/.bin/substrate serve\n");
|
|
2238
|
+
}
|
|
2239
|
+
return 0;
|
|
2240
|
+
}
|
|
2241
|
+
|
|
2242
|
+
//#endregion
|
|
2243
|
+
//#region src/cli/normalise-vocab/main.ts
|
|
2244
|
+
/**
|
|
2245
|
+
* `substrate normalise-vocab <namespace> <Type> <field> [--apply] [--reject a,b] [--json]`
|
|
2246
|
+
*
|
|
2247
|
+
* Rewrites stored aliases of a governed vocabulary field to their canonical
|
|
2248
|
+
* value, through the admin route `POST /substrate/v1/_system/normalise-vocab`. Dry
|
|
2249
|
+
* run unless `--apply`. Values no alias explains must be listed in `--reject`
|
|
2250
|
+
* (left as they are) before an apply goes ahead. Row metadata is preserved.
|
|
2251
|
+
*
|
|
2252
|
+
* Env: SUBSTRATE_URL, SUBSTRATE_ADMIN_TOKEN (admin Bearer token). Exit 0 on success,
|
|
2253
|
+
* 1 when the server refuses unmapped values, 2 on usage or transport errors.
|
|
2254
|
+
*/
|
|
2255
|
+
async function runNormaliseVocab(argv, options = {}) {
|
|
2256
|
+
const write = options.write ?? ((text) => process.stdout.write(text));
|
|
2257
|
+
const [namespace, type, field] = argv.filter((a, i) => !a.startsWith("--") && argv[i - 1] !== "--reject");
|
|
2258
|
+
if (namespace === void 0 || type === void 0 || field === void 0) {
|
|
2259
|
+
write("usage: substrate normalise-vocab <namespace> <Type> <field> [--apply] [--reject a,b] [--json]\n");
|
|
2260
|
+
return 2;
|
|
2261
|
+
}
|
|
2262
|
+
const baseUrl = trimTrailingSlashes(options.baseUrl ?? process.env.SUBSTRATE_URL ?? "");
|
|
2263
|
+
if (baseUrl === "") {
|
|
2264
|
+
write("SUBSTRATE_URL is required (e.g. https://substrate.example.com:3100)\n");
|
|
2265
|
+
return 2;
|
|
2266
|
+
}
|
|
2267
|
+
const token = options.token ?? process.env.SUBSTRATE_ADMIN_TOKEN;
|
|
2268
|
+
const rejectAt = argv.indexOf("--reject");
|
|
2269
|
+
const reject = rejectAt >= 0 ? (argv[rejectAt + 1] ?? "").split(",").filter((v) => v.length > 0) : [];
|
|
2270
|
+
const apply = argv.includes("--apply");
|
|
2271
|
+
if (token !== void 0 && token.length > 0) try {
|
|
2272
|
+
tokenAudience(baseUrl);
|
|
2273
|
+
} catch (err) {
|
|
2274
|
+
write(`${err instanceof Error ? err.message : String(err)}\n`);
|
|
2275
|
+
return 2;
|
|
2276
|
+
}
|
|
2277
|
+
let res;
|
|
2278
|
+
try {
|
|
2279
|
+
res = await noRedirectFetch(options.fetchImpl)(`${baseUrl}/substrate/v1/_system/normalise-vocab`, {
|
|
2280
|
+
method: "POST",
|
|
2281
|
+
headers: {
|
|
2282
|
+
"content-type": "application/json",
|
|
2283
|
+
...token !== void 0 && token.length > 0 ? { authorization: `Bearer ${token}` } : {}
|
|
2284
|
+
},
|
|
2285
|
+
body: JSON.stringify({
|
|
2286
|
+
namespace,
|
|
2287
|
+
type,
|
|
2288
|
+
field,
|
|
2289
|
+
apply,
|
|
2290
|
+
reject
|
|
2291
|
+
})
|
|
2292
|
+
});
|
|
2293
|
+
} catch (err) {
|
|
2294
|
+
write(`request failed: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
2295
|
+
return 2;
|
|
2296
|
+
}
|
|
2297
|
+
const body = await res.json().catch(() => ({}));
|
|
2298
|
+
if (argv.includes("--json")) write(`${JSON.stringify(body, null, 2)}\n`);
|
|
2299
|
+
if (res.status === 409) {
|
|
2300
|
+
if (!argv.includes("--json")) {
|
|
2301
|
+
write(`refused: ${body.message ?? "unmapped values"}\n`);
|
|
2302
|
+
for (const u of body.unmapped ?? []) write(` ${u.value} ${u.count}\n`);
|
|
2303
|
+
}
|
|
2304
|
+
return 1;
|
|
2305
|
+
}
|
|
2306
|
+
if (!res.ok) {
|
|
2307
|
+
if (!argv.includes("--json")) write(`HTTP ${res.status}: ${body.code ?? ""} ${body.message ?? ""}\n`);
|
|
2308
|
+
return 2;
|
|
2309
|
+
}
|
|
2310
|
+
if (argv.includes("--json")) return 0;
|
|
2311
|
+
write(`${namespace}/${type}.${field}: ${body.scanned ?? "?"} rows scanned${body.truncated ? " (TRUNCATED)" : ""}\n`);
|
|
2312
|
+
const rows = body.applied ? body.migrated ?? [] : body.plan ?? [];
|
|
2313
|
+
write(body.applied ? "rewritten:\n" : "would rewrite (dry run; pass --apply):\n");
|
|
2314
|
+
for (const p of rows) write(` ${p.from} -> ${p.to} ${p.count}\n`);
|
|
2315
|
+
if (rows.length === 0) write(" (nothing)\n");
|
|
2316
|
+
const left = body.applied ? body.stranded ?? [] : body.unmapped ?? [];
|
|
2317
|
+
if (left.length > 0) {
|
|
2318
|
+
write(body.applied ? "left as they are (rejected):\n" : "no alias explains (add one, widen the vocabulary, or --reject):\n");
|
|
2319
|
+
for (const u of left) write(` ${u.value} ${u.count}\n`);
|
|
2320
|
+
}
|
|
2321
|
+
return 0;
|
|
2322
|
+
}
|
|
2323
|
+
|
|
2324
|
+
//#endregion
|
|
2325
|
+
//#region src/cli/run/main.ts
|
|
2326
|
+
const USAGE = "usage: substrate run <script.mts> [--args <json>] [--dry-run | --write] [--max-writes <n>] [--out <file>]\n";
|
|
2327
|
+
/**
|
|
2328
|
+
* Runs a script's default export against a signed, guarded client and prints
|
|
2329
|
+
* one JSON report: the script's return value, every mutation, warnings, stats.
|
|
2330
|
+
*/
|
|
2331
|
+
async function runRun(argv, options = {}) {
|
|
2332
|
+
const write = options.write ?? ((s) => process.stdout.write(s));
|
|
2333
|
+
let script;
|
|
2334
|
+
let mode = "read";
|
|
2335
|
+
let maxWrites = 50;
|
|
2336
|
+
let out;
|
|
2337
|
+
let args = {};
|
|
2338
|
+
for (let i = 0; i < argv.length; i++) {
|
|
2339
|
+
const a = argv[i];
|
|
2340
|
+
if (a === "--dry-run" || a === "--write") {
|
|
2341
|
+
const next = a === "--write" ? "write" : "dry-run";
|
|
2342
|
+
if (mode !== "read" && mode !== next) {
|
|
2343
|
+
write("substrate run: --dry-run and --write are exclusive\n");
|
|
2344
|
+
return 2;
|
|
2345
|
+
}
|
|
2346
|
+
mode = next;
|
|
2347
|
+
} else if (a === "--max-writes" || a === "--out" || a === "--args") {
|
|
2348
|
+
const v = argv[++i];
|
|
2349
|
+
if (v === void 0 || v.startsWith("--")) {
|
|
2350
|
+
write(USAGE);
|
|
2351
|
+
return 2;
|
|
2352
|
+
}
|
|
2353
|
+
if (a === "--out") out = v;
|
|
2354
|
+
else if (a === "--args") {
|
|
2355
|
+
let parsed;
|
|
2356
|
+
try {
|
|
2357
|
+
parsed = JSON.parse(v);
|
|
2358
|
+
} catch {
|
|
2359
|
+
parsed = void 0;
|
|
2360
|
+
}
|
|
2361
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
2362
|
+
write("substrate run: --args takes a JSON object, e.g. '{\"project\":\"substrate\"}'\n");
|
|
2363
|
+
return 2;
|
|
2364
|
+
}
|
|
2365
|
+
args = parsed;
|
|
2366
|
+
} else {
|
|
2367
|
+
maxWrites = Number(v);
|
|
2368
|
+
if (!Number.isInteger(maxWrites) || maxWrites < 0) {
|
|
2369
|
+
write("substrate run: --max-writes takes a non-negative integer\n");
|
|
2370
|
+
return 2;
|
|
2371
|
+
}
|
|
2372
|
+
}
|
|
2373
|
+
} else if (script === void 0 && !a.startsWith("--")) script = a;
|
|
2374
|
+
else {
|
|
2375
|
+
write(USAGE);
|
|
2376
|
+
return 2;
|
|
2377
|
+
}
|
|
2378
|
+
}
|
|
2379
|
+
if (script === void 0) {
|
|
2380
|
+
write(USAGE);
|
|
2381
|
+
return 2;
|
|
2382
|
+
}
|
|
2383
|
+
let entry;
|
|
2384
|
+
try {
|
|
2385
|
+
entry = (await import(pathToFileURL(resolve(script)).href)).default;
|
|
2386
|
+
} catch (err) {
|
|
2387
|
+
write(`substrate run: cannot load ${script}: ${messageOf(err)}\n`);
|
|
2388
|
+
return 2;
|
|
2389
|
+
}
|
|
2390
|
+
if (typeof entry !== "function") {
|
|
2391
|
+
write(`substrate run: ${script} must default-export a function (s) => result\n`);
|
|
2392
|
+
return 2;
|
|
2393
|
+
}
|
|
2394
|
+
let client;
|
|
2395
|
+
let kid;
|
|
2396
|
+
try {
|
|
2397
|
+
({client, kid} = createClientFromEnv({
|
|
2398
|
+
...options.env !== void 0 ? { env: options.env } : {},
|
|
2399
|
+
...options.fetchImpl !== void 0 ? { fetch: options.fetchImpl } : {}
|
|
2400
|
+
}));
|
|
2401
|
+
} catch (err) {
|
|
2402
|
+
write(`substrate run: agent identity: ${messageOf(err)}\n`);
|
|
2403
|
+
return 2;
|
|
2404
|
+
}
|
|
2405
|
+
const report = await runScript(entry, {
|
|
2406
|
+
client,
|
|
2407
|
+
kid,
|
|
2408
|
+
mode,
|
|
2409
|
+
args,
|
|
2410
|
+
maxWrites
|
|
2411
|
+
});
|
|
2412
|
+
await client.close();
|
|
2413
|
+
const json = `${JSON.stringify(report, null, 2)}\n`;
|
|
2414
|
+
if (out !== void 0) {
|
|
2415
|
+
writeFileSync(out, json);
|
|
2416
|
+
write(`${report.ok ? "ok" : `failed: ${report.error}`} · ${report.mutations.length} mutation(s) · ${report.warnings.length} warning(s) · report in ${out}\n`);
|
|
2417
|
+
} else write(json);
|
|
2418
|
+
return report.ok ? 0 : 1;
|
|
2419
|
+
}
|
|
2420
|
+
function messageOf(err) {
|
|
2421
|
+
return err instanceof Error ? err.message : String(err);
|
|
2422
|
+
}
|
|
2423
|
+
|
|
2424
|
+
//#endregion
|
|
2425
|
+
//#region src/mcp/agent-jwt.ts
|
|
2426
|
+
/**
|
|
2427
|
+
* Optional agent-identity JWT provider for the MCP client. When
|
|
2428
|
+
* SUBSTRATE_AGENT_KEY_PATH (path to an Ed25519 private PEM) + SUBSTRATE_AGENT_KEY_ID are
|
|
2429
|
+
* set, returns a cached short-TTL EdDSA JWT (refreshed before expiry) that the
|
|
2430
|
+
* stack verifies in attribute mode → createdBy:{kind:'agent', id:<kid>}.
|
|
2431
|
+
* Returns null when neither is set (the MCP client then sends no Authorization);
|
|
2432
|
+
* setting only one throws.
|
|
2433
|
+
*
|
|
2434
|
+
* Fail-loud by design: a SET-but-missing/malformed SUBSTRATE_AGENT_KEY_PATH
|
|
2435
|
+
* throws at provider construction, crashing MCP startup — misconfiguration
|
|
2436
|
+
* must not silently degrade to anonymous writes.
|
|
2437
|
+
*/
|
|
2438
|
+
/**
|
|
2439
|
+
* Returns a provider that yields `Bearer <jwt>` (the full Authorization header
|
|
2440
|
+
* value) when SUBSTRATE_AGENT_KEY_PATH + SUBSTRATE_AGENT_KEY_ID are set, or null when
|
|
2441
|
+
* unconfigured so callers can skip the header entirely.
|
|
2442
|
+
*/
|
|
2443
|
+
function makeAgentJwtProvider(env, audience) {
|
|
2444
|
+
const keyPath = env.SUBSTRATE_AGENT_KEY_PATH;
|
|
2445
|
+
const kid = env.SUBSTRATE_AGENT_KEY_ID;
|
|
2446
|
+
const hasPath = keyPath !== void 0 && keyPath !== "";
|
|
2447
|
+
const hasKid = kid !== void 0 && kid !== "";
|
|
2448
|
+
if (!hasPath && !hasKid) return () => null;
|
|
2449
|
+
if (!hasPath || !hasKid) throw new Error(`SUBSTRATE_AGENT_KEY_PATH and SUBSTRATE_AGENT_KEY_ID must be set together; only ${hasPath ? "SUBSTRATE_AGENT_KEY_PATH" : "SUBSTRATE_AGENT_KEY_ID"} is set`);
|
|
2450
|
+
const provider = makeAuthorizationProvider({
|
|
2451
|
+
keyPath,
|
|
2452
|
+
kid
|
|
2453
|
+
}, {
|
|
2454
|
+
capabilities: resolveAgentCapabilities(env),
|
|
2455
|
+
...audience !== void 0 ? { audience } : {}
|
|
2456
|
+
});
|
|
2457
|
+
return () => provider();
|
|
2458
|
+
}
|
|
2459
|
+
|
|
2460
|
+
//#endregion
|
|
2461
|
+
//#region src/mcp/run-sandbox.ts
|
|
2462
|
+
/**
|
|
2463
|
+
* Runs agent-supplied script text in a sandboxed child process (Decision #642).
|
|
2464
|
+
*
|
|
2465
|
+
* The child runs under Node's permission model: it may read only its runner and
|
|
2466
|
+
* the script, and has no network, child processes, workers or environment. Each
|
|
2467
|
+
* ScriptContext call it makes is relayed to a real one from `runScript`, so the
|
|
2468
|
+
* read / dry-run / write modes, `maxWrites`, signing and the report are exactly
|
|
2469
|
+
* the CLI's.
|
|
2470
|
+
*/
|
|
2471
|
+
const DEFAULT_RUN_TIMEOUT_MS = 6e4;
|
|
2472
|
+
/** Heap ceiling for one script's child, in MiB. */
|
|
2473
|
+
const CHILD_HEAP_MB = 256;
|
|
2474
|
+
const OPS = /* @__PURE__ */ new Set([
|
|
2475
|
+
"query",
|
|
2476
|
+
"queryAll",
|
|
2477
|
+
"byShortId",
|
|
2478
|
+
"recall",
|
|
2479
|
+
"schema",
|
|
2480
|
+
"mutate",
|
|
2481
|
+
"log"
|
|
2482
|
+
]);
|
|
2483
|
+
/** The runner beside this module: `.js` in the built package, `.ts` when run from source. */
|
|
2484
|
+
function runnerPath() {
|
|
2485
|
+
const js = fileURLToPath(new URL("./script-runner.js", import.meta.url));
|
|
2486
|
+
return existsSync(js) ? js : fileURLToPath(new URL("./script-runner.ts", import.meta.url));
|
|
2487
|
+
}
|
|
2488
|
+
async function runSandboxed(source, opts) {
|
|
2489
|
+
const dir = mkdtempSync(join(tmpdir(), "substrate-run-"));
|
|
2490
|
+
const script = join(dir, "script.mts");
|
|
2491
|
+
writeFileSync(script, source);
|
|
2492
|
+
const runner = runnerPath();
|
|
2493
|
+
const timeoutMs = opts.timeoutMs ?? 6e4;
|
|
2494
|
+
try {
|
|
2495
|
+
return await runScript((s) => inChild(s, runner, script, timeoutMs), {
|
|
2496
|
+
client: opts.client,
|
|
2497
|
+
kid: opts.kid,
|
|
2498
|
+
...opts.mode !== void 0 ? { mode: opts.mode } : {},
|
|
2499
|
+
...opts.args !== void 0 ? { args: opts.args } : {},
|
|
2500
|
+
...opts.maxWrites !== void 0 ? { maxWrites: opts.maxWrites } : {},
|
|
2501
|
+
...opts.log !== void 0 ? { log: opts.log } : {}
|
|
2502
|
+
});
|
|
2503
|
+
} finally {
|
|
2504
|
+
rmSync(dir, {
|
|
2505
|
+
recursive: true,
|
|
2506
|
+
force: true
|
|
2507
|
+
});
|
|
2508
|
+
}
|
|
2509
|
+
}
|
|
2510
|
+
function inChild(s, runner, script, timeoutMs) {
|
|
2511
|
+
return new Promise((resolve, reject) => {
|
|
2512
|
+
const child = fork(runner, [], {
|
|
2513
|
+
execArgv: [
|
|
2514
|
+
"--permission",
|
|
2515
|
+
`--allow-fs-read=${runner}`,
|
|
2516
|
+
`--allow-fs-read=${script}`,
|
|
2517
|
+
`--max-old-space-size=${CHILD_HEAP_MB}`
|
|
2518
|
+
],
|
|
2519
|
+
env: {},
|
|
2520
|
+
stdio: [
|
|
2521
|
+
"ignore",
|
|
2522
|
+
"ignore",
|
|
2523
|
+
"ignore",
|
|
2524
|
+
"ipc"
|
|
2525
|
+
],
|
|
2526
|
+
serialization: "advanced"
|
|
2527
|
+
});
|
|
2528
|
+
let settled = false;
|
|
2529
|
+
const finish = (err, value) => {
|
|
2530
|
+
if (settled) return;
|
|
2531
|
+
settled = true;
|
|
2532
|
+
clearTimeout(timer);
|
|
2533
|
+
child.kill("SIGKILL");
|
|
2534
|
+
if (err !== null) reject(err);
|
|
2535
|
+
else resolve(value);
|
|
2536
|
+
};
|
|
2537
|
+
const timer = setTimeout(() => finish(/* @__PURE__ */ new Error(`script timed out after ${timeoutMs} ms`)), timeoutMs);
|
|
2538
|
+
child.on("error", (err) => finish(err));
|
|
2539
|
+
child.on("exit", (code, signal) => finish(/* @__PURE__ */ new Error(`script process exited (${signal ?? `code ${code}`}) before finishing`)));
|
|
2540
|
+
child.on("message", (m) => {
|
|
2541
|
+
if (m.type === "done") {
|
|
2542
|
+
if (m.ok === true) finish(null, m.result);
|
|
2543
|
+
else finish(new Error(m.error ?? "script failed"));
|
|
2544
|
+
return;
|
|
2545
|
+
}
|
|
2546
|
+
if (m.type !== "call" || m.id === void 0) return;
|
|
2547
|
+
const id = m.id;
|
|
2548
|
+
const reply = (r) => {
|
|
2549
|
+
if (!settled) child.send({
|
|
2550
|
+
type: "reply",
|
|
2551
|
+
id,
|
|
2552
|
+
...r
|
|
2553
|
+
});
|
|
2554
|
+
};
|
|
2555
|
+
if (m.op === void 0 || !OPS.has(m.op)) {
|
|
2556
|
+
reply({
|
|
2557
|
+
ok: false,
|
|
2558
|
+
error: `unknown context call '${String(m.op)}'`
|
|
2559
|
+
});
|
|
2560
|
+
return;
|
|
2561
|
+
}
|
|
2562
|
+
const fn = s[m.op];
|
|
2563
|
+
Promise.resolve().then(() => fn(...m.input ?? [])).then((value) => reply({
|
|
2564
|
+
ok: true,
|
|
2565
|
+
value
|
|
2566
|
+
}), (err) => reply({
|
|
2567
|
+
ok: false,
|
|
2568
|
+
error: err instanceof Error ? err.message : String(err)
|
|
2569
|
+
}));
|
|
2570
|
+
});
|
|
2571
|
+
child.send({
|
|
2572
|
+
type: "start",
|
|
2573
|
+
script,
|
|
2574
|
+
kid: s.kid,
|
|
2575
|
+
mode: s.mode,
|
|
2576
|
+
args: { ...s.args }
|
|
2577
|
+
});
|
|
2578
|
+
});
|
|
2579
|
+
}
|
|
2580
|
+
|
|
2581
|
+
//#endregion
|
|
2582
|
+
//#region src/mcp/schema-overview.ts
|
|
2583
|
+
/**
|
|
2584
|
+
* A compact map of the federated schema for agents: every namespace's types,
|
|
2585
|
+
* each as one line of `field: type`, plus the mutations autoCRUD did not
|
|
2586
|
+
* generate. Enough to pick a type and write a query; a write contract is
|
|
2587
|
+
* still one scoped `schema { namespace, type }` away.
|
|
2588
|
+
*/
|
|
2589
|
+
const NAME = /^[A-Za-z_][A-Za-z0-9_-]{0,63}$/;
|
|
2590
|
+
const ENUM_VALUE = /^[A-Za-z0-9_./-]{1,40}$/;
|
|
2591
|
+
const MAX_ENUM_VALUES = 20;
|
|
2592
|
+
const MAX_ENUM_TEXT = 200;
|
|
2593
|
+
function safe(text, pattern) {
|
|
2594
|
+
return typeof text === "string" && pattern.test(text) ? text : "<unsafe>";
|
|
2595
|
+
}
|
|
2596
|
+
function enumText(values) {
|
|
2597
|
+
const shown = [];
|
|
2598
|
+
let length = 0;
|
|
2599
|
+
for (const v of values.slice(0, MAX_ENUM_VALUES)) {
|
|
2600
|
+
const text = safe(v, ENUM_VALUE);
|
|
2601
|
+
if (length + text.length > MAX_ENUM_TEXT) break;
|
|
2602
|
+
shown.push(text);
|
|
2603
|
+
length += text.length + 1;
|
|
2604
|
+
}
|
|
2605
|
+
return [...shown, ...shown.length < values.length ? ["…"] : []].join("|");
|
|
2606
|
+
}
|
|
2607
|
+
function fieldType(f, ns) {
|
|
2608
|
+
let t;
|
|
2609
|
+
if (f.kind === "list" && f.of !== void 0) t = `${fieldType(f.of, ns)}[]`;
|
|
2610
|
+
else if (f.kind === "crossRef") t = `→${f.ns === ns || f.ns === void 0 ? "" : `${safe(f.ns, NAME)}/`}${safe(f.type, NAME)}`;
|
|
2611
|
+
else if (f.kind === "struct") t = `{${Object.keys(f.fields ?? {}).map((k) => safe(k, NAME)).join(", ")}}`;
|
|
2612
|
+
else if (Array.isArray(f.enum)) t = f.nullable ? `(${enumText(f.enum)})` : enumText(f.enum);
|
|
2613
|
+
else t = safe(f.type ?? f.kind, NAME);
|
|
2614
|
+
const tags = [];
|
|
2615
|
+
if (f.readonly === true || f.serverSet !== void 0 && f.serverSet !== false) tags.push("readonly");
|
|
2616
|
+
return `${t}${f.nullable ? "?" : ""}${tags.length > 0 ? ` [${tags.join(", ")}]` : ""}`;
|
|
2617
|
+
}
|
|
2618
|
+
const AUTO_CRUD = /^(get|create|update|delete)(.+)$/;
|
|
2619
|
+
function schemaOverview(federated) {
|
|
2620
|
+
const out = {};
|
|
2621
|
+
for (const [ns, schema] of Object.entries(federated)) {
|
|
2622
|
+
const typeDefs = schema.types ?? {};
|
|
2623
|
+
const types = {};
|
|
2624
|
+
for (const [name, def] of Object.entries(typeDefs)) types[safe(name, NAME)] = [...Object.entries(def.fields ?? {}).map(([field, shape]) => `${safe(field, NAME)}: ${fieldType(shape, ns)}`), ...(def.inboundRefs ?? []).map((r) => `←${safe(r.type, NAME)}.${safe(r.field, NAME)}`)].join(", ");
|
|
2625
|
+
const mutations = Object.keys(schema.mutations ?? {}).filter((m) => {
|
|
2626
|
+
const crud = AUTO_CRUD.exec(m);
|
|
2627
|
+
return !(crud?.[2] !== void 0 && crud[2] in typeDefs);
|
|
2628
|
+
}).map((m) => safe(m, NAME));
|
|
2629
|
+
out[safe(ns, NAME)] = mutations.length > 0 ? {
|
|
2630
|
+
types,
|
|
2631
|
+
mutations
|
|
2632
|
+
} : { types };
|
|
2633
|
+
}
|
|
2634
|
+
return out;
|
|
2635
|
+
}
|
|
2636
|
+
const overviewLegend = "Each type is `field: type` pairs. `a|b` is an enum, `?` nullable, `T[]` a list, `→Type` a reference (`→ns/Type` across namespaces). No field needs an index to filter; json filters by path only, and only non-json scalars sort. [readonly] fields are server-set. Generated get/create/update/delete<Type> mutations are not listed; `mutations` holds the rest. For a type's exact write contract call `schema` with `namespace` and `type`. `←Type.field` is a reference TO this type; follow it backwards with `_walk: { in: \"ns/Type.field\" }` in `where`, as a nested `fields` entry, or in `_agg`.";
|
|
2637
|
+
/** Past this, the instructions carry only type names; the full overview is one `schema` call away. */
|
|
2638
|
+
const INSTRUCTIONS_OVERVIEW_MAX = 12e3;
|
|
2639
|
+
const CONVENTIONS = [
|
|
2640
|
+
"Substrate holds typed records. Every query and mutate key is qualified: `<namespace>/<Type>` or `<namespace>/<mutation>`.",
|
|
2641
|
+
"- Filters go in `params.where`, never top-level params: `{ \"ns/Type\": { \"params\": { \"where\": { \"status\": { \"eq\": \"open\" } }, \"limit\": 20 }, \"fields\": { \"id\": true } } }`. A refused operator is named with the field and its kind; read it rather than guessing.",
|
|
2642
|
+
"- Read `errors` before `data`. A refused frame is ABSENT from `data`; a present empty list means nothing matched. A read with `_pagination.hasMore` is truncated: page with `nextCursor` (default page 100, max 1000).",
|
|
2643
|
+
"- Lists, counts and joins are exact `query` frames; `recall` (similarity search) needs `query`, `namespace` and `type`, and is for finding records by meaning.",
|
|
2644
|
+
"- In one `mutate` call the single-frame keys commit together or not at all, and an array under one key is one transaction. Send unrelated writes as separate calls.",
|
|
2645
|
+
"- `createdAt`, `updatedAt`, `createdBy` and `updatedBy` are server-set: never send them; read them through `_metadata`.",
|
|
2646
|
+
"- `whoami` says which identity this session reads and writes as.",
|
|
2647
|
+
"- With a shell, `substrate run <script.mts>` runs many reads and writes in one call and returns one small report (GET /substrate/v1/onboard, `instructions.codeMode`)."
|
|
2648
|
+
].join("\n");
|
|
2649
|
+
/** The MCP `instructions` text: the conventions, then this server's types as of session start. */
|
|
2650
|
+
function buildInstructions(overview) {
|
|
2651
|
+
if (overview === null) return `${CONVENTIONS}\n\nThe type list was unavailable at session start; call \`schema\` with no arguments for it.`;
|
|
2652
|
+
const full = JSON.stringify(overview);
|
|
2653
|
+
const body = full.length <= INSTRUCTIONS_OVERVIEW_MAX ? `${overviewLegend}\n${full}` : `Type names only (the full overview is large; call \`schema\` with no arguments for fields):\n${JSON.stringify(Object.fromEntries(Object.entries(overview).map(([ns, o]) => [ns, Object.keys(o.types)])))}`;
|
|
2654
|
+
return `${CONVENTIONS}\n\nTypes on this server at session start:\n${body}`;
|
|
2655
|
+
}
|
|
2656
|
+
/** This binary's own catalogue: the fallback when the server publishes none. */
|
|
2657
|
+
const localExpressions = {
|
|
2658
|
+
functions: EXPR_FUNCTIONS,
|
|
2659
|
+
aggregates: EXPR_AGGREGATES,
|
|
2660
|
+
limits: EXPR_LIMITS,
|
|
2661
|
+
dateUnits: DATE_UNITS
|
|
2662
|
+
};
|
|
2663
|
+
/** The `expressions` section of an unscoped schema reply, or undefined when it has none. */
|
|
2664
|
+
function serverExpressions(schema) {
|
|
2665
|
+
const e = schema?.expressions;
|
|
2666
|
+
return Array.isArray(e?.functions) && Array.isArray(e?.aggregates) && Array.isArray(e?.dateUnits) && e?.limits ? e : void 0;
|
|
2667
|
+
}
|
|
2668
|
+
/** The expression catalogue as compact lines for the no-args overview: one per function, then aggregates, limits and date units. */
|
|
2669
|
+
function expressionLines(c) {
|
|
2670
|
+
return [
|
|
2671
|
+
...c.functions.map((f) => {
|
|
2672
|
+
const args = f.args === "variadic" ? "..." : f.args.join(", ");
|
|
2673
|
+
return `${f.name}(${args}) → ${f.returns}: ${f.description}`;
|
|
2674
|
+
}),
|
|
2675
|
+
`aggregates: ${c.aggregates.join(", ")}`,
|
|
2676
|
+
`limits: maxDepth ${c.limits.maxDepth}, maxNodesPerExpr ${c.limits.maxNodesPerExpr}, maxNodesPerFrame ${c.limits.maxNodesPerFrame}`,
|
|
2677
|
+
`dateUnits: ${c.dateUnits.join(", ")}`
|
|
2678
|
+
];
|
|
2679
|
+
}
|
|
2680
|
+
/** Said beside a catalogue that is this binary's own, which may differ from what the server accepts. */
|
|
2681
|
+
const LOCAL_EXPRESSIONS_NOTE = "The server published no expression catalogue (GET /substrate/v1/schema), so this is the local binary's; the server may accept a different set.";
|
|
2682
|
+
|
|
2683
|
+
//#endregion
|
|
2684
|
+
//#region src/mcp/server.ts
|
|
2685
|
+
/**
|
|
2686
|
+
* stdio MCP server exposing Substrate's query/mutate/schema surface to
|
|
2687
|
+
* MCP-speaking agents. Spec: §5.7.
|
|
2688
|
+
*
|
|
2689
|
+
* Every tool is a thin JSON-over-HTTP proxy to a `/substrate/v1/*` route. Frame keys
|
|
2690
|
+
* are qualified (`<ns>/<Type>`, §4.7), so one call can fan across namespaces.
|
|
2691
|
+
* Subscribe / stream are long-lived frame streams and not exposed — stdio cannot host
|
|
2692
|
+
* long-lived push streams cleanly.
|
|
2693
|
+
*
|
|
2694
|
+
* Run by the `substrate mcp` subcommand via `runMcp`.
|
|
2695
|
+
*/
|
|
2696
|
+
/** The version the MCP server reports in its initialize serverInfo. */
|
|
2697
|
+
const MCP_SERVER_VERSION = typeof version === "string" && version.length > 0 ? version : "0.0.0";
|
|
2698
|
+
let agentJwt = () => null;
|
|
2699
|
+
/**
|
|
2700
|
+
* Call at MCP server startup (next to where SUBSTRATE_URL is read) to arm the
|
|
2701
|
+
* agent JWT; its `aud` is `endpoint`'s origin.
|
|
2702
|
+
*/
|
|
2703
|
+
function configureAgentAuth(env, endpoint) {
|
|
2704
|
+
agentJwt = makeAgentJwtProvider(env, new URL(endpoint).origin);
|
|
2705
|
+
}
|
|
2706
|
+
/** Request headers for `endpoint`; a token is only sent where `tokenAudience` allows. */
|
|
2707
|
+
function authHeaders(endpoint, extra) {
|
|
2708
|
+
const token = agentJwt();
|
|
2709
|
+
if (token === null) return extra;
|
|
2710
|
+
tokenAudience(endpoint);
|
|
2711
|
+
return {
|
|
2712
|
+
...extra,
|
|
2713
|
+
authorization: token
|
|
2714
|
+
};
|
|
2715
|
+
}
|
|
2716
|
+
const payloadSchema = (example) => z.unknown().transform((raw, ctx) => {
|
|
2717
|
+
if (typeof raw === "string") try {
|
|
2718
|
+
const parsed = JSON.parse(raw);
|
|
2719
|
+
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
2720
|
+
ctx.addIssue({
|
|
2721
|
+
code: "custom",
|
|
2722
|
+
message: "payload must decode to a JSON object"
|
|
2723
|
+
});
|
|
2724
|
+
return z.NEVER;
|
|
2725
|
+
}
|
|
2726
|
+
return parsed;
|
|
2727
|
+
} catch {
|
|
2728
|
+
ctx.addIssue({
|
|
2729
|
+
code: "custom",
|
|
2730
|
+
message: "payload must be valid JSON"
|
|
2731
|
+
});
|
|
2732
|
+
return z.NEVER;
|
|
2733
|
+
}
|
|
2734
|
+
if (raw === null || typeof raw !== "object" || Array.isArray(raw)) {
|
|
2735
|
+
ctx.addIssue({
|
|
2736
|
+
code: "custom",
|
|
2737
|
+
message: "payload must be an object or a JSON string decoding to an object"
|
|
2738
|
+
});
|
|
2739
|
+
return z.NEVER;
|
|
2740
|
+
}
|
|
2741
|
+
return raw;
|
|
2742
|
+
}).describe(example);
|
|
2743
|
+
/**
|
|
2744
|
+
* query/mutate input: `payload`, plus any other keys kept, so a bare
|
|
2745
|
+
* `{ "<ns>/<Type>": ... }` frame reaches the handler (frameOf) instead of being
|
|
2746
|
+
* stripped and refused. Optional, so the SDK does not refuse the bare call first.
|
|
2747
|
+
*/
|
|
2748
|
+
const frameInputSchema = (example) => z.object({ payload: payloadSchema(example).optional() }).loose();
|
|
2749
|
+
/** The frame a call carries: its `payload`, or the arguments themselves when every key is `<ns>/<name>`. */
|
|
2750
|
+
function frameOf(args) {
|
|
2751
|
+
if (args.payload !== void 0) return args.payload;
|
|
2752
|
+
const keys = Object.keys(args).filter((k) => k !== "payload");
|
|
2753
|
+
return keys.length > 0 && keys.every((k) => k.includes("/")) ? args : void 0;
|
|
2754
|
+
}
|
|
2755
|
+
const MISSING_PAYLOAD = "missing payload: wrap the frame as { \"payload\": { \"<ns>/<Type>\": { \"params\": {...}, \"fields\": {...} } } }";
|
|
2756
|
+
function errorResult(message) {
|
|
2757
|
+
return {
|
|
2758
|
+
content: [{
|
|
2759
|
+
type: "text",
|
|
2760
|
+
text: `Error: ${message}`
|
|
2761
|
+
}],
|
|
2762
|
+
isError: true
|
|
2763
|
+
};
|
|
2764
|
+
}
|
|
2765
|
+
function okResult(value) {
|
|
2766
|
+
return { content: [{
|
|
2767
|
+
type: "text",
|
|
2768
|
+
text: JSON.stringify(value)
|
|
2769
|
+
}] };
|
|
2770
|
+
}
|
|
2771
|
+
function toMessage(err) {
|
|
2772
|
+
if (!(err instanceof Error)) return String(err);
|
|
2773
|
+
if (err.cause instanceof Error && err.cause.message) return `${err.message}: ${err.cause.message}`;
|
|
2774
|
+
return err.message;
|
|
2775
|
+
}
|
|
2776
|
+
/**
|
|
2777
|
+
* POST a JSON body to a Substrate HTTP endpoint and return the parsed response.
|
|
2778
|
+
*
|
|
2779
|
+
* `errorMode` controls the !ok branch:
|
|
2780
|
+
* - 'parsedError' — try to parse the response as Substrate's `{error: {message}}`
|
|
2781
|
+
* envelope; throw the inner message. Falls back to `HTTP <status>` if the
|
|
2782
|
+
* body isn't shaped that way. Used by /substrate/v1/recall.
|
|
2783
|
+
* - 'textPreview' (default) — read the response as text and throw a
|
|
2784
|
+
* `POST <path> returned <status>: <preview>` message capped at 500 chars.
|
|
2785
|
+
* Used by /substrate/v1/nl + /substrate/v1/_system/drop-{namespace,type}.
|
|
2786
|
+
*/
|
|
2787
|
+
async function postSubstrate(endpoint, path, body, errorMode = "textPreview") {
|
|
2788
|
+
const url = `${trimTrailingSlashes(endpoint)}${path}`;
|
|
2789
|
+
let res;
|
|
2790
|
+
try {
|
|
2791
|
+
res = await substrateFetch(url, {
|
|
2792
|
+
method: "POST",
|
|
2793
|
+
headers: authHeaders(endpoint, {
|
|
2794
|
+
"content-type": "application/json",
|
|
2795
|
+
accept: "application/json"
|
|
2796
|
+
}),
|
|
2797
|
+
body: JSON.stringify(body),
|
|
2798
|
+
redirect: "error"
|
|
2799
|
+
});
|
|
2800
|
+
} catch (err) {
|
|
2801
|
+
throw new Error(`Substrate request to ${url} failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
2802
|
+
}
|
|
2803
|
+
if (!res.ok) {
|
|
2804
|
+
if (errorMode === "parsedError") {
|
|
2805
|
+
let parsed = null;
|
|
2806
|
+
try {
|
|
2807
|
+
parsed = await res.json();
|
|
2808
|
+
} catch {}
|
|
2809
|
+
const env = parsed !== null && typeof parsed === "object" ? parsed : void 0;
|
|
2810
|
+
const flat = typeof env?.message === "string" ? env.message : void 0;
|
|
2811
|
+
const message = env !== void 0 && "error" in env ? env.error?.message ?? flat ?? "unknown error" : flat ?? `HTTP ${res.status}`;
|
|
2812
|
+
throw new Error(message);
|
|
2813
|
+
}
|
|
2814
|
+
const preview = await res.text().catch(() => "<unreadable>");
|
|
2815
|
+
throw new Error(`POST ${path} returned ${res.status}: ${preview.slice(0, 500)}`);
|
|
2816
|
+
}
|
|
2817
|
+
return await res.json();
|
|
2818
|
+
}
|
|
2819
|
+
/**
|
|
2820
|
+
* GET a JSON body from a Substrate HTTP endpoint and return the parsed response.
|
|
2821
|
+
* The read-only sibling of `postSubstrate` — used for `/substrate/v1/federated-schema`.
|
|
2822
|
+
* Non-2xx throws `GET <path> returned <status>: <preview>` (capped 500 chars);
|
|
2823
|
+
* a transport failure wraps the cause with the failing URL.
|
|
2824
|
+
*/
|
|
2825
|
+
async function getSubstrate(endpoint, path) {
|
|
2826
|
+
const url = `${trimTrailingSlashes(endpoint)}${path}`;
|
|
2827
|
+
let res;
|
|
2828
|
+
try {
|
|
2829
|
+
res = await substrateFetch(url, {
|
|
2830
|
+
method: "GET",
|
|
2831
|
+
headers: authHeaders(endpoint, { accept: "application/json" }),
|
|
2832
|
+
redirect: "error"
|
|
2833
|
+
});
|
|
2834
|
+
} catch (err) {
|
|
2835
|
+
throw new Error(`Substrate request to ${url} failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
2836
|
+
}
|
|
2837
|
+
if (!res.ok) {
|
|
2838
|
+
const preview = await res.text().catch(() => "<unreadable>");
|
|
2839
|
+
throw new Error(`GET ${path} returned ${res.status}: ${preview.slice(0, 500)}`);
|
|
2840
|
+
}
|
|
2841
|
+
return await res.json();
|
|
2842
|
+
}
|
|
2843
|
+
async function recall(endpoint, body) {
|
|
2844
|
+
return postSubstrate(endpoint, "/substrate/v1/recall", body, "parsedError");
|
|
2845
|
+
}
|
|
2846
|
+
/** The server's expression catalogue, or undefined when it publishes none (an older server, or no default namespace). */
|
|
2847
|
+
async function expressionsOf(endpoint) {
|
|
2848
|
+
try {
|
|
2849
|
+
return serverExpressions(await getSubstrate(endpoint, "/substrate/v1/schema"));
|
|
2850
|
+
} catch {
|
|
2851
|
+
return;
|
|
2852
|
+
}
|
|
2853
|
+
}
|
|
2854
|
+
function createSchemaHandler(endpoint) {
|
|
2855
|
+
return async (args) => {
|
|
2856
|
+
const ns = args?.namespace;
|
|
2857
|
+
const type = args?.type;
|
|
2858
|
+
const shape = args?.shape;
|
|
2859
|
+
if ((ns === void 0 || ns === "") && args?.full !== true) try {
|
|
2860
|
+
const [federated, catalogue] = await Promise.all([getSubstrate(endpoint, "/substrate/v1/federated-schema"), expressionsOf(endpoint)]);
|
|
2861
|
+
return okResult({
|
|
2862
|
+
legend: overviewLegend,
|
|
2863
|
+
overview: schemaOverview(federated),
|
|
2864
|
+
expressions: expressionLines(catalogue ?? localExpressions),
|
|
2865
|
+
...catalogue === void 0 ? { expressionsNote: LOCAL_EXPRESSIONS_NOTE } : {}
|
|
2866
|
+
});
|
|
2867
|
+
} catch (err) {
|
|
2868
|
+
return errorResult(toMessage(err));
|
|
2869
|
+
}
|
|
2870
|
+
let path = "/substrate/v1/federated-schema";
|
|
2871
|
+
if ((ns === void 0 || ns === "") && args?.full === true) try {
|
|
2872
|
+
const [federated, catalogue] = await Promise.all([getSubstrate(endpoint, path), expressionsOf(endpoint)]);
|
|
2873
|
+
return okResult({
|
|
2874
|
+
...federated,
|
|
2875
|
+
$expressions: catalogue ?? {
|
|
2876
|
+
...localExpressions,
|
|
2877
|
+
note: "The server published no expression catalogue (GET /substrate/v1/schema), so this is the local binary's; the server may accept a different set."
|
|
2878
|
+
}
|
|
2879
|
+
});
|
|
2880
|
+
} catch (err) {
|
|
2881
|
+
return errorResult(toMessage(err));
|
|
2882
|
+
}
|
|
2883
|
+
if (ns !== void 0 && ns !== "") {
|
|
2884
|
+
if (type !== void 0 && type !== "") {
|
|
2885
|
+
const q = shape !== void 0 && shape !== "" ? `?shape=${encodeURIComponent(shape)}` : "";
|
|
2886
|
+
path = `/substrate/v1/schema/${encodeURIComponent(ns)}/${encodeURIComponent(type)}${q}`;
|
|
2887
|
+
} else path = `/substrate/v1/schema/${encodeURIComponent(ns)}`;
|
|
2888
|
+
}
|
|
2889
|
+
try {
|
|
2890
|
+
return okResult(await getSubstrate(endpoint, path));
|
|
2891
|
+
} catch (err) {
|
|
2892
|
+
return errorResult(toMessage(err));
|
|
2893
|
+
}
|
|
2894
|
+
};
|
|
2895
|
+
}
|
|
2896
|
+
function createQueryHandler(endpoint) {
|
|
2897
|
+
return async (args) => {
|
|
2898
|
+
const payload = frameOf(args);
|
|
2899
|
+
if (payload === void 0) return errorResult(MISSING_PAYLOAD);
|
|
2900
|
+
try {
|
|
2901
|
+
return okResult(await postSubstrate(endpoint, "/substrate/v1/query", payload));
|
|
2902
|
+
} catch (err) {
|
|
2903
|
+
return errorResult(toMessage(err));
|
|
2904
|
+
}
|
|
2905
|
+
};
|
|
2906
|
+
}
|
|
2907
|
+
function createMutateHandler(endpoint) {
|
|
2908
|
+
return async (args) => {
|
|
2909
|
+
const payload = frameOf(args);
|
|
2910
|
+
if (payload === void 0) return errorResult(MISSING_PAYLOAD);
|
|
2911
|
+
try {
|
|
2912
|
+
return okResult(await postSubstrate(endpoint, "/substrate/v1/mutate", payload));
|
|
2913
|
+
} catch (err) {
|
|
2914
|
+
return errorResult(toMessage(err));
|
|
2915
|
+
}
|
|
2916
|
+
};
|
|
2917
|
+
}
|
|
2918
|
+
function createNlHandler(endpoint) {
|
|
2919
|
+
return async (args) => {
|
|
2920
|
+
try {
|
|
2921
|
+
return okResult(await postSubstrate(endpoint, "/substrate/v1/nl", args));
|
|
2922
|
+
} catch (err) {
|
|
2923
|
+
return errorResult(toMessage(err));
|
|
2924
|
+
}
|
|
2925
|
+
};
|
|
2926
|
+
}
|
|
2927
|
+
function createProposeTypeHandler(endpoint) {
|
|
2928
|
+
return async ({ namespace, ...params }) => {
|
|
2929
|
+
try {
|
|
2930
|
+
return okResult(await postSubstrate(endpoint, "/substrate/v1/mutate", { [`${namespace}/propose_type`]: {
|
|
2931
|
+
params,
|
|
2932
|
+
fields: {
|
|
2933
|
+
id: true,
|
|
2934
|
+
typeName: true,
|
|
2935
|
+
applied: true,
|
|
2936
|
+
appliedAt: true
|
|
2937
|
+
}
|
|
2938
|
+
} }));
|
|
2939
|
+
} catch (err) {
|
|
2940
|
+
return errorResult(toMessage(err));
|
|
2941
|
+
}
|
|
2942
|
+
};
|
|
2943
|
+
}
|
|
2944
|
+
function createProposeFieldHandler(endpoint) {
|
|
2945
|
+
return async ({ namespace, targetType, fieldName, fieldSpec, proposedReason }) => {
|
|
2946
|
+
try {
|
|
2947
|
+
return okResult(await postSubstrate(endpoint, "/substrate/v1/mutate", { [`${namespace}/propose_field`]: {
|
|
2948
|
+
params: {
|
|
2949
|
+
targetType,
|
|
2950
|
+
fieldName,
|
|
2951
|
+
fieldSpec,
|
|
2952
|
+
proposedReason
|
|
2953
|
+
},
|
|
2954
|
+
fields: {
|
|
2955
|
+
id: true,
|
|
2956
|
+
targetType: true,
|
|
2957
|
+
fieldName: true,
|
|
2958
|
+
applied: true,
|
|
2959
|
+
appliedAt: true
|
|
2960
|
+
}
|
|
2961
|
+
} }));
|
|
2962
|
+
} catch (err) {
|
|
2963
|
+
return errorResult(toMessage(err));
|
|
2964
|
+
}
|
|
2965
|
+
};
|
|
2966
|
+
}
|
|
2967
|
+
/** `propose_index` lives on the proposals control plane; `namespace` names the target. */
|
|
2968
|
+
function createProposeIndexHandler(endpoint) {
|
|
2969
|
+
return async (args) => {
|
|
2970
|
+
try {
|
|
2971
|
+
return okResult(await postSubstrate(endpoint, "/substrate/v1/mutate", { "_system_proposals/propose_index": {
|
|
2972
|
+
params: Object.fromEntries(Object.entries(args).filter(([, v]) => v !== void 0)),
|
|
2973
|
+
fields: {
|
|
2974
|
+
id: true,
|
|
2975
|
+
action: true,
|
|
2976
|
+
namespace: true,
|
|
2977
|
+
type: true,
|
|
2978
|
+
indexName: true
|
|
2979
|
+
}
|
|
2980
|
+
} }));
|
|
2981
|
+
} catch (err) {
|
|
2982
|
+
return errorResult(toMessage(err));
|
|
2983
|
+
}
|
|
2984
|
+
};
|
|
2985
|
+
}
|
|
2986
|
+
function createDropNamespaceHandler(endpoint) {
|
|
2987
|
+
return async (input) => {
|
|
2988
|
+
try {
|
|
2989
|
+
return okResult(await postSubstrate(endpoint, "/substrate/v1/_system/drop-namespace", input));
|
|
2990
|
+
} catch (err) {
|
|
2991
|
+
return errorResult(toMessage(err));
|
|
2992
|
+
}
|
|
2993
|
+
};
|
|
2994
|
+
}
|
|
2995
|
+
function createDropTypeHandler(endpoint) {
|
|
2996
|
+
return async (input) => {
|
|
2997
|
+
try {
|
|
2998
|
+
return okResult(await postSubstrate(endpoint, "/substrate/v1/_system/drop-type", input));
|
|
2999
|
+
} catch (err) {
|
|
3000
|
+
return errorResult(toMessage(err));
|
|
3001
|
+
}
|
|
3002
|
+
};
|
|
3003
|
+
}
|
|
3004
|
+
/**
|
|
3005
|
+
* `run`: one script, sandboxed (Decision #642), signed as this MCP server's
|
|
3006
|
+
* agent. The client is made on first use and kept: building it reads the key.
|
|
3007
|
+
*/
|
|
3008
|
+
function createRunHandler(env = process.env) {
|
|
3009
|
+
let signed;
|
|
3010
|
+
return async (input) => {
|
|
3011
|
+
try {
|
|
3012
|
+
signed ??= createClientFromEnv({ env });
|
|
3013
|
+
if (signed.kid === null) return errorResult("run needs an agent identity: set SUBSTRATE_AGENT_KEY_ID and SUBSTRATE_AGENT_KEY_PATH");
|
|
3014
|
+
const report = await runSandboxed(input.script, {
|
|
3015
|
+
client: signed.client,
|
|
3016
|
+
kid: signed.kid,
|
|
3017
|
+
mode: input.mode ?? "read",
|
|
3018
|
+
args: input.args ?? {},
|
|
3019
|
+
maxWrites: input.maxWrites ?? 50,
|
|
3020
|
+
log: () => {}
|
|
3021
|
+
});
|
|
3022
|
+
return report.ok ? okResult(report) : {
|
|
3023
|
+
...okResult(report),
|
|
3024
|
+
isError: true
|
|
3025
|
+
};
|
|
3026
|
+
} catch (err) {
|
|
3027
|
+
return errorResult(toMessage(err));
|
|
3028
|
+
}
|
|
3029
|
+
};
|
|
3030
|
+
}
|
|
3031
|
+
function createMcpServer(endpoint, options = {}) {
|
|
3032
|
+
const server = new McpServer({
|
|
3033
|
+
name: "substrate",
|
|
3034
|
+
version: MCP_SERVER_VERSION
|
|
3035
|
+
}, options.instructions !== void 0 ? { instructions: options.instructions } : void 0);
|
|
3036
|
+
const TOOLS = [
|
|
3037
|
+
{
|
|
3038
|
+
name: "schema",
|
|
3039
|
+
config: {
|
|
3040
|
+
description: "Fetch the Substrate schema. No args → a compact overview: every namespace's types, one line of `field: type` each, enough to write a query. `full: true` → the whole federated schema plus the full expression catalogue under `$expressions` (large; rarely needed). SCOPED + CHEAP: pass `namespace` for one namespace, or `namespace` + `type` for a single type's write contract ({ fields, mutations }). Mine the scoped shape BEFORE a create/update so you send only declared fields — autoCRUD rejects undeclared fields with `unknown_field`. The type answer is compact: generated CRUD names the type ('Todo fields', 'Todo') instead of restating its field block; `mutationRefs` says how to read it. `shape: \"fields\"` drops the mutations to their names; `shape: \"full\"` inlines every block (several times larger).",
|
|
3041
|
+
inputSchema: {
|
|
3042
|
+
namespace: z.string().min(1).optional().describe("Scope to one namespace, e.g. \"notes\". Omit for the full federated schema."),
|
|
3043
|
+
type: z.string().min(1).optional().describe("Scope to one type within `namespace`, e.g. \"Todo\" — returns its field defs + create/update param shapes. Requires `namespace`."),
|
|
3044
|
+
shape: z.enum([
|
|
3045
|
+
"compact",
|
|
3046
|
+
"full",
|
|
3047
|
+
"fields"
|
|
3048
|
+
]).optional().describe("Projection of the type contract; requires `namespace` + `type`. \"compact\" (default) prints the field block once and names the type in generated CRUD. \"full\" inlines every mutation's params and returns. \"fields\" returns the field defs plus `mutationNames` — use it to answer \"what fields does this type have\"."),
|
|
3049
|
+
full: z.boolean().optional().describe("With no `namespace`: true returns the whole federated schema instead of the overview.")
|
|
3050
|
+
}
|
|
3051
|
+
},
|
|
3052
|
+
handler: createSchemaHandler(endpoint)
|
|
3053
|
+
},
|
|
3054
|
+
{
|
|
3055
|
+
name: "query",
|
|
3056
|
+
config: {
|
|
3057
|
+
description: "Query Substrate. payload is a single frame object mapping qualified keys (<ns>/<Type>) to { params, fields }. ONE call can span multiple namespaces — e.g. { \"prices/Price\": {...}, \"products/Product\": {...} }. The expression functions for select, where.$expr, groupBy and having are listed by the `schema` tool. WALKS: a reference field is followed backwards with `_walk`, which names the inbound field as \"ns/Type.field\" (the schema overview marks it `←Type.field`). It filters in `where`, aggregates in `fields._agg`, or nests rows in `fields`. Example: orders with an item priced over 50, and their item count: { \"shop/Order\": { \"params\": { \"where\": { \"_walk\": { \"in\": \"shop/Item.order\", \"where\": { \"price\": { \"gt\": 50 } } } } }, \"fields\": { \"id\": true, \"_agg\": { \"items\": { \"in\": \"shop/Item.order\", \"count\": true } } } } }. `out: \"field\"` follows a reference forwards. A `_walk` takes `quantifier` \"any\" (default), \"none\" or \"all\", or `pick` one neighbour by { \"latest\": <date> }, { \"max\": <f> } or { \"min\": <f> }; an \"any\" walk binds `as: \"L\"`, and a later comparison reads it as { \"$ref\": \"L.price\", \"times\": 1.25 } (`$.f` is the seed row, `_agg.<name>` an aggregate; `times`/`plus` scale numbers, `plus` shifts a date in ms). `_agg` also takes avg/sum/min/max: <field> or latest: { by, value }, and in `where` compares beside a comparison: { \"_agg\": { \"in\": ..., \"count\": true }, \"gte\": 3 }. Frame params: `count: true` answers { count }; `aggregate: { name: { sum: \"price\" } }` answers one group, or one per key of `groupBy: { day: { dateTrunc: [{ \"lit\": \"day\" }, \"seenAt\"] } }` (a map of names to expressions; `having` filters the groups, `offset` skips them, `orderBy` names their outputs). `select: { m: { sub: [\"_agg.x\", \"price\"] } }` adds a computed value to each row, ordered by `orderBy: [{ \"m\": \"desc\" }]`. An expression is a field name, a literal, `{ \"lit\": \"text\" }` or `{ \"fn\": [args] }`. Walks are query-only: subscribe, stream and mutate refuse them, as does a store without walks (walks_unsupported_store).",
|
|
3058
|
+
inputSchema: frameInputSchema("Query frame. Example: { \"default/Todo\": { \"params\": { \"limit\": 20 }, \"fields\": { \"id\": true, \"title\": true } } }")
|
|
3059
|
+
},
|
|
3060
|
+
handler: createQueryHandler(endpoint)
|
|
3061
|
+
},
|
|
3062
|
+
{
|
|
3063
|
+
name: "mutate",
|
|
3064
|
+
config: {
|
|
3065
|
+
description: "Run one or more mutations. payload is a single frame object mapping qualified keys (<ns>/<mutationName>) to { params, fields } (single-frame) or to an array of frames (batched atomic). ONE op-key can accept a single frame or an array. An ARRAY op-key is its own transaction: it commits or rolls back as a unit, independently of every other op-key in the call. The SINGLE-FRAME op-keys share one transaction when they sit on one tx scope (every namespace on one pg handle); across two tx scopes they are refused rather than split. So a single-frame op-key beside an array is NOT rolled back when the array fails — to make writes atomic together, put them in one array under one op-key. The memory dataset mints a fresh tx scope per namespace, so single-frame op-keys in two namespaces are refused there.",
|
|
3066
|
+
inputSchema: frameInputSchema("Mutation frame. Single: { \"default/createTodo\": { \"params\": { \"title\": \"...\" }, \"fields\": { \"id\": true } } }. Batched atomic: { \"default/updateTodo\": [ { \"params\": { \"id\": \"1\", \"done\": true }, \"fields\": { \"id\": true } }, { \"params\": { \"id\": \"2\", \"done\": true }, \"fields\": { \"id\": true } } ] }. By filter: { \"default/updateManyTodo\": { \"params\": { \"where\": { \"done\": { \"eq\": false } }, \"set\": { \"done\": true } }, \"fields\": { \"count\": true, \"ids\": true, \"idsTruncated\": true } } }; deleteMany<Type> takes only \"where\".")
|
|
3067
|
+
},
|
|
3068
|
+
handler: createMutateHandler(endpoint)
|
|
3069
|
+
},
|
|
3070
|
+
{
|
|
3071
|
+
name: "run",
|
|
3072
|
+
config: {
|
|
3073
|
+
description: "Run a short script against Substrate in one call: many queries and writes, one small report back. Use it when the work takes three or more dependent calls (ids passed from one call to the next, a census that must page, a join, recall-then-hydrate, a batch of writes chosen by a read). `script` is a JS/TS module whose default export is `async (s) => result`; type syntax is stripped. `s` has: kid, mode, args; query(frame) (throws on a refused frame); queryAll(key, { params, fields }) (pages to the end); byShortId(key, n | n[], fields?) (rows in input order; throws naming any missing); recall({ query, namespace, type, k?, where?, row? }); schema(namespace?, type?); mutate(frame); log(...). Return counts, ids and short handles, not whole rows. Modes: \"read\" (default) refuses mutate; \"dry-run\" records each mutate payload in the report without sending it; \"write\" sends up to maxWrites frames (default 50; an array element counts one). Dry-run first, read report.mutations, then write. The script runs sandboxed: no files, network, processes or environment; it reaches only the store, as this agent. Report: { ok, kid, mode, result, error?, mutations[{ frame, applied, errors? }], warnings, stats }.",
|
|
3074
|
+
inputSchema: {
|
|
3075
|
+
script: z.string().min(1).describe("Module text, e.g. \"export default async (s) => (await s.queryAll('cortext/Todo', { params: { where: { status: { eq: 'open' } } }, fields: { id: true } })).length\""),
|
|
3076
|
+
args: z.record(z.string(), z.unknown()).optional().describe("Passed to the script as s.args."),
|
|
3077
|
+
mode: z.enum([
|
|
3078
|
+
"read",
|
|
3079
|
+
"dry-run",
|
|
3080
|
+
"write"
|
|
3081
|
+
]).optional().describe("Default \"read\"."),
|
|
3082
|
+
maxWrites: z.number().int().nonnegative().optional().describe("Write mode: most mutate frames to send. Default 50.")
|
|
3083
|
+
}
|
|
3084
|
+
},
|
|
3085
|
+
handler: createRunHandler()
|
|
3086
|
+
},
|
|
3087
|
+
{
|
|
3088
|
+
name: "ask",
|
|
3089
|
+
config: {
|
|
3090
|
+
description: "Ask a natural-language question of one namespace. The parser answers first; a constrained judge handles what it escalates; a judge outage hands off honestly. Returns { outcome, trail }, where outcome.kind is rows | recall | clarify | refuse | unknown. unknown means part of the question was dropped and the answer would change which rows count; it carries dropped and the plan, and nothing ran. rows also carries dropped (what it does not reflect) and plan; an answer of count carries count, and one of aggregate carries aggregate (fn, field, and a value or groups keyed by `by`). Relative dates are read against asOf when given, else the server clock; the parsed trail event says which. Read-only. Requires the @mnmal-ai/substrate-nl plugin (404 otherwise).",
|
|
3091
|
+
inputSchema: {
|
|
3092
|
+
namespace: z.string().min(1).describe("Namespace to ask, e.g. \"notes\"."),
|
|
3093
|
+
question: z.string().min(1).describe("The question, e.g. \"open todos in billing\"."),
|
|
3094
|
+
project: z.string().min(1).optional().describe("Asking project; scopes project-bearing types."),
|
|
3095
|
+
type: z.string().min(1).optional().describe("force the reading to this type (resubmit a clarify option)"),
|
|
3096
|
+
asOf: z.string().min(1).optional().describe("\"now\" for relative dates (\"yesterday\", \"last week\") and month-day year inference: ISO-8601 date-time with an offset, e.g. 2026-10-02T12:00:00-05:00. Default: the server clock.")
|
|
3097
|
+
}
|
|
3098
|
+
},
|
|
3099
|
+
handler: createNlHandler(endpoint)
|
|
3100
|
+
},
|
|
3101
|
+
{
|
|
3102
|
+
name: "propose_type",
|
|
3103
|
+
config: {
|
|
3104
|
+
description: "Propose a brand-new type in an owned namespace. The plugin writes a TypeProposal record and registers the type + autoCRUD surface (createX, getX, updateX, deleteX) at runtime — future queries/mutations against this type succeed immediately. Requires the `substrate.propose` capability. Use `propose_field` to extend an existing type; this tool rejects if the type already exists (structured error code `type_already_proposed` or `type_already_exists`).",
|
|
3105
|
+
inputSchema: {
|
|
3106
|
+
namespace: z.string().min(1).describe("Target namespace (must be in the plugin's ownedNamespaces)."),
|
|
3107
|
+
typeName: z.string().min(1).describe("PascalCase type name, e.g. \"Dress\"."),
|
|
3108
|
+
fields: z.record(z.string(), z.unknown()).describe("Initial field map keyed by field name. Each value is a FieldSpecJson: { kind: \"scalar\" | \"crossRef\" | \"struct\" | \"list\", ... }. Example: { \"name\": { \"kind\": \"scalar\", \"type\": \"string\" }, \"owner\": { \"kind\": \"crossRef\", \"ns\": \"household\", \"type\": \"Person\", \"onDelete\": \"set_null\" } }."),
|
|
3109
|
+
proposedReason: z.string().min(1).describe("Short human-readable justification. Why is this type being added?"),
|
|
3110
|
+
orderedBy: z.record(z.string(), z.enum(["asc", "desc"])).optional().describe("What order do you usually read this in? { \"<field>\": \"asc\" | \"desc\" } over a date or number field, or createdAt / updatedAt. Omit for newest first."),
|
|
3111
|
+
indexes: z.array(z.record(z.string(), z.unknown())).optional().describe("Indexes the reads will need: [{ \"fields\": [\"brand\"] }]. Never unique or named."),
|
|
3112
|
+
expectedSize: z.enum(["small", "large"]).optional().describe("\"large\" if the type will hold many rows, so indexes are advised sooner.")
|
|
3113
|
+
}
|
|
3114
|
+
},
|
|
3115
|
+
handler: createProposeTypeHandler(endpoint)
|
|
3116
|
+
},
|
|
3117
|
+
{
|
|
3118
|
+
name: "propose_field",
|
|
3119
|
+
config: {
|
|
3120
|
+
description: "Propose a new field on an existing type. Used for additive schema evolution AND for declaring edges (propose a crossRef field to model a relationship, including cross-namespace edges). Idempotent on exact spec match; rejects with `destructive_change_required` if a field of the same name exists with a different spec. Requires the `substrate.propose` capability.",
|
|
3121
|
+
inputSchema: {
|
|
3122
|
+
namespace: z.string().min(1).describe("Namespace containing the target type (must be in ownedNamespaces)."),
|
|
3123
|
+
targetType: z.string().min(1).describe("Existing type to extend. May be code-declared or previously proposed."),
|
|
3124
|
+
fieldName: z.string().min(1).describe("camelCase field name to add."),
|
|
3125
|
+
fieldSpec: z.record(z.string(), z.unknown()).describe("Single FieldSpecJson: { kind: \"scalar\" | \"crossRef\" | \"struct\" | \"list\", ... }. For an edge: { \"kind\": \"crossRef\", \"ns\": \"<target-ns>\", \"type\": \"<TargetType>\", \"onDelete\": \"set_null\" }."),
|
|
3126
|
+
proposedReason: z.string().min(1).describe("Short human-readable justification.")
|
|
3127
|
+
}
|
|
3128
|
+
},
|
|
3129
|
+
handler: createProposeFieldHandler(endpoint)
|
|
3130
|
+
},
|
|
3131
|
+
{
|
|
3132
|
+
name: "propose_index",
|
|
3133
|
+
config: {
|
|
3134
|
+
description: "Apply or dismiss an index recommendation (query `_system/IndexRecommendation`), or change indexes directly. `add` / `orderedBy` work on proposed types only (`index_on_code_defined_type` otherwise); `remove` drops an engine-owned, non-unique index the engine no longer wants, on any type (`not_engine_owned`, `unique_index`, `still_desired`, `unknown_index`). A referenced recommendation moves to accepted or dismissed; a closed one is refused (`recommendation_closed`), one whose evidence no longer holds is superseded (`recommendation_stale`). Requires the `substrate.index` capability in the target namespace.",
|
|
3135
|
+
inputSchema: {
|
|
3136
|
+
namespace: z.string().min(1).describe("Namespace of the type whose indexes change."),
|
|
3137
|
+
type: z.string().min(1).describe("Type name, e.g. \"Sneaker\"."),
|
|
3138
|
+
action: z.enum([
|
|
3139
|
+
"add",
|
|
3140
|
+
"remove",
|
|
3141
|
+
"dismiss",
|
|
3142
|
+
"orderedBy"
|
|
3143
|
+
]),
|
|
3144
|
+
reason: z.string().min(1).describe("Short human-readable justification."),
|
|
3145
|
+
recommendationId: z.string().min(1).optional().describe("The IndexRecommendation row id; required for dismiss."),
|
|
3146
|
+
declaration: z.record(z.string(), z.unknown()).optional().describe("add: an index declaration, e.g. { \"fields\": [\"brand\", { \"field\": \"size\", \"dir\": \"desc\" }] }. Defaults to the recommendation's."),
|
|
3147
|
+
orderedBy: z.record(z.string(), z.enum(["asc", "desc"])).optional().describe("orderedBy: the natural order, one key, e.g. { \"releasedAt\": \"desc\" }."),
|
|
3148
|
+
indexName: z.string().min(1).optional().describe("remove: the index to drop. Defaults to the recommendation's.")
|
|
3149
|
+
}
|
|
3150
|
+
},
|
|
3151
|
+
handler: createProposeIndexHandler(endpoint)
|
|
3152
|
+
},
|
|
3153
|
+
{
|
|
3154
|
+
name: "drop_namespace",
|
|
3155
|
+
config: {
|
|
3156
|
+
description: "Admin: physically delete every row + short-id for a given namespace. Refuses by default if any edge references the namespace; pass cascade=\"delete\" to drop those edges too. Dry-run mode reports counts without mutating. Emits a AuditEvent { kind: \"namespace_dropped\" }. Requires the `admin` capability on the server side. DESTRUCTIVE — data cannot be recovered after a non-dry-run call.",
|
|
3157
|
+
inputSchema: {
|
|
3158
|
+
namespace: z.string().min(1).describe("Namespace to drop."),
|
|
3159
|
+
confirm: z.string().min(1).describe("Must equal `namespace` exactly (confirmation gate)."),
|
|
3160
|
+
dryRun: z.boolean().optional().describe("If true, report counts and roll back — no actual deletion."),
|
|
3161
|
+
cascade: z.enum(["refuse", "delete"]).optional().describe("Behaviour when edges reference the namespace. Default \"refuse\" — aborts if any edges exist.")
|
|
3162
|
+
}
|
|
3163
|
+
},
|
|
3164
|
+
handler: createDropNamespaceHandler(endpoint)
|
|
3165
|
+
},
|
|
3166
|
+
{
|
|
3167
|
+
name: "drop_type",
|
|
3168
|
+
config: {
|
|
3169
|
+
description: "Admin: physically delete every row + short-id for a given (namespace, type). Same cascade + dry-run semantics as drop_namespace. Emits a AuditEvent { kind: \"type_dropped\" }. Requires the `admin` capability. DESTRUCTIVE — data cannot be recovered after a non-dry-run call.",
|
|
3170
|
+
inputSchema: {
|
|
3171
|
+
namespace: z.string().min(1).describe("Namespace containing the type to drop."),
|
|
3172
|
+
type: z.string().min(1).describe("Type name to drop, e.g. \"Todo\"."),
|
|
3173
|
+
confirm: z.string().min(1).describe("Must equal \"<namespace>/<type>\" exactly (confirmation gate)."),
|
|
3174
|
+
dryRun: z.boolean().optional().describe("If true, report counts and roll back — no actual deletion."),
|
|
3175
|
+
cascade: z.enum(["refuse", "delete"]).optional().describe("Behaviour when edges reference the type. Default \"refuse\" — aborts if any edges exist.")
|
|
3176
|
+
}
|
|
3177
|
+
},
|
|
3178
|
+
handler: createDropTypeHandler(endpoint)
|
|
3179
|
+
},
|
|
3180
|
+
{
|
|
3181
|
+
name: "whoami",
|
|
3182
|
+
config: {
|
|
3183
|
+
description: "Ask the server who this MCP session is authenticated as. Returns { authEnabled, identity }, where `identity` is null for an anonymous caller and otherwise carries { id, grants } — a namespace -> capabilities map, AS RECORDED, so a wildcard stays a `'*'` key rather than expanding into every namespace. USE THIS BEFORE CONCLUDING A CREDENTIAL WORKS: on a reads-open stack an authenticated read and an anonymous one return byte-identical results, so a successful query is not evidence of identity. `authEnabled: false` means the deployment verifies nothing (a posture you cannot fix from here); `authEnabled: true` with `identity: null` means your own credential never arrived (SUBSTRATE_AGENT_KEY_ID / SUBSTRATE_AGENT_KEY_PATH). Those need different fixes, which is why they are separate fields.",
|
|
3184
|
+
inputSchema: {}
|
|
3185
|
+
},
|
|
3186
|
+
handler: async () => {
|
|
3187
|
+
try {
|
|
3188
|
+
const answer = await getSubstrate(endpoint, "/substrate/v1/whoami");
|
|
3189
|
+
const claimed = resolveAgentCapabilities(process.env);
|
|
3190
|
+
return okResult({
|
|
3191
|
+
...answer,
|
|
3192
|
+
client: {
|
|
3193
|
+
claimedCapabilities: claimed,
|
|
3194
|
+
note: "What this MCP client ASKED for. The effective identity is this claim intersected with the registry row, so a capability absent here is unreachable however the row is granted. Widen with SUBSTRATE_AGENT_CAPABILITIES (csv)."
|
|
3195
|
+
}
|
|
3196
|
+
});
|
|
3197
|
+
} catch (err) {
|
|
3198
|
+
return errorResult(toMessage(err));
|
|
3199
|
+
}
|
|
3200
|
+
}
|
|
3201
|
+
},
|
|
3202
|
+
{
|
|
3203
|
+
name: "recall",
|
|
3204
|
+
config: {
|
|
3205
|
+
description: "Semantic recall over embedded Substrate records. Returns top-K records by cosine similarity to the NL query. Use for finding prior context by meaning when structural `where`/`contains` queries miss. WHICH TYPES ARE EMBEDDED IS A SERVER FACT — ask it (`GET /substrate/v1/embed-stats` returns `embeddedTypes`) rather than assuming; a type is embedded iff its namespace definition declares an `embed.source`. The same operation is also reachable as `POST /substrate/v1/recall` (body param `query`) and as the frame key `recall/Hit` through query — where the search text is `params.to`, not `params.query` and not a `where` clause. See docs/surface-parity.md.",
|
|
3206
|
+
inputSchema: {
|
|
3207
|
+
query: z.string().min(1).describe("Natural-language query to embed and match against."),
|
|
3208
|
+
k: z.number().int().min(1).max(100).optional().describe("Max results. Default 10."),
|
|
3209
|
+
namespace: z.string().optional().describe("Namespace to search. Pass it: there is no default. Omitted, the server falls back to ITS OWN SUBSTRATE_NAMESPACE and refuses with `missing_namespace` if that is unset."),
|
|
3210
|
+
type: z.string().min(1).describe("Type to search, e.g. \"Note\". Required."),
|
|
3211
|
+
threshold: z.number().optional().describe("Minimum similarity (0..1). Default 0."),
|
|
3212
|
+
lexicalThreshold: z.number().optional().describe("Cosine floor for hits only the lexical (keyword) arm found. Default: same as threshold. 0 keeps them unfloored, so keyword matches survive a high threshold."),
|
|
3213
|
+
where: z.record(z.string(), z.unknown()).optional().describe("Row-field filter applied inside every candidate source, so k fills with matching rows: field -> { eq: scalar | null } or { in: [scalar | null, ...] }, every field must match; eq null matches a missing or null field. Example: { \"project\": { \"in\": [\"substrate\", null] } }."),
|
|
3214
|
+
rowFields: z.union([z.record(z.string(), z.literal(true)), z.literal(true)]).optional().describe("Row-body projection. Omitted, each hit's `row` carries `id` plus its handle fields (name, title, topic, synopsis, description), which is enough to decide on. Ask for more only when you need it: field -> true keeps just those fields plus `id`, e.g. { \"synopsis\": true, \"body\": true }; true returns the whole body.")
|
|
3215
|
+
}
|
|
3216
|
+
},
|
|
3217
|
+
handler: async (input) => {
|
|
3218
|
+
try {
|
|
3219
|
+
return okResult(await recall(endpoint, input));
|
|
3220
|
+
} catch (err) {
|
|
3221
|
+
return errorResult(toMessage(err));
|
|
3222
|
+
}
|
|
3223
|
+
}
|
|
3224
|
+
}
|
|
3225
|
+
];
|
|
3226
|
+
const allowed = resolveToolAllowlist(options.tools, TOOLS.map((t) => t.name));
|
|
3227
|
+
for (const t of TOOLS) {
|
|
3228
|
+
if (allowed !== null && !allowed.has(t.name)) continue;
|
|
3229
|
+
server.registerTool(t.name, t.config, t.handler);
|
|
3230
|
+
}
|
|
3231
|
+
return server;
|
|
3232
|
+
}
|
|
3233
|
+
/**
|
|
3234
|
+
* Parse SUBSTRATE_MCP_TOOLS. `null` means register everything. An unknown name
|
|
3235
|
+
* throws: a typo that silently registered nothing would read as "this server
|
|
3236
|
+
* has no such tool", which is a worse failure than refusing to start.
|
|
3237
|
+
*/
|
|
3238
|
+
function resolveToolAllowlist(raw, known) {
|
|
3239
|
+
if (raw === void 0 || raw.trim() === "") return null;
|
|
3240
|
+
const names = raw.split(",").map((n) => n.trim()).filter((n) => n !== "");
|
|
3241
|
+
for (const n of names) if (!known.includes(n)) throw new Error(`SUBSTRATE_MCP_TOOLS: unknown tool '${n}'. Valid names: ${[...known].sort().join(", ")}`);
|
|
3242
|
+
return new Set(names);
|
|
3243
|
+
}
|
|
3244
|
+
/** The overview for the session's instructions, or null when the server is slow or down. */
|
|
3245
|
+
async function fetchOverview(endpoint) {
|
|
3246
|
+
const timeout = new Promise((resolve) => setTimeout(() => resolve(null), 5e3).unref());
|
|
3247
|
+
const fetched = getSubstrate(endpoint, "/substrate/v1/federated-schema").then(schemaOverview, () => null);
|
|
3248
|
+
return Promise.race([fetched, timeout]);
|
|
3249
|
+
}
|
|
3250
|
+
/**
|
|
3251
|
+
* Boot the stdio MCP server. Resolves once the transport is connected; the
|
|
3252
|
+
* process keeps running until the host (Claude Code, etc.) closes stdio.
|
|
3253
|
+
*/
|
|
3254
|
+
async function runMcp() {
|
|
3255
|
+
const endpoint = process.env.SUBSTRATE_URL ?? "http://localhost:3100";
|
|
3256
|
+
configureAgentAuth(process.env, endpoint);
|
|
3257
|
+
const server = createMcpServer(endpoint, {
|
|
3258
|
+
...process.env.SUBSTRATE_MCP_TOOLS === void 0 ? {} : { tools: process.env.SUBSTRATE_MCP_TOOLS },
|
|
3259
|
+
instructions: buildInstructions(await fetchOverview(endpoint))
|
|
3260
|
+
});
|
|
3261
|
+
const transport = new StdioServerTransport();
|
|
3262
|
+
await server.connect(transport);
|
|
3263
|
+
}
|
|
3264
|
+
|
|
3265
|
+
//#endregion
|
|
3266
|
+
//#region src/cli.ts
|
|
3267
|
+
/**
|
|
3268
|
+
* Substrate CLI dispatcher. The unified `substrate` binary dispatches on argv[2]:
|
|
3269
|
+
*
|
|
3270
|
+
* substrate serve — boot the HTTP server (./main.ts)
|
|
3271
|
+
* substrate mcp — boot the stdio MCP transport (./mcp/server.ts)
|
|
3272
|
+
* substrate init — scaffold a runtime directory (./cli/init)
|
|
3273
|
+
* substrate check-enums — report stored values outside declared enums
|
|
3274
|
+
* substrate normalise-vocab — rewrite stored vocabulary aliases to canonical values
|
|
3275
|
+
* substrate backfill-field — write per-row values of one field, metadata preserved
|
|
3276
|
+
* substrate ask — ask a natural-language question (./cli/ask)
|
|
3277
|
+
* substrate run — run a script against a signed, guarded client (./cli/run)
|
|
3278
|
+
* substrate federate — enroll / join / check bridges (./cli/federate)
|
|
3279
|
+
*
|
|
3280
|
+
* No flags-vs-subcommand ambiguity: subcommands are always argv[2].
|
|
3281
|
+
* Unknown / missing subcommand prints help and exits non-zero.
|
|
3282
|
+
*/
|
|
3283
|
+
const HELP = `substrate — the Substrate CLI
|
|
3284
|
+
|
|
3285
|
+
Usage:
|
|
3286
|
+
substrate <subcommand> [args...]
|
|
3287
|
+
substrate --version Print this binary's package version (also -v, version).
|
|
3288
|
+
substrate <subcommand> --help Print this help.
|
|
3289
|
+
|
|
3290
|
+
Subcommands:
|
|
3291
|
+
serve Boot the Substrate HTTP server (default port 3100).
|
|
3292
|
+
mcp Boot the stdio MCP server. Reads SUBSTRATE_URL, and
|
|
3293
|
+
SUBSTRATE_MCP_TOOLS (csv) to register only those tools.
|
|
3294
|
+
init [dir] [flags] Scaffold a new Substrate runtime directory. Interactive by
|
|
3295
|
+
default; pass --yes for non-interactive scripted use.
|
|
3296
|
+
federate <verb> Federation onboarding: enroll / join / check.
|
|
3297
|
+
Run \`substrate federate\` for the verb list.
|
|
3298
|
+
ask <question> Ask a natural-language question (needs the substrate-nl plugin).
|
|
3299
|
+
Flags: --ns, --project, --json. Reads SUBSTRATE_URL.
|
|
3300
|
+
run <script.mts> [--args <json>] [--dry-run | --write] [--max-writes <n>] [--out <file>]
|
|
3301
|
+
Run a script's default export (s) => result against a client
|
|
3302
|
+
signed from SUBSTRATE_URL + SUBSTRATE_AGENT_KEY_ID/_PATH, and
|
|
3303
|
+
print one JSON report. --args is a JSON object, read as
|
|
3304
|
+
s.args. Read-only unless --dry-run (records mutations) or
|
|
3305
|
+
--write (sends up to --max-writes frames, 50).
|
|
3306
|
+
check-enums [--json] [--record]
|
|
3307
|
+
Report stored values outside a field's declared enum.
|
|
3308
|
+
Read-only over HTTP (--record writes verdict rows); reads
|
|
3309
|
+
SUBSTRATE_URL and SUBSTRATE_ENUM_CHECK_TOKEN. Exit 1 on
|
|
3310
|
+
drift, so it works as a CI/cron gate.
|
|
3311
|
+
normalise-vocab <ns> <Type> <field> [--apply] [--reject a,b] [--json]
|
|
3312
|
+
Rewrite stored aliases to their canonical value. Dry run
|
|
3313
|
+
unless --apply. Admin: reads SUBSTRATE_URL and
|
|
3314
|
+
SUBSTRATE_ADMIN_TOKEN.
|
|
3315
|
+
backfill-field <ns> <Type> <field> --values <file.json> [--apply] [--overwrite] [--json]
|
|
3316
|
+
Write per-row values of one field, metadata preserved. Dry
|
|
3317
|
+
run unless --apply. Admin: reads SUBSTRATE_URL and
|
|
3318
|
+
SUBSTRATE_ADMIN_TOKEN.
|
|
3319
|
+
|
|
3320
|
+
Environment for \`serve\` (the load-bearing ones; the full reference is in the docs):
|
|
3321
|
+
SUBSTRATE_DATABASE_URL 'memory' (default) or 'postgres://...'
|
|
3322
|
+
SUBSTRATE_PLUGIN_DIR Abs path to plugin install root (manifest-driven loader)
|
|
3323
|
+
PORT Listen port (default 3100)
|
|
3324
|
+
HOST Listen address (default 0.0.0.0, or 127.0.0.1 when
|
|
3325
|
+
auth is off). A non-loopback HOST with auth off fails
|
|
3326
|
+
boot unless SUBSTRATE_ALLOW_UNAUTHENTICATED_HOST=1.
|
|
3327
|
+
SUBSTRATE_AUTH_MODE 'attribute' or 'required'. Setting this or
|
|
3328
|
+
SUBSTRATE_AUTH_KEY_REGISTRY enables auth; with
|
|
3329
|
+
neither, every request is anonymous.
|
|
3330
|
+
SUBSTRATE_CURSOR_SECRET Seals pagination cursors under row-filter policies, so
|
|
3331
|
+
every process sharing it opens the same cursors.
|
|
3332
|
+
Unset, the key is per process and a sealed cursor stops
|
|
3333
|
+
opening after a restart. SUBSTRATE_STREAM_TICKET_SECRET
|
|
3334
|
+
is read when this is unset.
|
|
3335
|
+
SUBSTRATE_PUBLIC_ORIGIN This server's origins, comma-separated (e.g.
|
|
3336
|
+
https://host:3100,https://alias). A token must carry
|
|
3337
|
+
'aud' naming one; a token with no 'aud' is refused.
|
|
3338
|
+
Required with auth on a non-loopback
|
|
3339
|
+
HOST; on loopback it defaults to the loopback origins.
|
|
3340
|
+
SUBSTRATE_TLS_KEY_PATH Serve HTTPS. Both are required together — setting one
|
|
3341
|
+
SUBSTRATE_TLS_CERT_PATH without the other refuses to boot.
|
|
3342
|
+
SUBSTRATE_NAMESPACE Default namespace for recall requests that omit one.
|
|
3343
|
+
Without it such requests are refused rather than
|
|
3344
|
+
guessing a namespace that may not exist.
|
|
3345
|
+
|
|
3346
|
+
Natural-language query and the recall embedder are configured by plugin config
|
|
3347
|
+
files under SUBSTRATE_PLUGIN_DIR (substrate-nl.config.json,
|
|
3348
|
+
substrate-recall.config.json), not by environment variables.
|
|
3349
|
+
|
|
3350
|
+
See https://github.com/mnmal-ai/substrate for the full env reference.
|
|
3351
|
+
`;
|
|
3352
|
+
async function runCli(argv) {
|
|
3353
|
+
const [subcommand, ...rest] = argv;
|
|
3354
|
+
if (subcommand === void 0 || subcommand === "--help" || subcommand === "-h") {
|
|
3355
|
+
process.stdout.write(HELP);
|
|
3356
|
+
return subcommand === void 0 ? 1 : 0;
|
|
3357
|
+
}
|
|
3358
|
+
if (subcommand === "--version" || subcommand === "-v" || subcommand === "version") {
|
|
3359
|
+
process.stdout.write(`${version}\n`);
|
|
3360
|
+
return 0;
|
|
3361
|
+
}
|
|
3362
|
+
if (subcommand !== "federate" && (rest.includes("--help") || rest.includes("-h"))) {
|
|
3363
|
+
process.stdout.write(HELP);
|
|
3364
|
+
return 0;
|
|
3365
|
+
}
|
|
3366
|
+
switch (subcommand) {
|
|
3367
|
+
case "serve":
|
|
3368
|
+
await runServe();
|
|
3369
|
+
await new Promise(() => {});
|
|
3370
|
+
return 0;
|
|
3371
|
+
case "mcp":
|
|
3372
|
+
await runMcp();
|
|
3373
|
+
await new Promise(() => {});
|
|
3374
|
+
return 0;
|
|
3375
|
+
case "init": return await runInit(rest);
|
|
3376
|
+
case "check-enums": return await runCheckEnums(rest);
|
|
3377
|
+
case "normalise-vocab": return await runNormaliseVocab(rest);
|
|
3378
|
+
case "backfill-field": return await runBackfillField(rest);
|
|
3379
|
+
case "ask": return await runAsk(rest);
|
|
3380
|
+
case "run": return await runRun(rest);
|
|
3381
|
+
case "federate": return await runFederate(rest, process.env, (s) => process.stdout.write(s), (s) => process.stderr.write(s));
|
|
3382
|
+
default:
|
|
3383
|
+
process.stderr.write(`substrate: unknown subcommand '${subcommand}'\n\n`);
|
|
3384
|
+
process.stderr.write(HELP);
|
|
3385
|
+
return 1;
|
|
3386
|
+
}
|
|
3387
|
+
}
|
|
3388
|
+
|
|
3389
|
+
//#endregion
|
|
3390
|
+
export { parsePositive as a, DEFAULT_SIZING as i, defaultPluginsForPersona as n, parsePort as r, runCli, KNOWN_PLUGINS as t };
|