@edgehero/pi-dispatch 4.1.0 → 4.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +6 -4
- package/package.json +4 -2
- package/src/azure-host.mjs +28 -1
- package/src/azure-prompt.mjs +26 -3
- package/src/config.mjs +2 -2
- package/src/doctor.mjs +185 -13
- package/src/egress-allowlist.mjs +81 -0
- package/src/env-allowlist.mjs +8 -1
- package/src/forgejo-host.mjs +33 -1
- package/src/forgejo-prompt.mjs +26 -3
- package/src/github-host.mjs +21 -1
- package/src/github-prompt.mjs +26 -3
- package/src/gitlab-host.mjs +24 -1
- package/src/gitlab-prompt.mjs +26 -3
- package/src/host-pi.mjs +8 -5
- package/src/image-preflight.mjs +35 -1
- package/src/import-pi.mjs +113 -8
- package/src/mcp-config.mjs +466 -0
- package/src/mcp-preflight.mjs +262 -0
- package/src/model-catalog.mjs +2 -0
- package/src/model-endpoints.mjs +8 -3
- package/src/models-json.mjs +65 -53
- package/src/outbox.mjs +7 -1
- package/src/parked.mjs +18 -0
- package/src/prepare.mjs +12 -4
- package/src/processor.mjs +81 -4
- package/src/provider-steering.mjs +1 -0
- package/src/queue.mjs +10 -2
- package/src/reserved-env.mjs +2 -0
- package/src/run-container.mjs +6 -0
- package/src/run-history.mjs +29 -2
- package/src/schedules.mjs +1 -1
- package/src/session-store.mjs +45 -2
- package/src/start.mjs +59 -1
- package/src/triggers-file.mjs +7 -0
- package/src/triggers.mjs +152 -7
|
@@ -0,0 +1,466 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The overlay's `mcp.json` (REQ-MCP-OPERATOR-SERVERS, INT-OVERLAY-MCP-CONFIG): the MCP servers the operator offers to
|
|
3
|
+
* jobs, in pi's own `mcpServers` shape, read on the host. A job connects only servers named here, and only those its
|
|
4
|
+
* trigger names (`run.mcp`); a repo's `.pi/mcp.json` is never read for a job.
|
|
5
|
+
*
|
|
6
|
+
* The file is mounted into an adversarial-input container, so it carries no credential. A credential is written as a
|
|
7
|
+
* `${NAME}` reference, NAME in capitals, digits and `_`, which the job fills in from the variables its trigger already
|
|
8
|
+
* passes (`run.secrets`, PI_FORWARD_ENV), and it may go wherever servers put one:
|
|
9
|
+
* - an env value is exactly `${NAME}`, and a header value is `${NAME}` or an auth scheme before it
|
|
10
|
+
* (`Bearer ${NAME}`; `MCP_HEADER_SCHEMES`): a VALUE there is never written in the overlay at all;
|
|
11
|
+
* - each value of a url's query is exactly `${NAME}`, for the same reason;
|
|
12
|
+
* - a url's path and an argument may hold references amid literal text (`/mcp/${KEY}/sse`, `--token=${T}`).
|
|
13
|
+
* A literal path segment or argument is written as is: nothing can tell a key from a file name there, and two rules
|
|
14
|
+
* that tried (credential-like header and env names, credential-like flags) were measured refusing `--max-tokens 4096`
|
|
15
|
+
* while passing `X-Goog-Key`, `GH_PAT` and `--api_key=sk-...`. So the place for a credential is a reference, and the
|
|
16
|
+
* docs say plainly that a literal argument or path is readable by every job. A url carries no fragment and no user
|
|
17
|
+
* name or password. pi's own resolver would run a value starting with `!` and expand bare `$NAME`; the job resolves
|
|
18
|
+
* the references itself, never pi.
|
|
19
|
+
*
|
|
20
|
+
* Accepted is a subset of pi 1.1.0's schema (core/mcp-servers.js `validateMcpServerConfig`), strict on keys: a key
|
|
21
|
+
* pi-dispatch does not know is refused, so a key a later pi adds is read before a job relies on it. Refused outright:
|
|
22
|
+
* `auth` and `oauth` (a credential flow the container cannot finish, and a token file it must not hold), `cwd` (a job
|
|
23
|
+
* runs its servers in the checkout), and the legacy `sse` type, which pi refuses too. `allowPackageRunner` is
|
|
24
|
+
* pi-dispatch's own: it stays in the overlay's file and is stripped (`piEntry`) before a job hands an entry to pi.
|
|
25
|
+
*
|
|
26
|
+
* Pure apart from the one read (`readOverlayMcp`), which judges the file the way `readOverlayModels` judges the
|
|
27
|
+
* overlay's `models.json`.
|
|
28
|
+
*
|
|
29
|
+
* The one server a job may connect that this file does not define is the forge's own (`forgeMcpServer`,
|
|
30
|
+
* DES-FORGE-MCP), whose name `forge` the overlay may therefore not use.
|
|
31
|
+
*/
|
|
32
|
+
import { lstatSync as fsLstatSync, readFileSync as fsReadFileSync } from "node:fs";
|
|
33
|
+
import { basename, isAbsolute, join, posix } from "node:path";
|
|
34
|
+
import { findDuplicateKey } from "./json-duplicates.mjs";
|
|
35
|
+
|
|
36
|
+
/** The file's name, in the overlay and in a host agent dir. */
|
|
37
|
+
export const MCP_CONFIG_FILE_NAME = "mcp.json";
|
|
38
|
+
/** The largest overlay `mcp.json` read, in bytes. A server list is a few hundred bytes; this is a bound, not a budget. */
|
|
39
|
+
export const MCP_CONFIG_MAX_BYTES = 64 * 1024;
|
|
40
|
+
/**
|
|
41
|
+
* A server name: lowercase words of letters and digits joined by single `-`, up to 32. Narrower than pi's, so no two
|
|
42
|
+
* names share pi's namespace: pi writes `-` as `_` in `mcp__<server>__<tool>`, and a trailing `-` or a `--` would let one
|
|
43
|
+
* server's tool name read as another's (`a-` with `x` and `a` with `_x` are both `mcp__a___x`).
|
|
44
|
+
*/
|
|
45
|
+
export const MCP_SERVER_NAME_RE = /^(?=[a-z0-9-]{1,32}$)[a-z0-9]+(?:-[a-z0-9]+)*$/;
|
|
46
|
+
/** The exposures pi 1.1.0 takes, by their current names (pi's alias `codemode-deferred` is refused, not translated). */
|
|
47
|
+
export const MCP_EXPOSURES = Object.freeze(["codemode", "deferred", "direct", "hidden"]);
|
|
48
|
+
/** Top-level keys. */
|
|
49
|
+
export const MCP_TOP_KEYS = Object.freeze(["autoEnableCodemode", "codemodeModels", "mcpServers"]);
|
|
50
|
+
/** Keys every server may carry. */
|
|
51
|
+
const COMMON_KEYS = ["type", "description", "exposure", "toolExposure", "timeout", "enabled"];
|
|
52
|
+
/** Keys of a streamable HTTP server. */
|
|
53
|
+
export const MCP_HTTP_KEYS = Object.freeze([...COMMON_KEYS, "url", "headers"]);
|
|
54
|
+
/** Keys of a stdio server. `allowPackageRunner` is pi-dispatch's own. */
|
|
55
|
+
export const MCP_STDIO_KEYS = Object.freeze([...COMMON_KEYS, "command", "args", "env", "allowPackageRunner"]);
|
|
56
|
+
/** pi keys refused with a reason, rather than as unknown. */
|
|
57
|
+
export const MCP_REFUSED_KEYS = Object.freeze({
|
|
58
|
+
auth: "it sends a pi login's token, which a job does not hold",
|
|
59
|
+
oauth: "OAuth needs a sign-in in a browser and a token file, and a job has neither",
|
|
60
|
+
cwd: "a job runs its servers in the checkout",
|
|
61
|
+
});
|
|
62
|
+
/**
|
|
63
|
+
* Commands that fetch the server they run from a package registry at start (`npx -y pkg`, `uvx pkg`). A job reaches a
|
|
64
|
+
* registry only when the operator allowlists it, and runs whatever version it finds there, so such a server is
|
|
65
|
+
* refused unless its entry says `allowPackageRunner: true`.
|
|
66
|
+
*/
|
|
67
|
+
export const MCP_PACKAGE_RUNNERS = Object.freeze(["npx", "npm", "pnpm", "pnpx", "yarn", "bunx", "bun", "uvx", "uv", "pipx"]);
|
|
68
|
+
/**
|
|
69
|
+
* The folders an absolute stdio command may sit under: the job image's system folders and the overlay. Named rather
|
|
70
|
+
* than excluded, since the job's root filesystem is writable and a list of folders the job writes (`/workspace`,
|
|
71
|
+
* `/tmp`, `/var/tmp`, `/dev/shm`...) always has a next one. A bare name is looked up on the image's PATH.
|
|
72
|
+
*/
|
|
73
|
+
export const MCP_COMMAND_ROOTS = Object.freeze(["/usr", "/bin", "/sbin", "/opt", "/app"]);
|
|
74
|
+
/** The auth schemes a header value may name before its reference, compared without case. */
|
|
75
|
+
export const MCP_HEADER_SCHEMES = Object.freeze(["Bearer", "Basic", "Token", "Bot", "ApiKey", "Key", "Splunk", "SSWS", "Digest"]);
|
|
76
|
+
/** Names a plain object would not keep as its own key. */
|
|
77
|
+
const RESERVED_NAMES = ["__proto__", "constructor", "prototype"];
|
|
78
|
+
/** A header name, as HTTP takes it (RFC 9110 token). */
|
|
79
|
+
const HEADER_NAME_RE = /^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/;
|
|
80
|
+
/** An env value. */
|
|
81
|
+
const ENV_REF_RE = /^\$\{([A-Z_][A-Z0-9_]*)\}$/;
|
|
82
|
+
/** A header value: a reference, optionally after an auth scheme and a space. */
|
|
83
|
+
const HEADER_REF_RE = new RegExp(`^(?:(?:${MCP_HEADER_SCHEMES.join("|")}) )?\\$\\{([A-Z_][A-Z0-9_]*)\\}$`, "i");
|
|
84
|
+
/** Any reference, as it is written. */
|
|
85
|
+
const REF_RE = /\$\{([A-Z_][A-Z0-9_]*)\}/g;
|
|
86
|
+
|
|
87
|
+
function isRecord(value) {
|
|
88
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* The name a header (`header: true`) or env value references, or `{ problem }`. Every such value is a reference.
|
|
93
|
+
*/
|
|
94
|
+
export function refValue(value, { header = false } = {}) {
|
|
95
|
+
if (typeof value !== "string") return { problem: "is not a string" };
|
|
96
|
+
const match = (header ? HEADER_REF_RE : ENV_REF_RE).exec(value);
|
|
97
|
+
if (match) return { name: match[1] };
|
|
98
|
+
const shape = header ? `\`\${NAME}\`, or an auth scheme and a space before it (\`Bearer \${NAME}\`; ${MCP_HEADER_SCHEMES.join(", ")})` : "exactly `${NAME}`";
|
|
99
|
+
return { problem: `must be ${shape}, NAME in capitals, digits and \`_\`: a value is never written in the overlay, which every job can read` };
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Why text that may hold references amid literal text (a url path, an argument) breaks the grammar, or null; `refs`
|
|
104
|
+
* collects the names. Every `${` opens a whole reference, so nothing reads one way here and another in the job.
|
|
105
|
+
*/
|
|
106
|
+
function mixedProblem(value, refs) {
|
|
107
|
+
if (value.startsWith("!")) return "starts with `!`";
|
|
108
|
+
const rest = value.replace(REF_RE, (_, name) => {
|
|
109
|
+
refs.add(name);
|
|
110
|
+
return "";
|
|
111
|
+
});
|
|
112
|
+
if (rest.includes("${")) return "has a `${` that is not a whole `${NAME}` reference (NAME in capitals, digits and `_`)";
|
|
113
|
+
return null;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** A name from the operator's file, quoted as JSON, so a control character in it is shown and never sent. */
|
|
117
|
+
const shown = (name) => JSON.stringify(name);
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* The server, checked and normalised (`{ name, transport, ...pi keys }`), or `{ name, problem }`. `transport` is
|
|
121
|
+
* "http" or "stdio"; `allowPackageRunner` is kept on the normalised entry for pi-dispatch and stripped by `piEntry`.
|
|
122
|
+
*/
|
|
123
|
+
function checkServer(name, raw) {
|
|
124
|
+
const refuse = (problem) => ({ name, problem });
|
|
125
|
+
if (!MCP_SERVER_NAME_RE.test(name)) return refuse("the name must be lowercase letters and digits, words joined by single `-`, up to 32");
|
|
126
|
+
if (name.includes("dispatch")) return refuse("a name holding `dispatch` is reserved for the dispatch admin");
|
|
127
|
+
// Refused rather than shadowed: an overlay `forge` that won over the worker's would hand a github job's token to a
|
|
128
|
+
// server the operator wrote, and one that lost would be an entry that reads as offered and is never used.
|
|
129
|
+
if (name === FORGE_MCP_SERVER_NAME) return refuse("`forge` is reserved for the worker's own forge server (the job's GitHub or GitLab, as its own token), which a trigger names in run.mcp without an overlay entry");
|
|
130
|
+
if (!isRecord(raw)) return refuse("the entry is not an object");
|
|
131
|
+
for (const key of Object.keys(raw)) {
|
|
132
|
+
if (Object.hasOwn(MCP_REFUSED_KEYS, key)) return refuse(`\`${key}\` is not accepted: ${MCP_REFUSED_KEYS[key]}`);
|
|
133
|
+
}
|
|
134
|
+
const { type } = raw;
|
|
135
|
+
if (type === "sse") return refuse("the legacy SSE transport is not supported; use the server's streamable HTTP URL");
|
|
136
|
+
if (type !== undefined && !["http", "streamable-http", "stdio"].includes(type)) return refuse("`type` must be http, streamable-http or stdio");
|
|
137
|
+
const hasUrl = raw.url !== undefined;
|
|
138
|
+
const hasCommand = raw.command !== undefined;
|
|
139
|
+
if (hasUrl === hasCommand) return refuse("the entry needs exactly one of `url` (streamable HTTP) or `command` (stdio)");
|
|
140
|
+
const transport = hasUrl ? "http" : "stdio";
|
|
141
|
+
if (type !== undefined && (type === "stdio") !== (transport === "stdio")) return refuse(`\`type\` ${type} does not match a server with \`${hasUrl ? "url" : "command"}\``);
|
|
142
|
+
const allowed = transport === "http" ? MCP_HTTP_KEYS : MCP_STDIO_KEYS;
|
|
143
|
+
for (const key of Object.keys(raw)) {
|
|
144
|
+
if (!allowed.includes(key)) return refuse(`${shown(key)} is not a key a ${transport === "http" ? "streamable HTTP" : "stdio"} server takes here`);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
if (typeof raw.description !== "string" || raw.description.trim() === "") return refuse("`description` is required: one sentence on what the server offers, which pi puts in the job's prompt");
|
|
148
|
+
if (raw.description.length > 300 || /[\p{Cc}\p{Cf}\p{Zl}\p{Zp}]/u.test(raw.description)) return refuse("`description` must be one line of at most 300 characters, with no control or format characters");
|
|
149
|
+
if (raw.exposure !== undefined && !MCP_EXPOSURES.includes(raw.exposure)) return refuse(`\`exposure\` must be one of ${MCP_EXPOSURES.join(", ")}`);
|
|
150
|
+
if (raw.toolExposure !== undefined) {
|
|
151
|
+
if (!isRecord(raw.toolExposure)) return refuse("`toolExposure` must map tool names or `*` patterns to exposures");
|
|
152
|
+
for (const [tool, exposure] of Object.entries(raw.toolExposure)) {
|
|
153
|
+
if (!MCP_EXPOSURES.includes(exposure)) return refuse(`\`toolExposure\` ${shown(tool)} must be one of ${MCP_EXPOSURES.join(", ")}`);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
if (raw.enabled !== undefined && typeof raw.enabled !== "boolean") return refuse("`enabled` must be true or false");
|
|
157
|
+
if (raw.timeout !== undefined && !(Number.isInteger(raw.timeout) && raw.timeout > 0 && raw.timeout <= 600)) return refuse("`timeout` must be a whole number of seconds from 1 to 600");
|
|
158
|
+
|
|
159
|
+
const entry = { name, transport };
|
|
160
|
+
for (const key of ["description", "exposure", "toolExposure", "timeout", "enabled"]) if (raw[key] !== undefined) entry[key] = raw[key];
|
|
161
|
+
const refs = new Set();
|
|
162
|
+
|
|
163
|
+
if (transport === "http") {
|
|
164
|
+
if (typeof raw.url !== "string") return refuse("`url` must be a string");
|
|
165
|
+
if (raw.url.includes("#")) return refuse("`url` must not carry a fragment");
|
|
166
|
+
const urlProblem = mixedProblem(raw.url, refs);
|
|
167
|
+
if (urlProblem) return refuse(`\`url\` ${urlProblem}`);
|
|
168
|
+
// Read as a URL parser reads it, never by cutting the text: the parser also takes `https:/h`, a backslash and a
|
|
169
|
+
// tab as the start of the host. Parsed whole, with two different words standing in for each reference, the url
|
|
170
|
+
// must be written exactly as the parser writes it back (one spelling, no guessing at the others), and its host
|
|
171
|
+
// must come out the same both times, so no reference is ever part of it.
|
|
172
|
+
const stand = (word) => raw.url.replace(REF_RE, word);
|
|
173
|
+
if (!URL.canParse(stand("refa"))) return refuse("`url` is not a URL");
|
|
174
|
+
const url = new URL(stand("refa"));
|
|
175
|
+
if (url.host !== new URL(stand("refb")).host) return refuse("`url` holds a reference in its host; the host is written as is, since the egress check reads it before the job starts");
|
|
176
|
+
if (url.href !== stand("refa") && url.href !== `${stand("refa")}/`) return refuse(`\`url\` must be written in its plain form, as a URL parser writes it back (${JSON.stringify(url.href.replaceAll("refa", "${...}"))})`);
|
|
177
|
+
if (url.protocol !== "https:" && url.protocol !== "http:") return refuse("`url` must be http or https");
|
|
178
|
+
if (url.username !== "" || url.password !== "") return refuse("`url` must not carry a user name or password; send a credential in a header, as a `${NAME}` reference");
|
|
179
|
+
const query = raw.url.includes("?") ? raw.url.slice(raw.url.indexOf("?") + 1) : undefined;
|
|
180
|
+
if (query !== undefined) {
|
|
181
|
+
for (const pair of query.split("&")) {
|
|
182
|
+
const eq = pair.indexOf("=");
|
|
183
|
+
const value = eq === -1 ? "" : pair.slice(eq + 1);
|
|
184
|
+
if (!ENV_REF_RE.test(value)) return refuse(`\`url\` query value ${shown(eq === -1 ? pair : pair.slice(0, eq))} must be exactly \`\${NAME}\`: a query is where hosted servers put a key, so no value there is written in the overlay`);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
entry.url = raw.url;
|
|
188
|
+
if (raw.headers !== undefined) {
|
|
189
|
+
if (!isRecord(raw.headers)) return refuse("`headers` must map header names to strings");
|
|
190
|
+
for (const [header, value] of Object.entries(raw.headers)) {
|
|
191
|
+
if (!HEADER_NAME_RE.test(header) || RESERVED_NAMES.includes(header)) return refuse(`header name ${shown(header)} is not a valid HTTP header name`);
|
|
192
|
+
const ref = refValue(value, { header: true });
|
|
193
|
+
if (ref.problem) return refuse(`header ${shown(header)} ${ref.problem}`);
|
|
194
|
+
refs.add(ref.name);
|
|
195
|
+
}
|
|
196
|
+
entry.headers = { ...raw.headers };
|
|
197
|
+
}
|
|
198
|
+
} else {
|
|
199
|
+
if (typeof raw.command !== "string" || raw.command === "") return refuse("`command` must be a non-empty string");
|
|
200
|
+
if (raw.command.startsWith("!")) return refuse("`command` starts with `!`");
|
|
201
|
+
if (raw.command.includes("$")) return refuse("`command` holds a `$`; it is used as written, and a reference goes in an argument or `env`");
|
|
202
|
+
if (raw.command.startsWith("~")) return refuse("`command` must not start with `~`, which pi reads as the job's home folder");
|
|
203
|
+
if (!isAbsolute(raw.command) && raw.command.includes("/")) return refuse("`command` must be an absolute path or a bare name found on the job image's PATH");
|
|
204
|
+
if (isAbsolute(raw.command)) {
|
|
205
|
+
const normalised = posix.normalize(raw.command);
|
|
206
|
+
if (!MCP_COMMAND_ROOTS.some((r) => normalised.startsWith(`${r}/`))) return refuse(`\`command\` must sit under ${MCP_COMMAND_ROOTS.join(", ")} (the job image's own folders and the overlay at /opt/pi-global); the job writes the others`);
|
|
207
|
+
}
|
|
208
|
+
if (raw.allowPackageRunner !== undefined && typeof raw.allowPackageRunner !== "boolean") return refuse("`allowPackageRunner` must be true or false");
|
|
209
|
+
const runner = MCP_PACKAGE_RUNNERS.includes(basename(raw.command));
|
|
210
|
+
if (runner && raw.allowPackageRunner !== true) return refuse(`\`command\` ${basename(raw.command)} fetches the server from a package registry at start; install it in the job image, or set \`allowPackageRunner: true\` and allowlist the registry`);
|
|
211
|
+
entry.command = raw.command;
|
|
212
|
+
if (raw.args !== undefined) {
|
|
213
|
+
if (!Array.isArray(raw.args) || !raw.args.every((arg) => typeof arg === "string")) return refuse("`args` must be a list of strings");
|
|
214
|
+
for (const arg of raw.args) {
|
|
215
|
+
const problem = mixedProblem(arg, refs);
|
|
216
|
+
if (problem) return refuse(`an argument ${problem}`);
|
|
217
|
+
if (arg.startsWith("~")) return refuse("an argument starts with `~`, which pi reads as the job's home folder, a folder the job writes");
|
|
218
|
+
}
|
|
219
|
+
entry.args = [...raw.args];
|
|
220
|
+
}
|
|
221
|
+
if (raw.env !== undefined) {
|
|
222
|
+
if (!isRecord(raw.env)) return refuse("`env` must map variable names to strings");
|
|
223
|
+
for (const [key, value] of Object.entries(raw.env)) {
|
|
224
|
+
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(key) || RESERVED_NAMES.includes(key)) return refuse(`env name ${shown(key)} is not a variable name`);
|
|
225
|
+
const ref = refValue(value);
|
|
226
|
+
if (ref.problem) return refuse(`env ${key} ${ref.problem}`);
|
|
227
|
+
refs.add(ref.name);
|
|
228
|
+
}
|
|
229
|
+
entry.env = { ...raw.env };
|
|
230
|
+
}
|
|
231
|
+
entry.packageRunner = runner;
|
|
232
|
+
}
|
|
233
|
+
entry.refs = [...refs].sort();
|
|
234
|
+
return entry;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* Parse an overlay `mcp.json`. Throws an Error (marked `mcpConfig`) for a file that cannot be read as a whole: not
|
|
239
|
+
* JSON, a duplicate key, not an object, an unknown top-level key. Returns `{ autoEnableCodemode, servers, refused }`:
|
|
240
|
+
* `servers` the accepted entries by name, `refused` `[{ name, problem }]` for each entry that is not accepted. A
|
|
241
|
+
* refused entry is never offered to a job; doctor names it, and import-pi leaves it behind.
|
|
242
|
+
*/
|
|
243
|
+
export function parseOverlayMcp(text, path = MCP_CONFIG_FILE_NAME) {
|
|
244
|
+
const fail = (why) => {
|
|
245
|
+
const error = new Error(`${path}: ${why}`);
|
|
246
|
+
error.mcpConfig = true;
|
|
247
|
+
return error;
|
|
248
|
+
};
|
|
249
|
+
let value;
|
|
250
|
+
try {
|
|
251
|
+
value = JSON.parse(text);
|
|
252
|
+
} catch {
|
|
253
|
+
// Never the parser's message: V8 quotes the text around the error, which can be a header's value.
|
|
254
|
+
throw fail("not valid JSON");
|
|
255
|
+
}
|
|
256
|
+
const duplicate = findDuplicateKey(text);
|
|
257
|
+
if (duplicate) throw fail(`the key ${JSON.stringify(duplicate.at)} appears twice; JSON keeps only the last, so the file would not say what it reads as`);
|
|
258
|
+
if (!isRecord(value)) throw fail("expected an object with an `mcpServers` object");
|
|
259
|
+
for (const key of Object.keys(value)) {
|
|
260
|
+
if (!MCP_TOP_KEYS.includes(key)) throw fail(`${shown(key)} is not a top-level key here (accepted: ${MCP_TOP_KEYS.join(", ")})`);
|
|
261
|
+
}
|
|
262
|
+
if (value.autoEnableCodemode !== undefined && typeof value.autoEnableCodemode !== "boolean") throw fail("`autoEnableCodemode` must be true or false");
|
|
263
|
+
// pi-dispatch's own (pi's loader ignores a top-level key it does not know): whether a job's codemode scripts may call
|
|
264
|
+
// `models.classify()` and `models.generateImages()`. On by default, as in the pi CLI; every such call is metered,
|
|
265
|
+
// held to the job's model list, and refused under a cost cap when it cannot be bounded (DES-MCP-CODEMODE-MODELS).
|
|
266
|
+
if (value.codemodeModels !== undefined && typeof value.codemodeModels !== "boolean") throw fail("`codemodeModels` must be true or false");
|
|
267
|
+
if (value.mcpServers !== undefined && !isRecord(value.mcpServers)) throw fail("`mcpServers` must be an object of servers by name");
|
|
268
|
+
const servers = {};
|
|
269
|
+
const refused = [];
|
|
270
|
+
for (const [name, raw] of Object.entries(value.mcpServers ?? {})) {
|
|
271
|
+
const entry = checkServer(name, raw);
|
|
272
|
+
if (entry.problem) refused.push(entry);
|
|
273
|
+
else servers[name] = entry;
|
|
274
|
+
}
|
|
275
|
+
return { autoEnableCodemode: value.autoEnableCodemode ?? true, codemodeModels: value.codemodeModels ?? true, servers, refused };
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/**
|
|
279
|
+
* The forge's own MCP server (DES-FORGE-MCP): the one server a job may connect that the overlay does not define. A
|
|
280
|
+
* github or gitlab trigger names it in `run.mcp` as `forge`, and the worker writes its entry here, so the server, its
|
|
281
|
+
* pin (baked in the job image) and the token it is handed are this repository's, never a trigger's or the overlay's.
|
|
282
|
+
* The token is the one the job already holds for its CLI (GH_TOKEN, GITLAB_TOKEN; env-allowlist.mjs
|
|
283
|
+
* `buildContainerEnv`): no new credential, and no power the job's `gh` or `glab` does not already have.
|
|
284
|
+
*/
|
|
285
|
+
export const FORGE_MCP_SERVER_NAME = "forge";
|
|
286
|
+
/**
|
|
287
|
+
* The job kinds that may name it. Forgejo and Azure DevOps have no forge MCP server this repository has verified, and a
|
|
288
|
+
* cron job holds no forge token of its own (a `run.github` token is not a forge job's).
|
|
289
|
+
*/
|
|
290
|
+
export const FORGE_MCP_KINDS = Object.freeze(["github", "gitlab"]);
|
|
291
|
+
/**
|
|
292
|
+
* The commands a `forge` entry runs: never a server itself, but the image's wrapper around it
|
|
293
|
+
* (image/forge-mcp/forge-mcp-github.sh, forge-mcp-gitlab.sh), which withholds the server's merge and approval tools
|
|
294
|
+
* (CONST-MERGE-NEVER-AUTOMATIC) from every list and refuses every call to them. The withheld names live in the image,
|
|
295
|
+
* beside the pin they were read against, and never here. github-mcp-server is baked from its release tarball at an
|
|
296
|
+
* exact version and per-architecture sha256, @zereight/mcp-gitlab from a committed lockfile (image/Dockerfile).
|
|
297
|
+
*/
|
|
298
|
+
export const GITHUB_MCP_COMMAND = "/usr/local/bin/forge-mcp-github";
|
|
299
|
+
export const GITLAB_MCP_COMMAND = "/usr/local/bin/forge-mcp-gitlab";
|
|
300
|
+
/** The host github-mcp-server calls for every tool (REST and GraphQL), which the egress allowlist must hold. */
|
|
301
|
+
export const GITHUB_MCP_API_HOST = "api.github.com";
|
|
302
|
+
|
|
303
|
+
/**
|
|
304
|
+
* The REST root of a GitLab instance URL as the worker is configured with it (`GITLAB_URL`): its path kept, trailing
|
|
305
|
+
* `/` dropped, and `/api/v4` added once (a URL already ending in it is not given a second).
|
|
306
|
+
*/
|
|
307
|
+
export function gitlabApiV4(gitlabUrl) {
|
|
308
|
+
return `${String(gitlabUrl).trim().replace(/\/+$/, "").replace(/\/api\/v4$/i, "").replace(/\/+$/, "")}/api/v4`;
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/**
|
|
312
|
+
* Why `GITLAB_URL` cannot be written into the forge entry's GITLAB_API_URL, or null. mcp-gitlab splits that value on
|
|
313
|
+
* `,` into several instances, a query or a fragment would end up inside every path it builds, and a user name or
|
|
314
|
+
* password would be a credential written in the job's `/job/mcp.json`; the worker's own clients tolerate some of these,
|
|
315
|
+
* so the forge entry asks for itself rather than trusting that they were refused at load.
|
|
316
|
+
*/
|
|
317
|
+
export function gitlabUrlProblem(gitlabUrl) {
|
|
318
|
+
// Surrounding whitespace is not refused: the URL parser drops it, as it does for the worker's own clients, and
|
|
319
|
+
// gitlabApiV4 trims it, so a quoted .env value that works for them works here.
|
|
320
|
+
if (typeof gitlabUrl !== "string" || gitlabUrl.trim() === "") return "it is not set";
|
|
321
|
+
if (gitlabUrl.includes(",")) return "it holds a comma, which the server reads as a list of instances";
|
|
322
|
+
if (!URL.canParse(gitlabUrl)) return "it is not a URL";
|
|
323
|
+
const url = new URL(gitlabUrl);
|
|
324
|
+
if (url.protocol !== "https:" && url.protocol !== "http:") return "it is not http or https";
|
|
325
|
+
if (url.username !== "" || url.password !== "") return "it carries a user name or password";
|
|
326
|
+
if (gitlabUrl.includes("?") || gitlabUrl.includes("#")) return "it carries a query or a fragment";
|
|
327
|
+
return null;
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
/**
|
|
331
|
+
* The forge server's entry for a job of `kind`, in the normalised shape `checkServer` returns, or null for a kind that
|
|
332
|
+
* has none. `gitlabUrl` is the worker's configured GitLab URL (config.gitlab.apiUrl), written into the entry as a
|
|
333
|
+
* literal: it is the deployment's, not a secret, and the server would otherwise default to gitlab.com.
|
|
334
|
+
*
|
|
335
|
+
* github-mcp-server 2.0.2 reads its settings through viper with the `GITHUB_` prefix (cmd/github-mcp-server/main.go):
|
|
336
|
+
* GITHUB_PERSONAL_ACCESS_TOKEN is the token; GITHUB_TOOLSETS `default` is context, repos, issues, pull_requests and
|
|
337
|
+
* users, so `actions` (workflow runs and triggers) stays off; GITHUB_LOCKDOWN_MODE `1` (strconv.ParseBool) withholds
|
|
338
|
+
* content on a public repository from authors without push access, a best-effort filter and not an authorization
|
|
339
|
+
* boundary. With an EMPTY token the release binary falls back to its baked-in OAuth client and a browser login with a
|
|
340
|
+
* local callback listener; the runner refuses a reference the container holds empty (`mcp-secret-missing`) before
|
|
341
|
+
* the server starts, so that path is never reached.
|
|
342
|
+
*
|
|
343
|
+
* @zereight/mcp-gitlab 2.1.67 (build/config.js, build/index.js): GITLAB_PERSONAL_ACCESS_TOKEN is sent as a Bearer
|
|
344
|
+
* token; GITLAB_API_URL is the REST root; GITLAB_DISABLE_VERSION_CHECK must be exactly `true`, or it asks
|
|
345
|
+
* registry.npmjs.org for a newer release at every start. It starts an HTTP listener only on SSE or STREAMABLE_HTTP
|
|
346
|
+
* `true` and an OAuth flow only on GITLAB_USE_OAUTH `true`; a stdio server starts with only its own `env` and the
|
|
347
|
+
* runner's base names (`inheritEnv: false`, image/runner/src/mcp.mjs), so none of them can reach it.
|
|
348
|
+
*/
|
|
349
|
+
export function forgeMcpServer(kind, { gitlabUrl = "https://gitlab.com" } = {}) {
|
|
350
|
+
if (kind === "github") {
|
|
351
|
+
return {
|
|
352
|
+
name: FORGE_MCP_SERVER_NAME,
|
|
353
|
+
transport: "stdio",
|
|
354
|
+
description: "This job's GitHub repository host, through GitHub's own MCP server, as the job's own token: issues, pull requests, repositories and code.",
|
|
355
|
+
command: GITHUB_MCP_COMMAND,
|
|
356
|
+
env: { GITHUB_PERSONAL_ACCESS_TOKEN: "${GH_TOKEN}", GITHUB_LOCKDOWN_MODE: "1", GITHUB_TOOLSETS: "default" },
|
|
357
|
+
packageRunner: false,
|
|
358
|
+
refs: ["GH_TOKEN"],
|
|
359
|
+
};
|
|
360
|
+
}
|
|
361
|
+
if (kind === "gitlab") {
|
|
362
|
+
return {
|
|
363
|
+
name: FORGE_MCP_SERVER_NAME,
|
|
364
|
+
transport: "stdio",
|
|
365
|
+
description: "This job's GitLab instance, through a GitLab MCP server, as the job's own token: issues, merge requests, repositories and code.",
|
|
366
|
+
command: GITLAB_MCP_COMMAND,
|
|
367
|
+
env: { GITLAB_PERSONAL_ACCESS_TOKEN: "${GITLAB_TOKEN}", GITLAB_API_URL: gitlabApiV4(gitlabUrl), GITLAB_DISABLE_VERSION_CHECK: "true" },
|
|
368
|
+
packageRunner: false,
|
|
369
|
+
refs: ["GITLAB_TOKEN"],
|
|
370
|
+
};
|
|
371
|
+
}
|
|
372
|
+
return null;
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
/** The URL whose host and port the forge server calls, for the egress check: api.github.com, or the GitLab instance. */
|
|
376
|
+
export function forgeMcpEgressUrl(kind, { gitlabUrl = "https://gitlab.com" } = {}) {
|
|
377
|
+
return kind === "github" ? `https://${GITHUB_MCP_API_HOST}/` : gitlabApiV4(gitlabUrl);
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
/** The entry as pi takes it: pi's keys only, with `type` written out. */
|
|
381
|
+
export function piEntry(server) {
|
|
382
|
+
const { name, transport, refs, packageRunner, ...rest } = server;
|
|
383
|
+
return { type: transport === "http" ? "http" : "stdio", ...rest };
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
/** The per-job document's name in the job's folder, read in the container as `/job/mcp.json`. */
|
|
387
|
+
export const JOB_MCP_FILE = "mcp.json";
|
|
388
|
+
/** The per-job document's version: the runner refuses one it does not know. */
|
|
389
|
+
export const JOB_MCP_VERSION = 1;
|
|
390
|
+
|
|
391
|
+
/**
|
|
392
|
+
* The per-job document the worker writes to `/job/mcp.json` (INT-CONTAINER-JOB-INPUTS) for the servers a job's
|
|
393
|
+
* trigger names, in that order: each in pi's shape with every reference still a reference, `toolExposure` gaining
|
|
394
|
+
* `"dispatch_*": "hidden"` (an MCP tool by that name is never offered, the admin's tools being the one recursion
|
|
395
|
+
* vector), and the names each server references, so the runner resolves those and no other. Never a value: the
|
|
396
|
+
* worker holds the trigger's resolved secrets, and none of them reaches this file. `codemodeModels` is the overlay's
|
|
397
|
+
* (on unless it says false): codemode's `models.*` reach the runner's meter and guards like any model call
|
|
398
|
+
* (image/runner/test/codemode-models.integration.test.mjs).
|
|
399
|
+
*/
|
|
400
|
+
export function jobMcpDocument(parsed, names) {
|
|
401
|
+
const servers = {};
|
|
402
|
+
const refs = {};
|
|
403
|
+
for (const name of names) {
|
|
404
|
+
const server = parsed.servers[name];
|
|
405
|
+
const entry = piEntry(server);
|
|
406
|
+
// pi takes an exact tool name before any pattern, and the patterns in order (core/mcp-servers.js
|
|
407
|
+
// `getMcpToolExposure`), so the hide goes FIRST and a key of the operator's that names only such tools (an exact
|
|
408
|
+
// `dispatch_` name, which would beat the pattern, or a narrower pattern) goes. Any other key stays as written: a
|
|
409
|
+
// `dispatch*` pattern still governs `dispatcher_run`, and the hide, being first, still governs `dispatch_run`.
|
|
410
|
+
const kept = Object.entries(entry.toolExposure ?? {}).filter(([key]) => !key.startsWith("dispatch_"));
|
|
411
|
+
entry.toolExposure = Object.fromEntries([["dispatch_*", "hidden"], ...kept]);
|
|
412
|
+
servers[name] = entry;
|
|
413
|
+
refs[name] = [...server.refs];
|
|
414
|
+
}
|
|
415
|
+
return { version: JOB_MCP_VERSION, autoEnableCodemode: parsed.autoEnableCodemode, codemodeModels: parsed.codemodeModels !== false, servers, refs };
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
/** The text of an overlay `mcp.json` holding exactly `parsed`'s accepted servers, as import-pi writes it. */
|
|
419
|
+
export function overlayMcpText(parsed) {
|
|
420
|
+
const mcpServers = {};
|
|
421
|
+
for (const [name, server] of Object.entries(parsed.servers)) {
|
|
422
|
+
const { type, ...entry } = piEntry(server);
|
|
423
|
+
if (server.packageRunner) entry.allowPackageRunner = true;
|
|
424
|
+
mcpServers[name] = server.transport === "http" ? { type, ...entry } : entry;
|
|
425
|
+
}
|
|
426
|
+
const out = { mcpServers };
|
|
427
|
+
if (parsed.autoEnableCodemode === false) out.autoEnableCodemode = false;
|
|
428
|
+
if (parsed.codemodeModels === false) out.codemodeModels = false;
|
|
429
|
+
return `${JSON.stringify(out, null, 2)}\n`;
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
/** Where a server is: its URL's host and port, or its command's base name. Never a header, an env value or an argument. */
|
|
433
|
+
export function mcpServerWhere(server) {
|
|
434
|
+
if (server.transport === "http") return URL.canParse(server.url) ? new URL(server.url).host : "an unreadable url";
|
|
435
|
+
return basename(server.command);
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
const ABSENT_CODES = new Set(["ENOENT", "ELOOP", "ENOTDIR", "ENAMETOOLONG"]);
|
|
439
|
+
|
|
440
|
+
/**
|
|
441
|
+
* The overlay's `mcp.json` (`<globalPiDir>/mcp.json`), parsed: `null` when there is none, the `parseOverlayMcp` result
|
|
442
|
+
* otherwise. Judged as the overlay readers beside it judge `models.json` (`readOverlayModels`): a link of any kind, a
|
|
443
|
+
* named pipe, socket or device, and a file over `MCP_CONFIG_MAX_BYTES` are refused (marked `mcpConfig`); any other fs
|
|
444
|
+
* error is rethrown with its code.
|
|
445
|
+
*/
|
|
446
|
+
export function readOverlayMcp(globalPiDir, { readFileSync = fsReadFileSync, lstatSync = fsLstatSync } = {}) {
|
|
447
|
+
if (typeof globalPiDir !== "string" || globalPiDir === "") return null;
|
|
448
|
+
const path = join(globalPiDir, MCP_CONFIG_FILE_NAME);
|
|
449
|
+
const fail = (why) => {
|
|
450
|
+
const error = new Error(`${path}: ${why}`);
|
|
451
|
+
error.mcpConfig = true;
|
|
452
|
+
return error;
|
|
453
|
+
};
|
|
454
|
+
let text;
|
|
455
|
+
try {
|
|
456
|
+
const entry = lstatSync(path);
|
|
457
|
+
if (entry.isSymbolicLink()) throw fail("is a link; replace it with the file itself (the job's read-only mount cannot follow links reliably)");
|
|
458
|
+
if (!entry.isFile()) throw fail("is not a regular file; replace it with the file itself");
|
|
459
|
+
if (entry.size > MCP_CONFIG_MAX_BYTES) throw fail(`is larger than ${MCP_CONFIG_MAX_BYTES} bytes`);
|
|
460
|
+
text = String(readFileSync(path, "utf8"));
|
|
461
|
+
} catch (error) {
|
|
462
|
+
if (ABSENT_CODES.has(error?.code)) return null;
|
|
463
|
+
throw error;
|
|
464
|
+
}
|
|
465
|
+
return parseOverlayMcp(text, path);
|
|
466
|
+
}
|