@edgehero/pi-dispatch 1.9.0 → 1.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +9 -0
- package/package.json +7 -2
- package/src/backend-conformance.mjs +262 -0
- package/src/backend-local.mjs +222 -0
- package/src/backend-registry.mjs +157 -0
- package/src/backends.mjs +595 -0
- package/src/config.mjs +63 -1
- package/src/container-spec.mjs +226 -0
- package/src/docker-run.mjs +100 -106
- package/src/doctor.mjs +118 -0
- package/src/index.mjs +136 -12
- package/src/outbox.mjs +8 -0
- package/src/packages.mjs +7 -4
- package/src/processor.mjs +48 -11
- package/src/queue.mjs +14 -2
- package/src/sandbox-cli.mjs +36 -0
- package/src/schedules.mjs +1 -1
- package/src/start.mjs +118 -80
- package/src/triggers.mjs +120 -5
package/src/config.mjs
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
import { existsSync } from "node:fs";
|
|
9
9
|
import { hostname } from "node:os";
|
|
10
10
|
import { delimiter } from "node:path";
|
|
11
|
+
import { DEFAULT_BACKEND, backendRefusals, parseBackendFloor, parseBackendList } from "./backends.mjs";
|
|
11
12
|
import { DEFAULT_EGRESS_PROXY, egressArmed } from "./egress.mjs";
|
|
12
13
|
import { MINTED_TOKEN_VARS } from "./forges.mjs";
|
|
13
14
|
import { parseSecretProfiles } from "./secret-profiles.mjs";
|
|
@@ -158,6 +159,47 @@ function egressEnabled(env) {
|
|
|
158
159
|
}
|
|
159
160
|
}
|
|
160
161
|
|
|
162
|
+
/**
|
|
163
|
+
* PI_BACKENDS and PI_BACKEND_FLOOR (issue #227). Both parse in `backends.mjs` for `egressEnabled`'s reason
|
|
164
|
+
* directly above: `doctor` reads the environment itself, and three copies of one grammar is two chances to
|
|
165
|
+
* disagree about what an operator's floor says. Here they gain the `piDispatchConfig` tag only.
|
|
166
|
+
*
|
|
167
|
+
* ENV-ONLY, never the overlay and never the deployment pointer, on `secretResolverRoots`' rule: a bound
|
|
168
|
+
* that can be widened from the surface it bounds is not a bound. The pointer needs no edit to enforce it --
|
|
169
|
+
* `POINTER_ENV_ALLOWLIST` is an ALLOWLIST (of the path and URL variables `resolvePaths` reads), so a
|
|
170
|
+
* name absent from it is refused by omission, and a capability grant is never added to it.
|
|
171
|
+
*/
|
|
172
|
+
function backendSet(env) {
|
|
173
|
+
try {
|
|
174
|
+
return parseBackendList(env.PI_BACKENDS);
|
|
175
|
+
} catch (error) {
|
|
176
|
+
throw configError(error.message);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
function backendFloorOf(env) {
|
|
181
|
+
try {
|
|
182
|
+
return parseBackendFloor(env.PI_BACKEND_FLOOR);
|
|
183
|
+
} catch (error) {
|
|
184
|
+
throw configError(error.message);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* THE BOOT REFUSAL. A deployment whose configuration needs something its backends cannot provide is refused
|
|
190
|
+
* here, named backend by named property, rather than discovering it per job.
|
|
191
|
+
*
|
|
192
|
+
* The RULES live in `backends.mjs` as a pure function over an explicit backend list; this only re-tags them
|
|
193
|
+
* so the CLI prints them cleanly, which is `egressEnabled`'s arrangement. That split is not tidiness: a
|
|
194
|
+
* rule reachable only through `loadConfig` can only be exercised with backend names `parseBackendList`
|
|
195
|
+
* accepts, and there is exactly one today, so three of these rules survived a mutation pass unprotected
|
|
196
|
+
* while living here.
|
|
197
|
+
*/
|
|
198
|
+
function refuseBackendShortfall(config) {
|
|
199
|
+
const [first] = backendRefusals(config);
|
|
200
|
+
if (first) throw configError(first);
|
|
201
|
+
}
|
|
202
|
+
|
|
161
203
|
// The operator's global pi overlay dir (REQ-GLOBAL-PI-OVERLAY). Unset/empty = feature off. When set it
|
|
162
204
|
// must EXIST at boot -- a typo pointing at nothing would silently drop the operator's whole setup on
|
|
163
205
|
// every job, so fail loud like every other config error rather than degrade to nothing.
|
|
@@ -211,7 +253,12 @@ export function globalExtensionsEnabled(env) {
|
|
|
211
253
|
*/
|
|
212
254
|
export function loadConfig(env = process.env, { fileExists = existsSync } = {}) {
|
|
213
255
|
const model = env.PI_MODEL ?? "claude-sonnet-4-5-20250929"; // dated snapshot; deterministic per CONST-PI-VERSION-PINNED
|
|
214
|
-
|
|
256
|
+
// #227. Hoisted above the object because `defaultBackend` INDEXES `backends`, and a property cannot read
|
|
257
|
+
// a sibling of the literal it is in. (Calling the parser twice would be harmless -- `egressEnabled` is
|
|
258
|
+
// called twice a few properties down for the same reason -- so this is about the index, not the throw.)
|
|
259
|
+
const backends = backendSet(env);
|
|
260
|
+
const backendFloor = backendFloorOf(env);
|
|
261
|
+
const config = {
|
|
215
262
|
valkeyUrl: env.VALKEY_URL ?? "redis://127.0.0.1:6379",
|
|
216
263
|
// Issue #57. What this machine calls itself: the key of its registry row, the `host` on every log
|
|
217
264
|
// line and run record, and the BullMQ worker name. Always populated -- a deployment that declares
|
|
@@ -238,6 +285,15 @@ export function loadConfig(env = process.env, { fileExists = existsSync } = {})
|
|
|
238
285
|
// per-job network is built around. Read BEFORE forwardEnv below, because the forward list's refusal
|
|
239
286
|
// of the proxy variables is conditional on it.
|
|
240
287
|
egress: egressEnabled(env),
|
|
288
|
+
// #227. The blessed set and the minimum every member of it must declare. The default set is the one
|
|
289
|
+
// name every existing deployment is already running, so an operator who has never heard of either
|
|
290
|
+
// variable gets exactly what they had.
|
|
291
|
+
backends,
|
|
292
|
+
backendFloor,
|
|
293
|
+
// The FIRST blessed name, which is what a deployment means by "the one my jobs run on unless a
|
|
294
|
+
// trigger says otherwise". `parseBackendList` never returns empty, so the fallback is belt-and-braces
|
|
295
|
+
// against a future edit rather than a reachable branch today.
|
|
296
|
+
defaultBackend: backends[0] ?? DEFAULT_BACKEND,
|
|
241
297
|
egressProxy: env.PI_EGRESS_PROXY || DEFAULT_EGRESS_PROXY, // || (not ??) so an empty string falls back
|
|
242
298
|
forwardEnv: forwardEnvList(env.PI_FORWARD_ENV, egressEnabled(env)), // extra host var NAMES to forward (e.g. a custom provider's key); explicit allowlist, GitHub token names refused
|
|
243
299
|
authFromPi: env.PI_AUTH_FROM_PI !== "0", // ON by default: use the key in ~/.pi/agent/auth.json when the env has none (api-key only). PI_AUTH_FROM_PI=0 forces env-only.
|
|
@@ -344,6 +400,12 @@ export function loadConfig(env = process.env, { fileExists = existsSync } = {})
|
|
|
344
400
|
forgejo: loadForgejoAuth(env),
|
|
345
401
|
azure: loadAzureAuth(env),
|
|
346
402
|
};
|
|
403
|
+
|
|
404
|
+
// #227, and it runs AFTER the object is built rather than inside it: the refusal reads `egress` as well
|
|
405
|
+
// as the two backend fields, and a check woven between properties would depend on key order.
|
|
406
|
+
refuseBackendShortfall(config);
|
|
407
|
+
|
|
408
|
+
return config;
|
|
347
409
|
}
|
|
348
410
|
|
|
349
411
|
/**
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* WHAT the box is, with no Docker vocabulary in it (issue #227).
|
|
3
|
+
*
|
|
4
|
+
* This module is the half of `INT-CONTAINER-RUNTIME-CONTRACT` that is not about Docker: the container-side
|
|
5
|
+
* paths a job always sees, and the value that describes one container before any runtime spells it. The
|
|
6
|
+
* other half -- `ISOLATION_FLAGS`, `dockerArgsFromSpec` -- stays in `docker-run.mjs`, which re-exports
|
|
7
|
+
* everything below so no call site moved. Two modules import from here directly: `docker-run.mjs` for the
|
|
8
|
+
* spec builder, and `packages.mjs` for `CONTAINER_GLOBAL_PI_DIR`. The second one is the one to keep in mind
|
|
9
|
+
* when editing this file, because it is the reason the leaf property below is load-bearing.
|
|
10
|
+
*
|
|
11
|
+
* WHY IT IS ITS OWN FILE NOW AND WAS NOT BEFORE. Issue #261 split the spec from the argv but deliberately
|
|
12
|
+
* kept both in one file, and said exactly what would change that:
|
|
13
|
+
*
|
|
14
|
+
* "Rejected: a separate `container-spec.mjs`, which would have ... added an import edge before a second
|
|
15
|
+
* consumer exists; the split here is by function, and the move to its own module belongs to the PR that
|
|
16
|
+
* adds one."
|
|
17
|
+
*
|
|
18
|
+
* This is that PR. The second consumer is the backend seam: a backend that is not the local Docker daemon
|
|
19
|
+
* consumes a spec and never produces a Docker argv, so the description of a container has to be reachable
|
|
20
|
+
* without reaching the argv builder. Nothing about the value changed in the move -- same fields, same
|
|
21
|
+
* order, same guards -- because a byte-identical local deployment is #227's first constraint.
|
|
22
|
+
*
|
|
23
|
+
* It imports NOTHING, which is the property `packages.mjs` depends on when it derives the staged-packages
|
|
24
|
+
* root from `CONTAINER_GLOBAL_PI_DIR` rather than re-typing the path.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Where the operator's global pi overlay lands INSIDE the container (REQ-GLOBAL-PI-OVERLAY). Exported
|
|
29
|
+
* because packages.mjs derives the staged-packages root from it: the mount and that root are ONE fact on
|
|
30
|
+
* one side of the boundary, and two literals in two modules could drift apart with both test suites still
|
|
31
|
+
* green. A Linux container path, so it is always built with "/" -- never `path.join`, which yields
|
|
32
|
+
* backslashes when the worker itself runs on Windows.
|
|
33
|
+
*/
|
|
34
|
+
export const CONTAINER_GLOBAL_PI_DIR = "/opt/pi-global";
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* The session mount and the file inside it, exported together and used by both the argv builder in
|
|
38
|
+
* docker-run.mjs and the env builder in env-allowlist.mjs. Two literals in two modules is how a mount and
|
|
39
|
+
* the variable naming a path inside it drift apart with both suites green -- the runner would then look for
|
|
40
|
+
* a transcript at a path nothing mounted, find none, and cold-start every job without saying so.
|
|
41
|
+
*
|
|
42
|
+
* Nothing key-derived crosses the boundary: the container always sees the same constant path, so no
|
|
43
|
+
* repository name, no branch name and no host layout is legible from inside a job.
|
|
44
|
+
*/
|
|
45
|
+
export const CONTAINER_SESSION_DIR = "/session";
|
|
46
|
+
export const CONTAINER_SESSION_FILE = `${CONTAINER_SESSION_DIR}/current.jsonl`;
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* WHAT the box is, with no Docker vocabulary in it.
|
|
50
|
+
*
|
|
51
|
+
* Split from the argv builder so the description of a container exists as a VALUE before it becomes one
|
|
52
|
+
* runtime's flags. `buildDockerRunArgs` is unchanged in name, signature and output -- it is
|
|
53
|
+
* `dockerArgsFromSpec(containerSpec(opts))` -- so every caller and every assertion is untouched, and the
|
|
54
|
+
* only thing that is new is that the middle of that sentence can be read on its own.
|
|
55
|
+
*
|
|
56
|
+
* Mounts are structured (`{host, container, readOnly}`) rather than pre-flattened `host:container:ro`
|
|
57
|
+
* strings, because the flattening IS the Docker part: a runtime that does not bind-mount has to be able to
|
|
58
|
+
* see which host path becomes which container path, and what may be written.
|
|
59
|
+
*
|
|
60
|
+
* `dockerExtra` is named for what it is. It carries raw Docker flags (`-i -t --entrypoint bash`, a
|
|
61
|
+
* Linux-only `--user`), so it is the one field a non-Docker consumer must refuse rather than translate.
|
|
62
|
+
* Calling it `extraFlags` at the boundary would have hidden that.
|
|
63
|
+
*
|
|
64
|
+
* @param image pinned job image tag/digest
|
|
65
|
+
* @param env the closed env map from buildContainerEnv -- passed as explicit -e NAME=VALUE
|
|
66
|
+
* @param jobDir host path to the /job inputs dir (contains prompt.md and pi/); mounted /job:ro
|
|
67
|
+
* @param workspace host path to the fresh clone / local folder (mounted /workspace:rw)
|
|
68
|
+
* @param outboxDir host path to the /outbox chain-request dir (local jobs only); mounted /outbox:rw
|
|
69
|
+
* @param sessionDir host path to this job's OWN copy of its session transcript (REQ-RESUMABLE-SESSION);
|
|
70
|
+
* mounted /session:rw. Per-job, like jobDir -- never the shared store.
|
|
71
|
+
* @param globalPiDir host path to the operator's global pi overlay (REQ-GLOBAL-PI-OVERLAY); mounted /opt/pi-global:ro
|
|
72
|
+
* @param name container name (for `docker stop` at the timeout)
|
|
73
|
+
* @param memory e.g. "4g"; cpus e.g. "2"
|
|
74
|
+
* @param network the per-job egress network this container joins (REQ-EGRESS-ALLOWLIST); null = the
|
|
75
|
+
* docker default bridge, which is what every job did before that requirement existed
|
|
76
|
+
* @param extraFlags escape hatch for a Linux-only --user uid:gid on a bind-mounted local folder
|
|
77
|
+
*/
|
|
78
|
+
export function containerSpec({
|
|
79
|
+
image,
|
|
80
|
+
env,
|
|
81
|
+
jobDir,
|
|
82
|
+
workspace,
|
|
83
|
+
outboxDir,
|
|
84
|
+
sessionDir,
|
|
85
|
+
globalPiDir,
|
|
86
|
+
name,
|
|
87
|
+
memory = "4g",
|
|
88
|
+
cpus = "2",
|
|
89
|
+
network = null,
|
|
90
|
+
extraFlags = [],
|
|
91
|
+
}) {
|
|
92
|
+
if (!image) throw new Error("docker run: image is required");
|
|
93
|
+
if (!name) throw new Error("docker run: container name is required");
|
|
94
|
+
if (!workspace) throw new Error("docker run: workspace mount is required");
|
|
95
|
+
|
|
96
|
+
const mounts = [];
|
|
97
|
+
// The WHOLE /job dir is read-only (INT-CONTAINER-JOB-INPUTS): it holds prompt.md and pi/, and
|
|
98
|
+
// the agent cannot rewrite any of it. /workspace is the only writable mount.
|
|
99
|
+
if (jobDir) mounts.push({ host: jobDir, container: "/job", readOnly: true });
|
|
100
|
+
mounts.push({ host: workspace, container: "/workspace", readOnly: false });
|
|
101
|
+
// Local jobs get a writable /outbox host bind, the same host-bind mechanism as /workspace
|
|
102
|
+
// (DES-WORKER-ON-HOST). github jobs pass no outboxDir, so the request channel does not exist for
|
|
103
|
+
// them -- an untrusted issue author cannot chain (INT-OUTBOX-CONTRACT).
|
|
104
|
+
if (outboxDir) mounts.push({ host: outboxDir, container: "/outbox", readOnly: false });
|
|
105
|
+
|
|
106
|
+
// This job's OWN copy of its session transcript (REQ-RESUMABLE-SESSION, INT-SESSION-STORE-CONTRACT).
|
|
107
|
+
// Writable, because pi appends to it as the agent works -- and per-job, exactly like jobDir, which is
|
|
108
|
+
// the whole reason CONST-ISOLATION-CONTAINER-PER-JOB's "none host-wide" clause still reads true. The
|
|
109
|
+
// shared store under PI_SESSIONS_DIR is NEVER bind-mounted: one job here would otherwise be able to
|
|
110
|
+
// read and rewrite every other branch's and every other repository's transcripts, which is not a
|
|
111
|
+
// weakening of that constraint but its inversion. Absent unless the trigger armed run.resume AND a key
|
|
112
|
+
// resolved, so an unarmed job's argv is byte-identical to one built before this feature existed.
|
|
113
|
+
if (sessionDir) mounts.push({ host: sessionDir, container: CONTAINER_SESSION_DIR, readOnly: false });
|
|
114
|
+
|
|
115
|
+
// The operator's global pi overlay (REQ-GLOBAL-PI-OVERLAY): custom models, global skills, a global
|
|
116
|
+
// persona, layered UNDER each repo's own .pi/. Read-only -- it is operator-authored deploy-time config,
|
|
117
|
+
// the same trust class as the baked floor, but the agent still must not rewrite it. Both job kinds.
|
|
118
|
+
if (globalPiDir) mounts.push({ host: globalPiDir, container: CONTAINER_GLOBAL_PI_DIR, readOnly: true });
|
|
119
|
+
|
|
120
|
+
return {
|
|
121
|
+
image,
|
|
122
|
+
name,
|
|
123
|
+
memory,
|
|
124
|
+
cpus,
|
|
125
|
+
network,
|
|
126
|
+
// UNCONDITIONALLY true, and there is deliberately no parameter that can unset it. The boundary is
|
|
127
|
+
// not a thing a caller opts into -- CONST-ISOLATION-CONTAINER-PER-JOB is why every other flag here
|
|
128
|
+
// exists -- so the spec is simply unable to describe an unisolated container, and the builder in
|
|
129
|
+
// docker-run.mjs refuses one it is handed. A field that could be false would be a way to ask for less.
|
|
130
|
+
isolated: true,
|
|
131
|
+
mounts,
|
|
132
|
+
env,
|
|
133
|
+
dockerExtra: extraFlags,
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* The mounts of a spec, rendered as TRANSFERS for a runtime that cannot bind-mount (issue #227).
|
|
139
|
+
*
|
|
140
|
+
* A bind mount is not a file copy, and the whole value of this function is that it refuses to pretend
|
|
141
|
+
* otherwise. `DES-JOB-FILES-VIA-VOLUME-SUBPATH` already recorded the specific loss: `docker cp` "cannot
|
|
142
|
+
* give `/job` a kernel-enforced read-only mount, which `INT-CONTAINER-JOB-INPUTS` depends on" -- and every
|
|
143
|
+
* vendor upload API is `docker cp`-shaped. So `binds` says which kind of runtime is asking: a binding one
|
|
144
|
+
* gets `readOnlyEnforcedBy: "kernel"`, a copying one gets `"convention"`, and the second is a DECLARED
|
|
145
|
+
* DOWNGRADE rather than a neutral translation. That word is what a conformance suite reads, and what stops
|
|
146
|
+
* "we upload the files" being mistaken for "the agent cannot rewrite its own instructions".
|
|
147
|
+
*
|
|
148
|
+
* `0444` file modes are the second and weaker line, named here so nobody mistakes them for the first: they
|
|
149
|
+
* bind an agent that respects them and nothing else, whereas the kernel binds one that does not.
|
|
150
|
+
*
|
|
151
|
+
* DIRECTION FOLLOWS WRITABILITY, not a list of paths. An earlier draft hardcoded `/workspace` as the only
|
|
152
|
+
* mount coming back, and that was wrong in a way that would have been silent: `/outbox` and `/session` are
|
|
153
|
+
* writable too, and the HOST reads both after the container exits -- `collectChain` reads `jobDir/outbox`
|
|
154
|
+
* for the chain requests the agent wrote (`INT-OUTBOX-CONTRACT`), and `promoteSession` reads
|
|
155
|
+
* `jobDir/session` for the transcript pi appended to (`REQ-RESUMABLE-SESSION`). An adapter that brought
|
|
156
|
+
* back only `/workspace` would never enqueue a chained child and would cold-start every resume, both
|
|
157
|
+
* reporting success. So a mount the container may write is a mount the host may need back, and the rule is
|
|
158
|
+
* exactly that.
|
|
159
|
+
*
|
|
160
|
+
* THE NESTING IS MODELLED, because it is the trap. `session/` nests inside `jobDir` for every job kind, and
|
|
161
|
+
* `workspace/` and `outbox/` nest for their own kinds, so an adapter that uploads each mount independently
|
|
162
|
+
* produces a container where `/job/workspace` also exists with a stale copy of the tree -- path-equivalent
|
|
163
|
+
* to nothing the local backend ever produces, and silently divergent rather than broken. `contains` names,
|
|
164
|
+
* for each entry, the other container paths whose host path lies inside this one, so an adapter can exclude
|
|
165
|
+
* them from the upload instead of discovering the overlap in review.
|
|
166
|
+
*
|
|
167
|
+
* @param binds true when the runtime bind-mounts (the local backend); false when it copies.
|
|
168
|
+
*/
|
|
169
|
+
export function transfersFromSpec(spec, { binds = true } = {}) {
|
|
170
|
+
const mounts = spec?.mounts ?? [];
|
|
171
|
+
return mounts.map((m) => ({
|
|
172
|
+
host: m.host,
|
|
173
|
+
container: m.container,
|
|
174
|
+
readOnly: m.readOnly === true,
|
|
175
|
+
// The distinction this function exists for. A bind is enforced by the kernel; a copy is enforced by
|
|
176
|
+
// whatever the agent chooses to respect, which is not enforcement. `null` where nothing is claimed.
|
|
177
|
+
readOnlyEnforcedBy: m.readOnly === true ? (binds ? "kernel" : "convention") : null,
|
|
178
|
+
// Writable means the container may change it, which means the host may need it back. See the header:
|
|
179
|
+
// `/outbox` and `/session` are both read after the run, and hardcoding `/workspace` missed them.
|
|
180
|
+
direction: m.readOnly === true ? "in" : "in-out",
|
|
181
|
+
// Other mounts whose HOST path is nested inside this one, by container path. Uploading this entry
|
|
182
|
+
// without excluding these duplicates their trees under it.
|
|
183
|
+
contains: mounts.filter((o) => isInside(m.host, o.host)).map((o) => o.container),
|
|
184
|
+
}));
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* Is `inner` a host path strictly beneath `outer`?
|
|
189
|
+
*
|
|
190
|
+
* SEPARATOR-AGNOSTIC, because host paths are built with `path.join` (`join(jobDir, "workspace")`) and this
|
|
191
|
+
* worker runs on Windows -- where those are backslash-separated while an earlier draft compared against a
|
|
192
|
+
* hardcoded `/`. That draft reported NO nesting at all on Windows, which is precisely the platform where a
|
|
193
|
+
* silently divergent upload would be hardest to spot. The container paths above are always `/`-built and
|
|
194
|
+
* are not the ones compared here.
|
|
195
|
+
*
|
|
196
|
+
* A trailing separator on either side is ignored, and equality is not containment.
|
|
197
|
+
*/
|
|
198
|
+
function isInside(outer, inner) {
|
|
199
|
+
const strip = (v) => String(v ?? "").replace(/[\\/]+$/, "");
|
|
200
|
+
const a = strip(outer);
|
|
201
|
+
const b = strip(inner);
|
|
202
|
+
if (a === "" || a === b) return false;
|
|
203
|
+
const next = b.charAt(a.length);
|
|
204
|
+
return b.startsWith(a) && (next === "/" || next === "\\");
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* What an adapter LOSES by copying instead of bind-mounting, as `[{ container, was, becomes }]`.
|
|
209
|
+
*
|
|
210
|
+
* Empty for a runtime that binds. Non-empty is not a failure: it is the list an adapter must declare, and
|
|
211
|
+
* the reason `readOnlyJobInputs` is a property in the backend table at all rather than an assumption. An
|
|
212
|
+
* adapter that returns entries here and still declares `readOnlyJobInputs: enforced` is making exactly the
|
|
213
|
+
* claim `CONST-EGRESS-POLICY-IN-THE-ARGV` calls worse than no claim.
|
|
214
|
+
*
|
|
215
|
+
* `backend-conformance.mjs` is the consumer, and it is what makes this a control rather than a contract: a
|
|
216
|
+
* backend declaring `binds: false` and `readOnlyJobInputs: enforced` FAILS on the strength of this list.
|
|
217
|
+
* A backend that declares no `binds` at all makes the harness abstain rather than pass, because a bundle
|
|
218
|
+
* that has not said how it moves files has not earned the word.
|
|
219
|
+
*/
|
|
220
|
+
export function copyDowngrades(spec) {
|
|
221
|
+
const bound = transfersFromSpec(spec, { binds: true });
|
|
222
|
+
const copied = transfersFromSpec(spec, { binds: false });
|
|
223
|
+
return bound
|
|
224
|
+
.map((b, i) => ({ container: b.container, was: b.readOnlyEnforcedBy, becomes: copied[i].readOnlyEnforcedBy }))
|
|
225
|
+
.filter((d) => d.was !== null && d.was !== d.becomes);
|
|
226
|
+
}
|
package/src/docker-run.mjs
CHANGED
|
@@ -6,28 +6,21 @@
|
|
|
6
6
|
* array (never a shell string): no interpolation, no injection, and the env allowlist is passed
|
|
7
7
|
* with explicit `-e NAME` where the value is read from the argv env map, never `--env-file` and
|
|
8
8
|
* never a host pass-through.
|
|
9
|
-
*/
|
|
10
|
-
|
|
11
|
-
/**
|
|
12
|
-
* Where the operator's global pi overlay lands INSIDE the container (REQ-GLOBAL-PI-OVERLAY). Exported
|
|
13
|
-
* because packages.mjs derives the staged-packages root from it: the mount and that root are ONE fact on
|
|
14
|
-
* one side of the boundary, and two literals in two modules could drift apart with both test suites still
|
|
15
|
-
* green. A Linux container path, so it is always built with "/" -- never `path.join`, which yields
|
|
16
|
-
* backslashes when the worker itself runs on Windows.
|
|
17
|
-
*/
|
|
18
|
-
export const CONTAINER_GLOBAL_PI_DIR = "/opt/pi-global";
|
|
19
|
-
|
|
20
|
-
/**
|
|
21
|
-
* The session mount and the file inside it, exported together and used by both the argv builder here and
|
|
22
|
-
* the env builder in env-allowlist.mjs. Two literals in two modules is how a mount and the variable
|
|
23
|
-
* naming a path inside it drift apart with both suites green -- the runner would then look for a
|
|
24
|
-
* transcript at a path nothing mounted, find none, and cold-start every job without saying so.
|
|
25
9
|
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
10
|
+
* HOW Docker spells a container. WHAT a container IS moved to `container-spec.mjs` (issue #227), which is
|
|
11
|
+
* the move #261 deferred to "the PR that adds a second consumer" -- the backend seam is that consumer, since
|
|
12
|
+
* a backend that is not the local Docker daemon consumes a spec and never produces an argv. The constants
|
|
13
|
+
* and `containerSpec` are RE-EXPORTED below, so every existing import of this module still resolves and
|
|
14
|
+
* every assertion in the suite is untouched. This file is now the LOCAL backend's half of that contract.
|
|
28
15
|
*/
|
|
29
|
-
|
|
30
|
-
|
|
16
|
+
|
|
17
|
+
// The portable half. Re-exported rather than moved out of reach: `run-container.mjs` reads
|
|
18
|
+
// CONTAINER_SESSION_FILE beside `buildDockerRunArgs` and is Docker-bound anyway, and `containerSpec` is
|
|
19
|
+
// imported from this module by the suite. `CONST-EGRESS-POLICY-IN-THE-ARGV`'s Code evidence names
|
|
20
|
+
// `buildDockerRunArgs`, which never moved; the entry that names `containerSpec` is design.md's 2026-08-31
|
|
21
|
+
// row, and this move is recorded in its own row rather than by leaving that pointer to rot.
|
|
22
|
+
export { containerSpec, CONTAINER_GLOBAL_PI_DIR, CONTAINER_SESSION_DIR, CONTAINER_SESSION_FILE } from "./container-spec.mjs";
|
|
23
|
+
import { containerSpec } from "./container-spec.mjs";
|
|
31
24
|
|
|
32
25
|
/** The fixed isolation flags. Not configurable -- these ARE the boundary. */
|
|
33
26
|
export const ISOLATION_FLAGS = [
|
|
@@ -50,103 +43,104 @@ export const ISOLATION_FLAGS = [
|
|
|
50
43
|
];
|
|
51
44
|
|
|
52
45
|
/**
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
* `
|
|
58
|
-
*
|
|
46
|
+
* HOW Docker spells it. The only consumer of a spec today, and the local backend's translation step:
|
|
47
|
+
* a second backend implements this function's job against its own runtime and shares everything above it.
|
|
48
|
+
*/
|
|
49
|
+
/**
|
|
50
|
+
* Flags a `dockerExtra` may not carry, because docker resolves a repeated option LAST-WINS and this array
|
|
51
|
+
* is appended AFTER `ISOLATION_FLAGS`. `--privileged` supersedes `--cap-drop=ALL`; `--network` supersedes
|
|
52
|
+
* the per-job one; `--pull` supersedes `--pull=never`; `--rm=false` supersedes `--rm` (verified against
|
|
53
|
+
* docker 27.4.0, not assumed); a second `-v` adds a mount the spec never declared; a second `--name` wins
|
|
54
|
+
* and leaves a container outside both reapers' filter and beyond `docker stop`.
|
|
59
55
|
*
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
*
|
|
56
|
+
* The list covers every member of `ISOLATION_FLAGS`, the container name, and the near-synonyms that reach
|
|
57
|
+
* the same effect without repeating a listed flag (`--volumes-from` for a mount, `--memory-swap` for the
|
|
58
|
+
* memory bound). It is a DENY-list and therefore only as good as its coverage: docker's surface is large
|
|
59
|
+
* and a release can add another way in. So it NARROWS the gap rather than closing it, and the backend
|
|
60
|
+
* table's `isolation` and `ephemeral` words rest on this plus the single production caller passing fixed
|
|
61
|
+
* literals, never on this alone.
|
|
63
62
|
*
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
63
|
+
* Without this the two standing assertions that "every member of ISOLATION_FLAGS reaches the argv" would
|
|
64
|
+
* still pass on an argv with no boundary left, because membership is not effectiveness. Nothing passes any
|
|
65
|
+
* of these today -- `sandbox.mjs` sends `-i -t --entrypoint bash` plus loopback-bound `-p` flags, all of
|
|
66
|
+
* which stay allowed -- so this closes a hole rather than changing a behaviour, and it makes "the builder
|
|
67
|
+
* CANNOT DECLINE the boundary" true of the whole argv instead of of one boolean field.
|
|
67
68
|
*
|
|
68
|
-
*
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
-
*
|
|
72
|
-
*
|
|
73
|
-
*
|
|
74
|
-
*
|
|
75
|
-
* @param globalPiDir host path to the operator's global pi overlay (REQ-GLOBAL-PI-OVERLAY); mounted /opt/pi-global:ro
|
|
76
|
-
* @param name container name (for `docker stop` at the timeout)
|
|
77
|
-
* @param memory e.g. "4g"; cpus e.g. "2"
|
|
78
|
-
* @param network the per-job egress network this container joins (REQ-EGRESS-ALLOWLIST); null = the
|
|
79
|
-
* docker default bridge, which is what every job did before that requirement existed
|
|
80
|
-
* @param extraFlags escape hatch for a Linux-only --user uid:gid on a bind-mounted local folder
|
|
69
|
+
* `--user` is DELIBERATELY NOT HERE, and it is the one that looks like it belongs. It is a documented,
|
|
70
|
+
* tested feature -- the Linux-only `uid:gid` on a bind-mounted local folder, so files the agent writes into
|
|
71
|
+
* an operator's own directory come back owned by the operator rather than by root. It also does not touch
|
|
72
|
+
* this boundary: it changes which uid runs, not what that uid may do, and `--cap-drop=ALL` plus
|
|
73
|
+
* `no-new-privileges` hold either way. What it does bear on is `nonRoot`, which the backend table already
|
|
74
|
+
* declares `asserted` for the separate reason that the image's `USER pi` is what provides it. Denying it
|
|
75
|
+
* here would break local-folder jobs to make a word honest that is already honest.
|
|
81
76
|
*/
|
|
82
|
-
export
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
//
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
memory,
|
|
128
|
-
cpus,
|
|
129
|
-
network,
|
|
130
|
-
// UNCONDITIONALLY true, and there is deliberately no parameter that can unset it. The boundary is
|
|
131
|
-
// not a thing a caller opts into -- CONST-ISOLATION-CONTAINER-PER-JOB is why every other flag here
|
|
132
|
-
// exists -- so the spec is simply unable to describe an unisolated container, and the builder below
|
|
133
|
-
// refuses one it is handed. A field that could be false would be a way to ask for less.
|
|
134
|
-
isolated: true,
|
|
135
|
-
mounts,
|
|
136
|
-
env,
|
|
137
|
-
dockerExtra: extraFlags,
|
|
138
|
-
};
|
|
139
|
-
}
|
|
77
|
+
export const DOCKER_EXTRA_FORBIDDEN = [
|
|
78
|
+
// Each of the seven logical flags in ISOLATION_FLAGS, and the argv member beside them that the worker's
|
|
79
|
+
// own machinery reads back. A flag missing from here is a flag the array cannot defend.
|
|
80
|
+
"--rm", // `--rm=false` leaves the container behind; verified against docker 27.4.0. `ephemeral` rests on it.
|
|
81
|
+
"--init",
|
|
82
|
+
"--shm-size",
|
|
83
|
+
// The container NAME is not an isolation flag and is the sharpest entry on this list: both boot reapers
|
|
84
|
+
// match `pi-job-` as a substring and the abort path is `docker stop <name>`, so a second `--name` wins
|
|
85
|
+
// last and leaves a container no sweep finds and no timeout can stop. `ephemeral` and `abortable` both
|
|
86
|
+
// rest on it.
|
|
87
|
+
"--name",
|
|
88
|
+
"--privileged",
|
|
89
|
+
"--cap-add",
|
|
90
|
+
"--security-opt",
|
|
91
|
+
"--pids-limit",
|
|
92
|
+
"--memory",
|
|
93
|
+
"-m",
|
|
94
|
+
"--cpus",
|
|
95
|
+
"--network",
|
|
96
|
+
"--net",
|
|
97
|
+
"--pull",
|
|
98
|
+
// Add a mount, which is what `-v`/`--volume`/`--mount` are blocked for; `mountSet` rests on all five.
|
|
99
|
+
"--volumes-from",
|
|
100
|
+
"--tmpfs",
|
|
101
|
+
// Relax the memory bound without repeating `--memory`.
|
|
102
|
+
"--memory-swap",
|
|
103
|
+
"--oom-kill-disable",
|
|
104
|
+
// Widen what the process may do without repeating a flag already listed.
|
|
105
|
+
"--ulimit",
|
|
106
|
+
"--sysctl",
|
|
107
|
+
"--group-add",
|
|
108
|
+
"--cgroup-parent",
|
|
109
|
+
"--device-cgroup-rule",
|
|
110
|
+
"--gpus",
|
|
111
|
+
"--runtime",
|
|
112
|
+
"-v",
|
|
113
|
+
"--volume",
|
|
114
|
+
"--mount",
|
|
115
|
+
"--device",
|
|
116
|
+
"--pid",
|
|
117
|
+
"--ipc",
|
|
118
|
+
"--uts",
|
|
119
|
+
"--userns",
|
|
120
|
+
"--cgroupns",
|
|
121
|
+
];
|
|
140
122
|
|
|
141
|
-
/**
|
|
142
|
-
* HOW Docker spells it. The only consumer of a spec today.
|
|
143
|
-
*/
|
|
144
123
|
export function dockerArgsFromSpec(spec) {
|
|
145
124
|
// The builder CANNOT DECLINE the boundary. `containerSpec` cannot produce anything but `true`, so this
|
|
146
125
|
// only ever fires on a hand-built spec -- and a hand-built spec that forgot the field is exactly the
|
|
147
126
|
// case that must fail loudly rather than quietly emit a container with no isolation flags at all.
|
|
148
127
|
if (spec?.isolated !== true) throw new Error("docker run: refusing to build an argv for a spec that is not isolated");
|
|
149
128
|
|
|
129
|
+
// Same refusal, one level down. `dockerExtra` is raw Docker flags by design, and it lands after the
|
|
130
|
+
// boundary where a repeat supersedes it -- so the escape hatch is bounded by what it may not say.
|
|
131
|
+
// Split on `=` so `--network=foo` is caught alongside `--network foo`.
|
|
132
|
+
for (const flag of spec.dockerExtra ?? []) {
|
|
133
|
+
// REFUSED, not skipped. Skipping a non-string still PUSHED it into the argv below, so
|
|
134
|
+
// `new String("--privileged")` and `{ toString: () => "--privileged" }` walked past the check and
|
|
135
|
+
// then reached docker as the flag they stringify to.
|
|
136
|
+
if (typeof flag !== "string") {
|
|
137
|
+
throw new Error(`docker run: dockerExtra must contain only strings; got ${typeof flag}`);
|
|
138
|
+
}
|
|
139
|
+
if (DOCKER_EXTRA_FORBIDDEN.includes(flag.split("=", 1)[0])) {
|
|
140
|
+
throw new Error(`docker run: refusing a dockerExtra flag that would supersede the isolation boundary: ${flag}`);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
150
144
|
// `--network` sits HERE, beside --memory and --cpus, and deliberately NOT inside ISOLATION_FLAGS.
|
|
151
145
|
// That array is the LITERAL, value-free, unconditional set, and two separate places assert every member
|
|
152
146
|
// of it reaches the sandbox argv *against the imported array, not a copy* (CONST-ISOLATION-CONTAINER-PER-JOB
|