@edgehero/pi-dispatch 0.3.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/outbox.mjs CHANGED
@@ -119,6 +119,20 @@ export function makeCollectChain({ queue, enqueue = enqueueLocalJob, readFlowGat
119
119
  continue;
120
120
  }
121
121
 
122
+ // Commands are NEVER AI-reachable (issue #189): a request naming one refuses outright,
123
+ // with no opt-in to widen. The flow gate below reads a COMMITTED artifact -- the target
124
+ // repo's SKILL.md frontmatter at the pinned sha, merge-gated and reviewable -- but a
125
+ // command is an operator-staged pi extension with no committed artifact a gate could
126
+ // read, so there is nothing to gate ON and fail-closed is the only honest answer. First
127
+ // in the semantic ladder, before the charset check: which field the request used is
128
+ // decided before any opinion about its spelling. A completed command job's OWN /outbox
129
+ // may still chain INTO flows through the unchanged gate below; nothing chains into a
130
+ // command.
131
+ if (req.command !== undefined) {
132
+ refuse("chain-command-refused", i);
133
+ continue;
134
+ }
135
+
122
136
  // Explicit property reads ONLY -- `req` is never spread into job data.
123
137
  const flow = req.flow;
124
138
  const task = req.task;
package/src/packages.mjs CHANGED
@@ -8,7 +8,10 @@
8
8
  * pi's package resolver treats any spec that is not `npm:`/`git:`/a URL as a LOCAL path: it resolves the
9
9
  * directory in place -- no install, no network, no writes -- and a `pi` manifest there contributes
10
10
  * extensions, skills, prompts and themes. That is precisely what lets a job container load them with
11
- * network egress denied, which is the whole point of staging at all.
11
+ * NO JOB-TIME INSTALL, which is the whole point of staging at all. This said "with network egress denied"
12
+ * until issue #202, and that was the third instance of one claim corrected in the spec and left standing in
13
+ * its twin: nothing denied egress when it was written, and what staging plus PI_OFFLINE=1 actually buys is
14
+ * that no install happens, which is true whatever the egress policy is.
12
15
  *
13
16
  * Versions are EXACT, never a range (CONST-PI-VERSION-PINNED): a floating range turns a silent upstream
14
17
  * minor into every queued job becoming a no-op with NO signal -- the queue still reports success, the
@@ -24,8 +27,8 @@
24
27
  * manifest must degrade to "no staged packages", not crash the worker mid-queue.
25
28
  */
26
29
 
27
- import { existsSync, readFileSync } from "node:fs";
28
- import { join } from "node:path";
30
+ import { existsSync, readdirSync, readFileSync } from "node:fs";
31
+ import { basename, dirname, join, resolve } from "node:path";
29
32
  import { configError } from "./config.mjs";
30
33
  // ENTRY_NAME_RE / ADMIN_RE are import-pi's -- imported rather than re-declared so the staged dir charset
31
34
  // and the admin block cannot drift between the stager and this validator (doctor.mjs sets the precedent
@@ -269,6 +272,89 @@ export function readStageManifest({ globalPiDir, readFile = readFileSync, fileEx
269
272
  }
270
273
  }
271
274
 
275
+ /**
276
+ * Enumerate the skills the staged packages would contribute to a job (issue #189). Returns
277
+ * `{ skills: [{ name, package, dir }], unenumerable: [packageName...] }`, and NEVER throws --
278
+ * readStageManifest's policy, because the consumers are advisory (doctor's per-trigger flow lines,
279
+ * and issue #188's topology) and a half-staged tree must degrade to "nothing visible", not a crash.
280
+ *
281
+ * The semantics mirror pi's collectPackageResources at the 0.80.7 pin EXACTLY, because an enumerator
282
+ * that agrees with pi by hand is how doctor comes to report a tier pi then ignores:
283
+ * - a `pi` manifest object means its `skills` entries are the ONLY sources -- a manifest WITHOUT a
284
+ * `skills` key contributes NO skills and gets NO convention fallback (readPiManifest short-circuits
285
+ * before the dir walk);
286
+ * - no `pi` key at all falls through to the convention `skills/` dir (RESOURCE_DIRS);
287
+ * - a manifest entry may be a SKILL.md file, a skill dir, or a dir of skill dirs -- pi walks
288
+ * recursively, so this walks the same shape (bounded, it is host-advisory);
289
+ * - a glob (`*`/`?`) or override (`!`/`+`/`-` prefix) entry makes the whole package UNENUMERABLE
290
+ * rather than guessed at: patterns can also DISABLE files, and a wrong ✓ (a skill reported that pi
291
+ * filters out) is the direction an advisory line must never err in. Reported, not silently skipped.
292
+ *
293
+ * Names are DIR basenames (or the SKILL.md's parent dir), which is pi's fallback naming rule; a
294
+ * frontmatter `name:` rename is invisible here. That approximation is deliberate and one-directional:
295
+ * it can only turn a would-be ✓ into a ⚠, and the runner's flow_not_loaded check compares against the
296
+ * names pi actually loaded (DES-FLOW-RESOLUTION-TWO-ADVISORY-LAYERS).
297
+ */
298
+ export function readStagedSkills({ globalPiDir, readFile = readFileSync, fileExists = existsSync, readDir = readdirSync } = {}) {
299
+ const out = { skills: [], unenumerable: [] };
300
+ const manifest = readStageManifest({ globalPiDir, readFile, fileExists });
301
+ if (!manifest) return out;
302
+
303
+ for (const pkg of manifest.packages) {
304
+ const root = join(globalPiDir, PACKAGES_SUBDIR, pkg.dir);
305
+ let sources;
306
+ try {
307
+ const parsed = JSON.parse(readFile(join(root, "package.json"), "utf8"));
308
+ const pi = parsed !== null && typeof parsed === "object" && parsed.pi !== null && typeof parsed.pi === "object" ? parsed.pi : null;
309
+ if (pi) {
310
+ const entries = Array.isArray(pi.skills) ? pi.skills.filter((e) => typeof e === "string") : [];
311
+ if (entries.some((e) => e.startsWith("!") || e.startsWith("+") || e.startsWith("-") || e.includes("*") || e.includes("?"))) {
312
+ out.unenumerable.push(pkg.name);
313
+ continue;
314
+ }
315
+ // `..`-carrying entries are dropped rather than resolved: the stager never writes one, and
316
+ // following one would make an advisory reader walk outside the staged tree. A segment
317
+ // test, not a prefix test, so it holds on Windows separators too (parsePackagePaths' rule).
318
+ sources = entries.filter((e) => !e.split(/[\\/]/).includes("..")).map((e) => resolve(root, e));
319
+ } else {
320
+ sources = [join(root, "skills")];
321
+ }
322
+ } catch {
323
+ continue; // no readable package.json: pi would skip it too
324
+ }
325
+ for (const source of sources) {
326
+ for (const skillFile of walkSkillFiles(source, fileExists, readDir)) {
327
+ out.skills.push({ name: basename(dirname(skillFile)), package: pkg.name, dir: pkg.dir });
328
+ }
329
+ }
330
+ }
331
+ return out;
332
+ }
333
+
334
+ /**
335
+ * The SKILL.md files under one manifest source, the shapes pi's collectFilesFromPaths accepts: the file
336
+ * itself, a dir holding SKILL.md, or a tree of skill dirs (walked to a small fixed depth -- pi recurses
337
+ * unbounded, but a host-advisory reader stops where real layouts stop). Never throws.
338
+ */
339
+ function walkSkillFiles(source, fileExists, readDir, depth = 3) {
340
+ if (basename(source) === "SKILL.md") return fileExists(source) ? [source] : [];
341
+ const found = [];
342
+ const own = join(source, "SKILL.md");
343
+ if (fileExists(own)) found.push(own);
344
+ if (depth === 0) return found;
345
+ let children;
346
+ try {
347
+ children = readDir(source, { withFileTypes: true });
348
+ } catch {
349
+ return found; // absent or unreadable: nothing visible here
350
+ }
351
+ for (const child of children) {
352
+ if (!child.isDirectory?.()) continue;
353
+ found.push(...walkSkillFiles(join(source, child.name), fileExists, readDir, depth - 1));
354
+ }
355
+ return found;
356
+ }
357
+
272
358
  /**
273
359
  * The CONTAINER paths of the staged packages, in manifest order -- what gets handed to pi as local package
274
360
  * specs. Built with template literals and never `path.join`: the worker may run on Windows, where `join`
@@ -190,15 +190,28 @@ export async function prepareGithubWorkspace(
190
190
  const session = job.resume === true ? resolveSession(job, { jobDir, resolved, piVersion }) : null;
191
191
 
192
192
  // Issue text is DATA: it enters the USER prompt (buildGithubPrompt), never a system prompt.
193
+ //
194
+ // A command job (issue #189) skips the per-forge envelope entirely: the prompt is the slash
195
+ // invocation `/${job.command}` and nothing else -- no data heading, no quoted issue text, and NO
196
+ // trailing newline, because pi hands everything after the first space to the handler as its
197
+ // argument string verbatim. CONST-ISSUE-TEXT-IS-DATA is preserved and arguably STRENGTHENED:
198
+ // payload text reaches a command job only as event.json below, a file the handler chooses to
199
+ // parse, never interpolated into prompt prose at all. The envelope's never-merge discipline is
200
+ // not lost either -- that discipline addresses MODEL prose, and a command handler is
201
+ // operator-staged code, the same trust tier extensions hold generally.
193
202
  writeFile(
194
203
  join(jobDir, "prompt.md"),
195
204
  // `replica`/`replicas` (REQ-REPLICA-RUNS) are host-assigned integers off job.data, so they are safe
196
205
  // to interpolate, and they are what makes this job's branch differ from its sibling's. This is the
197
- // SHARED forge preparer, so the gitlab/forgejo/azure builders receive the two keys and destructure
198
- // them away -- harmless, and always undefined while replicas are github-only.
199
- // `review` rides beside `comment` and, like `replica`/`replicas`, is destructured away by the
200
- // gitlab/forgejo/azure builders -- harmless, and always undefined while reviews are github-only.
201
- buildPrompt({ flow: job.flow, target: job.target, comment: job.trigger?.comment, resumed: session?.resume === true, replica: job.replica, replicas: job.replicas, review: job.trigger?.review, instructions: job.instructions }),
206
+ // SHARED forge preparer, and since #187 all four builders READ the two keys rather than destructuring
207
+ // them away: every forge mints `pi/issue-<n>-r<i>` through the same issueBranch.
208
+ // `review` rides beside `comment` and IS still destructured away by the gitlab/forgejo/azure builders
209
+ // -- harmless, and always undefined, because reviews remain github-only. That asymmetry is why both are
210
+ // spelled out: one of these keys crossed the forge boundary and one did not, and a reader who assumes
211
+ // they move together will widen the wrong one.
212
+ job.command
213
+ ? `/${job.command}`
214
+ : buildPrompt({ flow: job.flow, target: job.target, comment: job.trigger?.comment, resumed: session?.resume === true, replica: job.replica, replicas: job.replicas, review: job.trigger?.review, instructions: job.instructions }),
202
215
  { mode: 0o444 },
203
216
  );
204
217
 
package/src/prepare.mjs CHANGED
@@ -93,10 +93,19 @@ export function makePrepareWorkspace({
93
93
  // flow can discover the trigger context (mirroring the github prompt, which names the same
94
94
  // file); nothing in-container reads it otherwise. The pointer sits AFTER the flow hint and
95
95
  // BEFORE the operator's task, which stays verbatim (CONST-ISSUE-TEXT-IS-DATA).
96
+ //
97
+ // A command job (issue #189) bypasses ALL of that: the prompt is the slash invocation itself,
98
+ // `/name args`, and nothing else -- no pointer line, no task, and critically NO trailing
99
+ // newline, because pi hands everything after the first space to the handler as its argument
100
+ // string verbatim, so a newline appended here would land inside the args. The pointer is
101
+ // prose addressed to a MODEL reading instructions; a command handler is code, and its context
102
+ // channel is /job/event.json itself, which prepare-local already writes for every local job.
96
103
  const pointer = "Context about this run -- its trigger and schedule -- is in /job/event.json.\n\n";
97
- const task = job.flow
98
- ? `Use the "${job.flow}" skill for this task.\n\n${pointer}${job.task ?? ""}`
99
- : `${pointer}${job.task ?? ""}`;
104
+ const task = job.command
105
+ ? `/${job.command}`
106
+ : job.flow
107
+ ? `Use the "${job.flow}" skill for this task.\n\n${pointer}${job.task ?? ""}`
108
+ : `${pointer}${job.task ?? ""}`;
100
109
  const event = localEventContext(job, queueJobId, findPreviousRun);
101
110
  return discardOnPolicy(stampSandbox(await prepareLocal({ folder: job.folder, task, jobDir, event }), sandbox), jobDir);
102
111
  }
package/src/processor.mjs CHANGED
@@ -41,6 +41,9 @@ export async function runJob(job, deps) {
41
41
  // this job names is on this host (image-preflight.mjs). Default admits everything, so a wiring that
42
42
  // omits it behaves exactly as before -- the container's own failure stays the backstop.
43
43
  imagePreflight = async () => ({ ok: true }),
44
+ // REQ-EGRESS-ALLOWLIST. Default admits everything, so a wiring that omits it behaves exactly as a
45
+ // deployment with no egress policy does -- which is also what the real factory returns when unarmed.
46
+ egressPreflight = async () => ({ ok: true }),
44
47
  // (session, { piVersion }) => { promoted, reason, bytes }. Promotes this job's transcript back into
45
48
  // the store, on a COMPLETED exit only. Never throws. The default is a no-op so a wiring that omits
46
49
  // it behaves exactly as before -- no store, no promotion, no session in the record.
@@ -159,6 +162,23 @@ export async function runJob(job, deps) {
159
162
  log("refused_image_replicas_unsupported", { image: img.replicaUnsupported, declared: img.declared });
160
163
  return { outcome: "policy", reason: "job-image-replicas-unsupported", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
161
164
  }
165
+ if (img.commandUnsupported) {
166
+ // The image is present and does not declare command support (issue #189), so its runner
167
+ // predates run.command: it reads no PI_COMMAND, and the bare `/name args` prompt reaches the
168
+ // model as PROSE -- no handler runs, the agent improvises, and the queue records a clean exit
169
+ // 0. The in-container half of the gate (the runner's own command-unregistered refusal) does
170
+ // not exist on such an image, which is exactly why the host must refuse first.
171
+ //
172
+ // Determinate, so a refusal rather than a retry, and pre-spend, because no version of this
173
+ // gets better by running. Like the replica branch above, the message names the FIX rather
174
+ // than the label that noticed it.
175
+ await comment(
176
+ job,
177
+ `Refused: the job image "${img.commandUnsupported}" does not declare command support (\`dev.pi-dispatch.capabilities\` ${img.declared.length > 0 ? `declares: ${img.declared.join(", ")}` : "is absent"}), so its runner would not dispatch \`run.command\`. Rebuild the image from a version that has this feature. Not run.`,
178
+ );
179
+ log("refused_image_commands_unsupported", { image: img.commandUnsupported, declared: img.declared });
180
+ return { outcome: "policy", reason: "job-image-commands-unsupported", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
181
+ }
162
182
  if (img.unavailable) {
163
183
  // docker itself did not answer -- transient infra, NOT a determinate refusal. THROWN so BullMQ
164
184
  // retries (CONST-RETRY-INFRA-ONLY). `container-never-started` is literally true here, and it reuses
@@ -167,6 +187,46 @@ export async function runJob(job, deps) {
167
187
  throw new InfraRetry("docker unavailable, image preflight could not run", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
168
188
  }
169
189
 
190
+ // REQ-EGRESS-ALLOWLIST. The egress policy this deployment claims must be able to serve this job
191
+ // BEFORE the job costs anything. It is one `docker inspect` when the policy is armed and ZERO spawns
192
+ // when it is not, so a deployment without one pays nothing at all.
193
+ //
194
+ // PLACEMENT, and it is the same ladder the image preflight sits at the top of. A missing proxy blocks
195
+ // EVERY job of EVERY kind on this host -- like a missing image -- and unlike a missing image it blocks
196
+ // them EXPENSIVELY: the container starts, the provider is unreachable, the runner exits 1, exit 1 is
197
+ // the retryable class, `attempts: 2`, and `releaseBudget` refunds only `container-never-started` --
198
+ // this container started. So each such job spends two job-count slots and buys nothing with either,
199
+ // and a cron-driven deployment empties its daily cap before anyone reads the first failure. That cost
200
+ // is what makes this a pre-spend gate rather than a doc: measured at three provider attempts,
201
+ // `Request timed out.`, exit 1, ~40 seconds, zero tokens (docs/egress.md).
202
+ //
203
+ // A RETURN, never a throw (CONST-RETRY-INFRA-ONLY): retrying never makes an absent proxy appear.
204
+ const egress = await egressPreflight(job);
205
+ if (egress.proxyMissing || egress.proxyStopped) {
206
+ const proxy = egress.proxyMissing ?? egress.proxyStopped;
207
+ const state = egress.proxyMissing ? "is not on this host" : "is not running";
208
+ await comment(job, `Refused: this deployment runs jobs behind an egress policy and its allowlist proxy "${proxy}" ${state}, so the job could not reach the provider and would burn its budget slot proving it. Start it with \`docker compose -f deploy/docker-compose.yml --profile egress up -d\`, or set PI_EGRESS=0 to run without an egress policy. Not run.`);
209
+ // The proxy's NAME is operator-authored deployment config, never payload -- the same PII class as
210
+ // the image ref on the refusal above.
211
+ log(egress.proxyMissing ? "refused_egress_proxy_missing" : "refused_egress_proxy_stopped", { proxy });
212
+ return {
213
+ outcome: "policy",
214
+ reason: egress.proxyMissing ? "egress-proxy-missing" : "egress-proxy-stopped",
215
+ exitCode: null,
216
+ turns: null,
217
+ tokens: null,
218
+ provider: job.provider ?? null,
219
+ model: job.model ?? null,
220
+ budgetReserved: false, // refused before reserveBudget, so no job-count slot was consumed
221
+ };
222
+ }
223
+ if (egress.unavailable) {
224
+ // The daemon did not answer, so this is indeterminate rather than a refusal -- the same
225
+ // determinate/indeterminate split the image preflight draws one gate up, and thrown for the same
226
+ // reason. Pre-reserve, so the refund below is a no-op and still honest if this gate ever moves.
227
+ throw new InfraRetry("docker unavailable, egress preflight could not run", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
228
+ }
229
+
170
230
  // REQ-RESUMABLE-SESSION's one fail-CLOSED case. Everything else in that feature fails OPEN and
171
231
  // NAMES itself -- absent, expired, too-large, unparseable, locked, promote-failed -- because a cold
172
232
  // start is a correct run. This one cannot be: with no `sessionsDir`, resolveSession returns null
package/src/queue.mjs CHANGED
@@ -16,11 +16,16 @@ export function makeQueue(connection) {
16
16
  * removeOnComplete keeps the dedup window ~= the retention. Unlike webhooks, local jobs are not
17
17
  * redelivered, so a modest window is enough.
18
18
  */
19
- export async function enqueueLocalJob(queue, { folder, flow, task, provider, model, maxTurns, image, skillsDir, chainDepth, parentJobId, jobId, now = new Date() }) {
19
+ export async function enqueueLocalJob(queue, { folder, flow, task, command, provider, model, maxTurns, image, skillsDir, chainDepth, parentJobId, jobId, now = new Date() }) {
20
20
  const minute = now.toISOString().slice(0, 16); // YYYY-MM-DDTHH:MM -- the dedup window
21
21
  // A caller-supplied jobId (the outbox collector's retry-idempotent chainedJobId) wins; otherwise the
22
- // minute-windowed localJobId is the dedup key.
23
- const id = jobId ?? localJobId({ folder, flow, task, minute });
22
+ // minute-windowed localJobId is the dedup key. A command job (issue #189) fills the flow slot with
23
+ // `cmd:<command>` rather than leaving it empty: a command trigger carries no flow/task, so without it
24
+ // two DIFFERENT commands on one folder in one minute would hash identically and the second would
25
+ // vanish silently -- and the `cmd:` prefix keeps a command named X from colliding with a flow named X
26
+ // (`:` is outside the skill-name charset, so no real flow can spell the prefixed form). A flow job's
27
+ // key is byte-identical to before the feature.
28
+ const id = jobId ?? localJobId({ folder, flow: command !== undefined ? `cmd:${command}` : flow, task, minute });
24
29
  // image/chainDepth/parentJobId land on `data` only when present, so a plain non-chained job's data is
25
30
  // byte-identical. `image` is the container image this job runs in (INT-TRIGGERS-FILE-CONTRACT); absent
26
31
  // resolves the deployment default at job start, never a value frozen here.
@@ -29,6 +34,10 @@ export async function enqueueLocalJob(queue, { folder, flow, task, provider, mod
29
34
  folder,
30
35
  flow,
31
36
  task,
37
+ // The registered pi command this job dispatches instead of a flow (issue #189). Conditional like
38
+ // `image`, so a flow job's data keeps exactly the keys it has today; the parse-level XOR means a
39
+ // job carrying it has flow/task undefined, which JSON serialization drops.
40
+ ...(command !== undefined && { command }),
32
41
  provider,
33
42
  model,
34
43
  maxTurns,
@@ -115,7 +124,7 @@ export async function enqueueGitLabJob(queue, fields) {
115
124
  * window, replicas never coalesce against each other, and an unflagged job's dedup id is the same string it
116
125
  * has always been.
117
126
  */
118
- export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, target, flow, trigger, provider, model, maxTurns, packages, image, skillsDir, instructions, resume, replica, replicas }) {
127
+ export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, target, flow, command, trigger, provider, model, maxTurns, packages, image, skillsDir, instructions, resume, replica, replicas }) {
119
128
  const jobId = forgeDeliveryJobId(kind, trigger?.deliveryId, replica);
120
129
  // `packages` (whether to load the operator-staged pi packages) and `image` (which container image to run)
121
130
  // come off the MATCHED trigger (INT-TRIGGERS-FILE-CONTRACT / REQ-GLOBAL-PI-OVERLAY) and land on `data`
@@ -132,6 +141,11 @@ export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, tar
132
141
  ...(azure !== undefined && { azure }),
133
142
  target,
134
143
  flow,
144
+ // The registered pi command this trigger dispatches instead of a flow (issue #189). Conditional
145
+ // like `packages`/`image` below, so an unflagged trigger's job data is byte-identical -- and at
146
+ // JOB level, never inside `trigger`, for their reason too: an execution knob is not a fact about
147
+ // the delivery, and `trigger` is copied verbatim into /job/event.json.
148
+ ...(command !== undefined && { command }),
135
149
  trigger,
136
150
  provider,
137
151
  model,
@@ -156,7 +170,13 @@ export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, tar
156
170
  };
157
171
  await queue.add(kind, data, {
158
172
  jobId,
159
- deduplication: { id: `${repo}${targetSeparator(kind, target?.type)}${target.number}:${flow}${replica !== undefined ? `:r${replica}` : ""}`, ttl: SEMANTIC_WINDOW_MS }, // ttl in ms
173
+ // A command job (issue #189) fills the semantic key's flow slot with `cmd:<command>`: a command
174
+ // trigger carries no flow, so the slot would otherwise read `undefined` for every command and one
175
+ // command's 10-minute window would swallow a different command's delivery on the same target. The
176
+ // `cmd:` prefix keeps a command named X from coalescing against a flow named X -- `:` is outside
177
+ // the skill-name charset, so no real flow can spell the prefixed form -- and a flow job's key
178
+ // stays byte-identical to before the feature.
179
+ deduplication: { id: `${repo}${targetSeparator(kind, target?.type)}${target.number}:${command !== undefined ? `cmd:${command}` : flow}${replica !== undefined ? `:r${replica}` : ""}`, ttl: SEMANTIC_WINDOW_MS }, // ttl in ms
160
180
  attempts: 2,
161
181
  backoff: { type: "exponential", delay: 60_000 },
162
182
  removeOnComplete: { age: 31 * 24 * 3600 }, // age in seconds -- do not cross units with the ms ttl above
@@ -1,5 +1,6 @@
1
1
  import { spawn } from "node:child_process";
2
2
  import { buildDockerRunArgs, CONTAINER_SESSION_FILE } from "./docker-run.mjs";
3
+ import { createJobNetwork, networkNameFor, removeJobNetwork } from "./egress.mjs";
3
4
  import { buildContainerEnv } from "./env-allowlist.mjs";
4
5
  import { resolveJobImage } from "./image-preflight.mjs";
5
6
  import { InfraRetry } from "./processor.mjs";
@@ -40,6 +41,8 @@ export function makeRunContainer({
40
41
  forwardEnv = [],
41
42
  authFromPi = false, // fall back to ~/.pi/agent/auth.json for the provider key when the env has none
42
43
  forgeHosts = {}, // per-forge self-hosted instance URLs, so a forge CLI in the container talks to the right one
44
+ egress = false, // REQ-EGRESS-ALLOWLIST: put this job on its own --internal network behind the allowlist proxy
45
+ egressProxy, // the proxy component attached to that network; undefined = egress.mjs's default name
43
46
  }) {
44
47
  // async so a synchronous throw (e.g. buildContainerEnv on an unconfigured provider) surfaces as
45
48
  // a rejection, uniformly awaitable by the processor and by tests.
@@ -59,6 +62,8 @@ export function makeRunContainer({
59
62
  forgeKind: job?.kind,
60
63
  forgeHosts,
61
64
  hostEnv,
65
+ egress, // REQ-EGRESS-ALLOWLIST: emits HTTPS_PROXY/HTTP_PROXY/NO_PROXY/NODE_USE_ENV_PROXY, or nothing
66
+ egressProxy,
62
67
  allowGlobalExtensions, // REQ-GLOBAL-PI-OVERLAY: false emits the explicit PI_GLOBAL_ALLOW_EXTENSIONS=0 opt-out
63
68
  // REQ-GLOBAL-PI-OVERLAY: the per-job value comes off `job` (like maxTurns), the staged set off
64
69
  // the closure (like allowGlobalExtensions) -- so a trigger can withhold what the operator staged.
@@ -74,9 +79,20 @@ export function makeRunContainer({
74
79
  // The constant is imported rather than re-typed so the mount below and this variable name one
75
80
  // path -- two literals is how they drift with both suites green.
76
81
  sessionFile: prepared.session ? CONTAINER_SESSION_FILE : undefined,
82
+ // Issue #189: the flow name, structurally, so the runner can verify it against the loaded
83
+ // skill set. Off `job` like maxTurns; absent (a bare run.task cron job) emits no variable.
84
+ flow: typeof job.flow === "string" && job.flow.trim() !== "" ? job.flow : undefined,
85
+ // Issue #189: the command name, structurally, so the runner can refuse an unregistered one
86
+ // before any spend (command-unregistered). Same guard shape as `flow` directly above, and
87
+ // mutually exclusive with it by parse -- a job carries one or the other, never both.
88
+ command: typeof job.command === "string" && job.command.trim() !== "" ? job.command : undefined,
77
89
  authFromPi, // source the provider key from pi's auth.json when the env has none
78
90
  });
79
91
 
92
+ // `-net` on this container's own name (egress.mjs). null when no policy is armed, and docker-run's
93
+ // guard then omits the flag entirely, so the argv is byte-identical to one built before this feature.
94
+ const network = egress ? networkNameFor(name) : null;
95
+
80
96
  const args = buildDockerRunArgs({
81
97
  // Same split as packagePaths above: the per-job value off `job`, the deployment value off the closure,
82
98
  // so a trigger can name its own toolchain (INT-TRIGGERS-FILE-CONTRACT). Resolved through the SAME
@@ -92,13 +108,26 @@ export function makeRunContainer({
92
108
  sessionDir: prepared.session?.hostDir,
93
109
  globalPiDir, // undefined/null -> docker-run's guard skips the /opt/pi-global mount
94
110
  name,
111
+ network, // REQ-EGRESS-ALLOWLIST: null when no policy is armed, and the flag is then absent
95
112
  });
96
113
 
114
+ // REQ-EGRESS-ALLOWLIST. This job's own --internal network, created here rather than at boot because
115
+ // it holds exactly two endpoints -- this container and the proxy -- and that is what makes job-to-job
116
+ // traffic structurally impossible rather than merely discouraged. A shared network could not do it:
117
+ // `enable_icc=false` would block job-to-job AND job-to-proxy, since ICC governs every container pair
118
+ // on the bridge and the proxy is a container.
119
+ //
120
+ // A failure to build it is INFRA, not policy: nothing has been spent, a retry may well succeed, and
121
+ // `container-never-started` is literally true, so the reservation is given back (processor.mjs).
122
+ if (network && !(await createJobNetwork(spawnFn, { network, proxy: egressProxy }))) {
123
+ throw new InfraRetry("container-never-started", { reason: "container-never-started" });
124
+ }
125
+
97
126
  // Host-side per-job log sink, teed off `onOutput`. `name` is `pi-job-<jobId>`; the sink
98
127
  // sanitizes internally. No container mount, no env var -- the sink lives on this side only.
99
128
  const sink = openJobLog(name);
100
129
 
101
- return await new Promise((resolve, reject) => {
130
+ const run = new Promise((resolve, reject) => {
102
131
  const child = spawnFn("docker", args, { stdio: ["ignore", "pipe", "pipe"] });
103
132
  // A throwing sink.write is swallowed so a misbehaving sink cannot break the tee or hang the run.
104
133
  const tee = (chunk) => {
@@ -134,5 +163,14 @@ export function makeRunContainer({
134
163
  resolve(aborted ? { code: code ?? 137, aborted: true, turns, tokens, session, usage } : { code: code ?? 1, aborted: false, turns, tokens, session, usage });
135
164
  });
136
165
  });
166
+
167
+ // The network outlives the container by exactly this `finally`. Best-effort and never throwing: the
168
+ // container has already exited, its code is the job's answer, and a teardown fault must not rewrite
169
+ // that answer. What a failure leaves behind is a memberless network, which the boot reaper sweeps.
170
+ try {
171
+ return await run;
172
+ } finally {
173
+ if (network) await removeJobNetwork(spawnFn, { network, proxy: egressProxy });
174
+ }
137
175
  };
138
176
  }
@@ -246,7 +246,7 @@ function rebuildUsage(u) {
246
246
  * path embeds the operator's OS account name.
247
247
  *
248
248
  * `reason` is a fixed enum passthrough (worker-abort | over-budget | unprotected-branch |
249
- * runner-policy | job-image-missing | ...), never free-form or payload text. `exitCode`, `turns`, and `budgetReserved`
249
+ * runner-policy | job-image-missing | egress-proxy-missing | ...), never free-form or payload text. `exitCode`, `turns`, and `budgetReserved`
250
250
  * default to `null` when the outcome does not carry them, so the record shape is stable whether or not
251
251
  * the source reports those fields.
252
252
  */
@@ -1,6 +1,8 @@
1
+ import { spawn } from "node:child_process";
1
2
  import { parseArgs } from "node:util";
2
3
  import { loadConfig } from "./config.mjs";
3
4
  import { sanitizeJobId } from "./run-history.mjs";
5
+ import { createJobNetwork, egressEnv, networkNameFor, removeJobNetwork } from "./egress.mjs";
4
6
  import { buildSandboxRunArgs, launchSandbox, listRunningSandboxes, parsePublish, resolveSandbox, sandboxContainerName } from "./sandbox.mjs";
5
7
  import { listSandboxes, pinSandbox } from "./sandbox-store.mjs";
6
8
 
@@ -22,6 +24,9 @@ export async function runSandbox(argv = [], { env = process.env, deps = {} } = {
22
24
  isTty = Boolean(process.stdin.isTTY && process.stdout.isTTY),
23
25
  running = listRunningSandboxes,
24
26
  launch = launchSandbox,
27
+ // The docker spawn used for this session's egress network, seamed like `launch` so the tests never
28
+ // touch a daemon. Not used when PI_EGRESS=0.
29
+ spawnNetwork = spawn,
25
30
  now = () => Date.now(),
26
31
  } = deps;
27
32
 
@@ -89,6 +94,10 @@ export async function runSandbox(argv = [], { env = process.env, deps = {} } = {
89
94
  else err(`warning: could not pin ${jobId}: ${pinned.reason}\n`);
90
95
  }
91
96
 
97
+ // REQ-EGRESS-ALLOWLIST: this session's own network, exactly like a job's, named off its own container
98
+ // so the reaper's `pi-job-` filter never touches it -- a worker restart must not tear the network out
99
+ // from under a shell an operator is sitting in.
100
+ const network = config.egress ? networkNameFor(resolved.name) : null;
92
101
  const args = buildSandboxRunArgs({
93
102
  image: resolved.manifest.image,
94
103
  name: resolved.name,
@@ -97,15 +106,27 @@ export async function runSandbox(argv = [], { env = process.env, deps = {} } = {
97
106
  publish,
98
107
  term: env.TERM,
99
108
  idleSeconds: config.sandboxIdleMinutes * 60,
109
+ network,
110
+ egressEnv: egressEnv({ proxy: config.egressProxy, armed: config.egress }),
100
111
  });
101
112
 
102
113
  out(`opening ${resolved.name} — image ${resolved.manifest.image}, workspace ${resolved.manifest.workspace}\n`);
103
114
  out("no credentials are set in this container. exit the shell to dispose of it.\n");
104
115
  if (publish.length > 0) out(`published: ${publish.filter((f) => f !== "-p").join(", ")}\n`);
105
116
 
106
- const { code, error } = await launch({ args });
107
- if (error) return fail(err, `could not start docker: ${error.message}`);
108
- return code ?? 0;
117
+ // No pre-spend gate here, deliberately: that is a MONEY gate and a sandbox spends nothing. A missing
118
+ // proxy fails at `docker run` with docker's own message, in front of an operator at a terminal, which
119
+ // is the one place a late failure is cheap.
120
+ if (network && !(await createJobNetwork(spawnNetwork, { network, proxy: config.egressProxy }))) {
121
+ return fail(err, `could not create the egress network ${network} -- is the proxy running? \`docker compose -f deploy/docker-compose.yml --profile egress up -d\``);
122
+ }
123
+ try {
124
+ const { code, error } = await launch({ args });
125
+ if (error) return fail(err, `could not start docker: ${error.message}`);
126
+ return code ?? 0;
127
+ } finally {
128
+ if (network) await removeJobNetwork(spawnNetwork, { network, proxy: config.egressProxy });
129
+ }
109
130
  }
110
131
 
111
132
  /**
package/src/sandbox.mjs CHANGED
@@ -79,8 +79,10 @@ function inPortRange(n) {
79
79
  * @param publish already-parsed `-p` flags
80
80
  * @param term the host's TERM, so the shell renders
81
81
  * @param idleSeconds bash's own TMOUT; 0 omits it
82
+ * @param network this session's own egress network (REQ-EGRESS-ALLOWLIST); null = the default bridge
83
+ * @param egressEnv the proxy variables that go with it, or {} when no policy is armed
82
84
  */
83
- export function buildSandboxRunArgs({ image, name, workspace, jobDir, publish = [], term, idleSeconds = 0 }) {
85
+ export function buildSandboxRunArgs({ image, name, workspace, jobDir, publish = [], term, idleSeconds = 0, network = null, egressEnv: proxyEnv = {} }) {
84
86
  return buildDockerRunArgs({
85
87
  image,
86
88
  name,
@@ -89,9 +91,20 @@ export function buildSandboxRunArgs({ image, name, workspace, jobDir, publish =
89
91
  // The ONLY two variables, and neither is a credential. TERM so the shell renders; TMOUT so a
90
92
  // forgotten session closes itself. `buildDockerRunArgs` skips undefined, so an unset TERM or a
91
93
  // disabled idle timeout emits nothing rather than an empty string.
94
+ // A sandbox joins the SAME kind of network a job did, by the same builder, so the boundary cannot
95
+ // land on job containers and miss this one. Leaving sandboxes on the default bridge was the tempting
96
+ // alternative and it is the wrong one: it reads as a convenience (install a missing dependency while
97
+ // debugging) and it is a WIDER reach than the run the sandbox exists to reproduce. A shell that can
98
+ // go where the run could not is not reproducing the run. Nothing an operator wants is lost, because
99
+ // the forge and the registry are on the allowlist a job needed anyway.
100
+ network,
92
101
  env: {
93
102
  TERM: term || undefined,
94
103
  TMOUT: idleSeconds > 0 ? String(idleSeconds) : undefined,
104
+ // Still NO CREDENTIALS, and that clause is untouched: a proxy URL is not a credential, and
105
+ // buildContainerEnv is still not reused here. The env is two variables about the terminal and,
106
+ // when a policy is armed, three about the network.
107
+ ...proxyEnv,
95
108
  },
96
109
  // Ahead of the env and the mounts, and well ahead of the image, which buildDockerRunArgs keeps as
97
110
  // the final positional. `--entrypoint` also clears the image's CMD; this repo's Dockerfile sets
package/src/schedules.mjs CHANGED
@@ -68,7 +68,13 @@ function normalizeCronSchedule({ on, run }, path, existsSync) {
68
68
  // cron-only field: it is carried into the local `/job/event.json` (INT-CONTAINER-JOB-INPUTS) so a
69
69
  // scheduled job can name its own trigger; the INT-TRIGGERS-FILE-CONTRACT byte-match acceptance is
70
70
  // amended for exactly this field.
71
- const data = { kind: "local", folder: run.folder, flow: run.flow, task: run.task, provider: run.provider, model: run.model, maxTurns: run.maxTurns, github: run.github, packages: run.packages, image: run.image, ...(run.skillsDir !== undefined && { skillsDir: run.skillsDir }), resume: run.resume, trigger: { id: on.id, pattern: on.pattern } };
71
+ //
72
+ // `command` (issue #189) is conditional like `skillsDir`, not present-and-undefined like flow/task,
73
+ // and both spellings serve the same byte-identity: a flow trigger's stored repeatable must not grow a
74
+ // key. A command trigger carries no flow/task at all (the validator enforces the XOR), so those two
75
+ // keys hold undefined here and drop at JSON serialization -- the command schedule's data is exactly
76
+ // kind/folder/command plus the shared fields.
77
+ const data = { kind: "local", folder: run.folder, flow: run.flow, task: run.task, ...(run.command !== undefined && { command: run.command }), provider: run.provider, model: run.model, maxTurns: run.maxTurns, github: run.github, packages: run.packages, image: run.image, ...(run.skillsDir !== undefined && { skillsDir: run.skillsDir }), resume: run.resume, trigger: { id: on.id, pattern: on.pattern } };
72
78
  // Retention only; the deterministic repeat:<id>:<millis> jobId supplies dedup, so no jobId here, and
73
79
  // scheduler jobs are not retried (DES-CRON-VIA-BULLMQ-SCHEDULER) so no attempts/backoff.
74
80
  const opts = { removeOnComplete: { age: 24 * 3600 }, removeOnFail: { age: 7 * 24 * 3600 } };