@edgehero/pi-dispatch 1.2.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/.env.example CHANGED
@@ -89,6 +89,17 @@ PI_JOB_IMAGE=pi-job:latest # the DEFAULT job image. Any trigger may nam
89
89
  # PI_FORWARD_ENV= # comma-separated extra env var NAMES to forward into the container (e.g. a CUSTOM provider's key). Explicit allowlist, not a pass-through.
90
90
  # GITHUB_TOKEN/GH_TOKEN are refused here -- the worker mints per-job tokens
91
91
 
92
+ # --- Per-trigger vault secrets (docs/secrets.md, issue #225) ---
93
+ # A trigger may name references ("secrets": { "STRIPE_KEY": "op://ci/stripe/api-key" }) and the profile that
94
+ # resolves them. The worker runs YOUR script once per reference, on the HOST, before the container starts.
95
+ # The job receives values; it never holds your manager's credential and cannot enumerate your vault.
96
+ # PI_SECRET_PROFILES= # name:/absolute/path pairs, comma separated (each entry splits on its FIRST colon, so a Windows C:\ path parses).
97
+ # A resolver is one line: `exec op read --no-newline "$1"`, `exec pass show "$1"`, `exec vault kv get -field=... "$1"`.
98
+ # Exit 2 if the reference is wrong, exit 1 if you could not reach your manager (that one retries). Unset = feature off.
99
+ # PI_SECRET_RESOLVER_ROOTS= # OS-path-delimited (; on Windows, : elsewhere) directories a PANEL-declared resolver may live in.
100
+ # Default empty = fail-closed: `/dispatch secrets add` can declare nothing, and only PI_SECRET_PROFILES above is honoured.
101
+ # PI_SECRET_RESOLVE_TIMEOUT_MS= # default 10000, per reference. Sits before a paid container and is multiplied by the reference count, so tighter than doctor's 30s.
102
+
92
103
  PI_SCHEDULER_STALL_MAX=2 # tear down a scheduler after N consecutive stalls (money backstop)
93
104
 
94
105
  # --- Egress policy: what a job container may reach on the network (docs/egress.md) ---
@@ -49,7 +49,11 @@ services:
49
49
  PI_TRIGGERS_FILE: /config/triggers.json
50
50
  # The repo-root triggers.json (`pi-dispatch init` scaffolds it -- run that first: mounting a path
51
51
  # that does not exist makes Docker create it as a DIRECTORY and the boot fails confusingly).
52
- # Read-only: the receiver live-reloads this file on change; it never writes it.
52
+ # Read-only: the receiver live-reloads this file on change; it never writes it. One consequence of
53
+ # a single-FILE bind mount (issue #231): every write to this file is an atomic tmp+rename that
54
+ # SWAPS THE INODE, and the mount stays pinned to the old one -- so the worker's one-shot disarm is
55
+ # invisible in here until the container restarts, and the worker's own pre-spend check is what
56
+ # keeps a spent one-shot from running again in the meantime. A restart picks up the current file.
53
57
  volumes:
54
58
  - ../triggers.json:/config/triggers.json:ro
55
59
  # Loopback only, like Valkey's port above: the operator's reverse proxy or tunnel (TLS, public
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@edgehero/pi-dispatch",
3
- "version": "1.2.0",
3
+ "version": "1.4.0",
4
4
  "type": "module",
5
5
  "description": "Self-hosted job harness for the pi coding agent: a BullMQ worker that drains the queue, mints scoped forge tokens, and runs one container per job — plus the pi-dispatch CLI (init, up, doctor, service).",
6
6
  "keywords": [
@@ -52,6 +52,7 @@
52
52
  "./job-id": "./src/job-id.mjs",
53
53
  "./forges": "./src/forges.mjs",
54
54
  "./triggers": "./src/triggers.mjs",
55
+ "./triggers-file": "./src/triggers-file.mjs",
55
56
  "./packages": "./src/packages.mjs",
56
57
  "./pause-windows": "./src/pause-windows.mjs",
57
58
  "./identity": "./src/identity.mjs",
package/src/config.mjs CHANGED
@@ -9,6 +9,7 @@ import { existsSync } from "node:fs";
9
9
  import { delimiter } from "node:path";
10
10
  import { DEFAULT_EGRESS_PROXY, egressArmed } from "./egress.mjs";
11
11
  import { MINTED_TOKEN_VARS } from "./forges.mjs";
12
+ import { parseSecretProfiles } from "./secret-profiles.mjs";
12
13
 
13
14
  export function configError(message) {
14
15
  const error = new Error(message);
@@ -278,6 +279,22 @@ export function loadConfig(env = process.env, { fileExists = existsSync } = {})
278
279
  chainMaxPerJob: nonNegativeInt(env, "PI_CHAIN_MAX_PER_JOB", CHAIN_MAX_PER_JOB_DEFAULT), // INT-OUTBOX-CONTRACT: max request-<n>.json collected per parent
279
280
  dispatchRunPerHour: nonNegativeInt(env, "PI_DISPATCH_RUN_PER_HOUR", 3), // DES-ADMIN-VIA-PI-EXTENSION; 0 = disable dispatch_run
280
281
  dispatchRunRoots: delimitedList(env.PI_DISPATCH_RUN_ROOTS), // DES-AI-TRIGGER-FLOW-GATE: default [] fails closed — no folder passes, dispatch_run refuses everything
282
+ // REQ-TRIGGER-SECRETS. The operator's declared resolvers, `name:absolute-path` pairs, comma separated.
283
+ // Each entry splits on its FIRST colon so a Windows `C:\...` path survives -- the same drive-letter
284
+ // hazard `delimitedList` above exists for, arriving from the other side. Unset = the feature is off and
285
+ // any trigger naming secrets refuses pre-spend, which is why there is no default profile to fall into.
286
+ secretProfiles: parseSecretProfiles(env.PI_SECRET_PROFILES),
287
+ // The directories a resolver may live in. Default [] FAILS CLOSED exactly as dispatchRunRoots does, and
288
+ // for a sharper version of its reason: this bounds paths that can arrive from the settings overlay,
289
+ // which is not the reviewed artifact `triggers.json` is. Unset means the panel can declare no profile
290
+ // at all and only PI_SECRET_PROFILES above is honoured. Env-only, never the overlay and never the
291
+ // deployment pointer: `deployment-pointer.mjs` already refuses to carry PI_DISPATCH_RUN_ROOTS
292
+ // "because a pointer that could widen the AI-run folder allowlist would be a second, unreviewed door",
293
+ // and a bound that can be widened from the surface it bounds is not a bound.
294
+ secretResolverRoots: delimitedList(env.PI_SECRET_RESOLVER_ROOTS),
295
+ // Per-reference ceiling. Tighter than doctor's 30s on purpose: this runs before a paid container, is
296
+ // multiplied by the reference count, and holds a PI_CONCURRENCY slot while it waits.
297
+ secretResolveTimeoutMs: positiveInt(env, "PI_SECRET_RESOLVE_TIMEOUT_MS", 10000),
281
298
  github: { ...loadGitHubAuth(env, fileExists), allowGhResume: env.PI_SESSIONS_ALLOW_GH_SOURCE === "1" },
282
299
  gitlab: loadGitLabAuth(env),
283
300
  forgejo: loadForgejoAuth(env),
package/src/doctor.mjs CHANGED
@@ -46,7 +46,7 @@
46
46
  */
47
47
  import { chmodSync, closeSync, existsSync, lstatSync, mkdirSync, mkdtempSync, openSync, readdirSync, readFileSync, readSync, rmSync, statSync } from "node:fs";
48
48
  import { homedir, tmpdir } from "node:os";
49
- import { dirname, join } from "node:path";
49
+ import { dirname, join, delimiter } from "node:path";
50
50
  import { fileURLToPath } from "node:url";
51
51
  import { spawn as nodeSpawn } from "node:child_process";
52
52
  import { defaultSandboxDir, globalExtensionsEnabled } from "./config.mjs";
@@ -58,6 +58,7 @@ import { copySkillTree } from "./copy-tree.mjs";
58
58
  import { SKILL_NAME_RE } from "./flow-gate.mjs";
59
59
  import { DEFAULT_EGRESS_PROXY, egressArmed } from "./egress.mjs";
60
60
  import { installedUnitPaths, readUnitSeam } from "./service.mjs";
61
+ import { parseSecretProfiles } from "./secret-profiles.mjs";
61
62
  import { parseTriggers } from "./triggers.mjs";
62
63
 
63
64
  const NODE_FLOOR = [22, 19]; // pi's engine floor (22.19.0)
@@ -263,7 +264,7 @@ export async function collectChecks(env, seams) {
263
264
  // image checks just below, and `optingOut`/`requiring` colour the staged-packages lines further down.
264
265
  // `optingOut` counts the only value that withholds the staged set; `requiring` counts an explicit
265
266
  // run.packages: true, which arms nothing any more but is still an operator statement of intent.
266
- const { requiring, optingOut, resuming, replicating, instructing, commands, images, skillsDirs, forges, repositories, flows, parseError, path: triggersFilePath } = readTriggerFacts(env, fileExists, cwd);
267
+ const { requiring, optingOut, resuming, replicating, instructing, commands, secreting, onceArmed, onceSpent, secretProfiles, localSecretFolders, images, skillsDirs, forges, repositories, flows, parseError, path: triggersFilePath } = readTriggerFacts(env, fileExists, cwd);
267
268
  // FIRST, and fail rather than warn: every check below this line reads counts that a parse failure
268
269
  // zeroed, so a green run here would be reporting on a file nobody could read. The receiver loads this
269
270
  // file unconditionally and refuses to start without it, which is the consequence worth naming.
@@ -1096,6 +1097,52 @@ export async function collectChecks(env, seams) {
1096
1097
  });
1097
1098
  }
1098
1099
 
1100
+ // REQ-TRIGGER-SECRETS. Only reported when a trigger actually binds one, on the run.resume block's
1101
+ // reasoning below: a deployment that uses no secrets should not be told about a variable it has no
1102
+ // reason to set.
1103
+ if (secreting > 0) {
1104
+ const declared = parseSecretProfilesSafe(env.PI_SECRET_PROFILES);
1105
+ const names = Object.keys(declared).sort();
1106
+ if (declared.error) {
1107
+ checks.push({ ok: false, label: "PI_SECRET_PROFILES does not parse", fix: `${declared.error} -- the worker refuses to boot until this is fixed, rather than dropping the entry and leaving you a profile you believe is wired` });
1108
+ } else {
1109
+ // A HARD FAIL, not a warning, and worded like the run.resume/PI_SESSIONS_DIR check below for the
1110
+ // same reason: these jobs refuse pre-spend until it is set, deliberately, rather than running
1111
+ // without their secrets and looking like they worked.
1112
+ const missing = secretProfiles.filter((name) => !(name in declared));
1113
+ checks.push({
1114
+ ok: missing.length === 0,
1115
+ label: missing.length === 0 ? `${secreting} trigger(s) bind secrets, and every profile they name is declared` : `${secreting} trigger(s) bind secrets, but ${missing.length} named profile(s) are not declared: ${missing.join(", ")}`,
1116
+ fix: `declare them in PI_SECRET_PROFILES as name:/absolute/path pairs (a resolver is one line, e.g. \`exec op read --no-newline "$1"\`) -- these jobs refuse pre-spend until you do`,
1117
+ });
1118
+ // The declared table, so an operator sees what is wired without reading .env. NAMES and paths only:
1119
+ // this is doctor's own stdout on the operator's host, not a public issue comment.
1120
+ if (names.length > 0) {
1121
+ checks.push({ ok: true, label: `Secret resolver profiles declared: ${names.map((n) => `${n} -> ${declared[n]}`).join(", ")}` });
1122
+ }
1123
+ // The panel-authoring bound. Unset is the SAFE default rather than a defect, so this is a fact line
1124
+ // when closed and a disclosure when open.
1125
+ const roots = (env.PI_SECRET_RESOLVER_ROOTS ?? "").split(delimiter).map((r) => r.trim()).filter(Boolean);
1126
+ checks.push({
1127
+ ok: true,
1128
+ ...(roots.length === 0
1129
+ ? { label: "PI_SECRET_RESOLVER_ROOTS is unset, so only PI_SECRET_PROFILES declares resolvers (the panel can declare none)" }
1130
+ : { warn: true, label: `PI_SECRET_RESOLVER_ROOTS admits panel-declared resolvers under: ${roots.join(", ")}`, fix: "keep those directories writable by nobody but the account the worker runs as: whoever can write a resolver there can run code as the worker" }),
1131
+ });
1132
+ }
1133
+ // The local-workspace disclosure. Not a failure: a nightly deploy binding a secret is exactly what
1134
+ // this feature is for. But a local job's /workspace IS the folder, read-write and un-cloned, so an
1135
+ // agent that persists a credential to make its next command simpler writes it into a real repository.
1136
+ if (localSecretFolders.length > 0) {
1137
+ checks.push({
1138
+ ok: true,
1139
+ warn: true,
1140
+ label: `${localSecretFolders.length} local trigger(s) bind secrets and run IN the operator's own folder: ${localSecretFolders.join(", ")}`,
1141
+ fix: "a local job edits that folder in place, so a credential the agent writes to .env, .netrc or .git-credentials lands in your real repository (and survives in a retained sandbox for PI_SANDBOX_RETENTION_HOURS). Nothing scans for that: keep those folders out of anything you push",
1142
+ });
1143
+ }
1144
+ }
1145
+
1099
1146
  // REQ-RESUMABLE-SESSION. Only reported when a trigger actually asked for it: a deployment that does
1100
1147
  // not use resume should not be told about a directory it has no reason to create.
1101
1148
  if (resuming > 0) {
@@ -1195,6 +1242,31 @@ export async function collectChecks(env, seams) {
1195
1242
  });
1196
1243
  }
1197
1244
 
1245
+ // One-shot close triggers (issue #231, DES-ONE-SHOT-DISARM-IN-THE-FILE). Advisory only -- doctor
1246
+ // never touches triggers -- and counted from the RAW file (readTriggerFacts says why). Two lines
1247
+ // with different lives: the armed line names the count and, when PI_TRIGGERS_FILE is unset, warns
1248
+ // that the disarm resolves ./triggers.json against the WORKER SERVICE's working directory -- a
1249
+ // service unit whose WorkingDirectory differs from the receiver's would disarm a file nobody
1250
+ // matches against, the split-file hazard no mechanism can detect. The spent line states the
1251
+ // deliberate degradation: a spent entry counts toward NO parsed fact above (forges, flows,
1252
+ // webhook-secret), mirroring what the receiver serves at its next boot.
1253
+ if (onceArmed > 0) {
1254
+ checks.push({
1255
+ ok: true,
1256
+ warn: env.PI_TRIGGERS_FILE === undefined,
1257
+ label: `${onceArmed} one-shot trigger(s) armed (on.once) -- the worker disarms the entry in ${env.PI_TRIGGERS_FILE === undefined ? "./triggers.json resolved against the worker service's working directory; set PI_TRIGGERS_FILE so worker and receiver name the same file from anywhere" : "PI_TRIGGERS_FILE"} after the run record exists`,
1258
+ fix: "set PI_TRIGGERS_FILE to an absolute path in both services' environments",
1259
+ });
1260
+ }
1261
+ if (onceSpent > 0) {
1262
+ checks.push({
1263
+ ok: true,
1264
+ warn: false,
1265
+ label: `${onceSpent} one-shot trigger(s) already spent (on.disarmed) -- spent entries match nothing and count toward no credential or flow check; delete on.disarmed to re-arm, or delete the entry once its history no longer matters`,
1266
+ fix: "",
1267
+ });
1268
+ }
1269
+
1198
1270
  // REQ-SCOPED-PAUSE-WINDOWS, the panel-writes-what-the-worker-ignores trap (issue #99). Three defaults
1199
1271
  // that are individually defensible and together silent:
1200
1272
  //
@@ -1410,16 +1482,40 @@ async function repoFlowAtHead(spawn, folder, flow) {
1410
1482
  return mode === "100644" && type === "blob" ? "present" : "absent";
1411
1483
  }
1412
1484
 
1485
+ /**
1486
+ * `parseSecretProfiles`, but doctor never throws. A malformed PI_SECRET_PROFILES is a finding to REPORT,
1487
+ * not a reason for the diagnostic tool to die: the operator running doctor is very likely running it
1488
+ * BECAUSE the worker refused to boot on that exact line, and a stack trace instead of a check is the least
1489
+ * useful possible answer. Returns the table, or `{ error }` carrying the parser's own message.
1490
+ */
1491
+ function parseSecretProfilesSafe(raw) {
1492
+ try {
1493
+ return parseSecretProfiles(raw);
1494
+ } catch (err) {
1495
+ return { error: err?.message ?? "unparseable" };
1496
+ }
1497
+ }
1498
+
1413
1499
  function readTriggerFacts(env, fileExists, cwd) {
1414
- const none = { requiring: 0, optingOut: 0, resuming: 0, replicating: 0, instructing: 0, commands: 0, images: [], skillsDirs: [], forges: [], repositories: [], flows: [], parseError: null, path: null };
1500
+ const none = { requiring: 0, optingOut: 0, resuming: 0, replicating: 0, instructing: 0, commands: 0, secreting: 0, onceArmed: 0, onceSpent: 0, secretProfiles: [], localSecretFolders: [], images: [], skillsDirs: [], forges: [], repositories: [], flows: [], parseError: null, path: null };
1415
1501
  try {
1416
1502
  // Unset falls back to ./triggers.json in cwd, MIRRORING the receiver's own default
1417
1503
  // (receiver/src/config.mjs) -- the two must read the same file, or doctor preflights a deployment
1418
1504
  // the receiver will not boot. An absent file still means "no triggers at all", exactly as before.
1419
1505
  const path = env.PI_TRIGGERS_FILE ?? join(cwd, "triggers.json");
1420
1506
  if (!fileExists(path)) return none;
1421
- const triggers = parseTriggers(readFileSync(path, "utf8"), path);
1507
+ const text = readFileSync(path, "utf8");
1508
+ const triggers = parseTriggers(text, path);
1509
+ // The one-shot facts are counted from the RAW entries, not the parsed records, because the
1510
+ // validator collapses a disarmed entry to a sentinel that carries neither `once` nor
1511
+ // `disarmed` -- exactly so nothing can match it -- which also erases it from every parsed
1512
+ // count above. Doctor is the surface that must still SEE the spent entry: "why did nothing
1513
+ // fire" is answered by a spent row, and only the raw file still holds it. Safe unguarded:
1514
+ // parseTriggers just accepted this same text, so JSON.parse cannot throw here.
1515
+ const rawEntries = JSON.parse(text)?.triggers ?? [];
1422
1516
  return {
1517
+ onceArmed: rawEntries.filter((t) => t?.on?.once === true && t.on.disarmed === undefined).length,
1518
+ onceSpent: rawEntries.filter((t) => t?.on?.disarmed !== undefined).length,
1423
1519
  requiring: triggers.filter((t) => t.run.packages === true).length,
1424
1520
  resuming: triggers.filter((t) => t.run.resume === true).length,
1425
1521
  // REQ-PER-TRIGGER-INSTRUCTION. Counted beside `resuming` for the same reason: it is a per-trigger
@@ -1432,6 +1528,17 @@ function readTriggerFacts(env, fileExists, cwd) {
1432
1528
  // tuple list already filters to `typeof f.flow === "string"`, so a command trigger drops out
1433
1529
  // of the flow-tier probes naturally -- no exclusion needed there.
1434
1530
  commands: triggers.filter((t) => typeof t.run.command === "string").length,
1531
+ // REQ-TRIGGER-SECRETS. Counted beside `instructing` for its reason: a per-trigger choice that
1532
+ // changes what every job of it can reach, and one that lives only in triggers.json.
1533
+ secreting: triggers.filter((t) => t.run.secrets !== undefined).length,
1534
+ // The distinct profile NAMES the file selects, deduped like `images`/`skillsDirs`: the checks below
1535
+ // cost a stat each, and two triggers naming one profile are one question. `default` is substituted
1536
+ // for an absent field so the table answers what the worker will actually look up.
1537
+ secretProfiles: [...new Set(triggers.filter((t) => t.run.secrets !== undefined).map((t) => t.run.secretsProfile ?? "default"))].sort(),
1538
+ // LOCAL triggers that bind secrets, by folder. A local job's /workspace IS this folder, bind-mounted
1539
+ // read-write with no clone, so a credential an agent writes into .env lands in the operator's real
1540
+ // repository rather than a temp dir that gets swept. Deduped for skillsDirs' reason.
1541
+ localSecretFolders: [...new Set(triggers.filter((t) => t.run.secrets !== undefined && t.run.kind === "local" && typeof t.run.folder === "string").map((t) => t.run.folder))].sort(),
1435
1542
  optingOut: triggers.filter((t) => t.run.packages === false).length,
1436
1543
  images: [...new Set(triggers.map((t) => t.run.image).filter((i) => typeof i === "string"))].sort(),
1437
1544
  // REQ-PER-TRIGGER-SKILLS. The distinct host directories the file names, deduped like `images`,
@@ -111,7 +111,7 @@ function resolveEnvName(provider, cred) {
111
111
  * `allowGlobalExtensions` defaults to TRUE here, matching loadConfig's default (REQ-GLOBAL-PI-OVERLAY): a
112
112
  * caller that says nothing gets the operator's staged setup, and only an explicit `false` withholds it.
113
113
  */
114
- export function buildContainerEnv({ provider, model, maxTurns, maxTokens, jobId, githubToken, forgeKind, forgeHosts = {}, hostEnv, allowGlobalExtensions = true, packagePaths = [], forwardEnv = [], sessionFile = null, flow = null, command = null, authFromPi = false, egress = false, egressProxy, agentDir, readFile = readFileSync }) {
114
+ export function buildContainerEnv({ provider, model, maxTurns, maxTokens, jobId, githubToken, forgeKind, forgeHosts = {}, hostEnv, allowGlobalExtensions = true, packagePaths = [], forwardEnv = [], secrets = {}, sessionFile = null, flow = null, command = null, authFromPi = false, egress = false, egressProxy, agentDir, readFile = readFileSync }) {
115
115
  // The provider credential(s), by pi's expected variable name(s) -- from the worker env, or (when
116
116
  // PI_AUTH_FROM_PI is set and the env has none) host-side from pi's auth.json. Throws (config) if
117
117
  // neither source yields one, which the processor turns into a pre-spend refusal.
@@ -182,6 +182,36 @@ export function buildContainerEnv({ provider, model, maxTurns, maxTokens, jobId,
182
182
  if (hostEnv[name] !== undefined) env[name] = hostEnv[name];
183
183
  }
184
184
 
185
+ // The trigger's own secrets (REQ-TRIGGER-SECRETS), resolved HOST-SIDE by the processor before anything
186
+ // spent and injected exactly the way the provider credential is -- never a vault credential handed to the
187
+ // container to fetch them itself. `docs/secrets.md`'s rule survives intact: what crosses the boundary is a
188
+ // value, and the thing that can FETCH values stays on the host.
189
+ //
190
+ // AFTER the PI_FORWARD_ENV loop, for that loop's own stated reason: a name on the operator's blanket host
191
+ // list must not silently outrank the specific reference this trigger declared.
192
+ //
193
+ // BEFORE the egress assign, and that direction is deliberate rather than incidental. A secret named
194
+ // HTTPS_PROXY that WON would point this job away from the proxy its --internal network was built around,
195
+ // while reading exactly like the control working -- an OUTAGE dressed as a policy. config.mjs refuses
196
+ // those names in PI_FORWARD_ENV outright while the policy is armed, for the same reason.
197
+ //
198
+ // BEFORE the mint below too, so the per-job scoped token still wins. A vault-supplied GITHUB_TOKEN overwriting
199
+ // the mint would hand every container a long-lived operator credential: CONST-TOKEN-SCOPED-PER-JOB
200
+ // defeated by a config line, which is the inversion forwardEnvList refuses at boot.
201
+ //
202
+ // Ordering is the BACKSTOP, not the gate. parseTriggers refuses every statically knowable one of these
203
+ // names at load, and the processor refuses the provider's own credential variables and the PI_FORWARD_ENV
204
+ // names pre-spend, where the resolved provider and the host env are in hand. This is the same division of
205
+ // labour the minted token already keeps ("and loadConfig refuses those names at load anyway").
206
+ //
207
+ // A LOOP rather than Object.assign, so a non-string or empty value becomes an ABSENT variable rather than
208
+ // `NAME=`: docker-run skips `undefined` but not `""`, and "never an empty string" is the rule PI_PACKAGES,
209
+ // PI_SESSION_FILE and PI_FLOW already keep. The resolver guarantees non-empty; this is the defense in
210
+ // depth at the DI seam that the empty-token guard keeps for the mint.
211
+ for (const [name, value] of Object.entries(secrets ?? {})) {
212
+ if (typeof value === "string" && value !== "") env[name] = value;
213
+ }
214
+
185
215
  // The shipped egress policy's variables (REQ-EGRESS-ALLOWLIST), AFTER the PI_FORWARD_ENV loop so a
186
216
  // forwarded name can never override them -- the same ordering, for the same reason, as the minted token
187
217
  // below (and loadConfig refuses those names outright while the policy is armed anyway).
package/src/forges.mjs CHANGED
@@ -135,6 +135,20 @@ export function forgeSpec(kind) {
135
135
  */
136
136
  export const MINTED_TOKEN_VARS = new Set(FORGE_KINDS.flatMap((kind) => FORGES[kind].tokenVars));
137
137
 
138
+ /**
139
+ * Every environment variable name any forge's SELF-HOSTED INSTANCE URL can land in. A sibling of
140
+ * `MINTED_TOKEN_VARS` and derived the same way, from the table rather than by hand, so a forge added
141
+ * later cannot be missed here either.
142
+ *
143
+ * Separate from `MINTED_TOKEN_VARS` because `hostVar` is a separate column: the token set is
144
+ * `tokenVars` only, so a name like `GITLAB_HOST` is in NEITHER set until this one exists. That gap is
145
+ * not theoretical -- `buildContainerEnv` writes this variable after the mint, so a trigger field able
146
+ * to name it would be silently overwritten, and the job would run without the value it asked for on a
147
+ * clean exit 0. github's row is `null` and contributes nothing, which is why the filter is here and
148
+ * not at the call site.
149
+ */
150
+ export const FORGE_HOST_VARS = new Set(FORGE_KINDS.map((kind) => FORGES[kind].hostVar).filter((name) => typeof name === "string"));
151
+
138
152
  /**
139
153
  * The separator between a repo label and a target number, for this forge and this target type: the
140
154
  * forge's own notation for a pull/merge request, and `#` for an issue everywhere.
package/src/get-token.mjs CHANGED
@@ -104,10 +104,15 @@ export async function makeGitHubAuth(cfg, deps = {}) {
104
104
  );
105
105
  }
106
106
  const repositoryNames = [repoNameOf(repo)]; // scope to the ONE repo; owner stripped
107
+ // An optional PERMISSIONS narrowing (issue #231): the receiver's closer-permission lookup asks
108
+ // for `{ metadata: "read" }`, so the token it holds for that one question cannot write anything
109
+ // even if leaked. Job mints never pass this and keep the installation's full grant -- narrowing
110
+ // is the caller's statement of intent, not a default this mint could guess.
111
+ const permissions = job?.permissions;
107
112
  let minted;
108
113
  try {
109
114
  const appAuth = createAppAuth(auth);
110
- minted = await appAuth({ type: "installation", repositoryNames });
115
+ minted = await appAuth({ type: "installation", repositoryNames, ...(permissions && { permissions }) });
111
116
  } catch (error) {
112
117
  throw classifyAppMintError(error);
113
118
  }
package/src/index.mjs CHANGED
@@ -101,6 +101,17 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
101
101
  tokenCap: settings.dailyTokenCap,
102
102
  ...deps,
103
103
  runContainer: (ctx) => deps.runContainer({ ...ctx, name, signal }),
104
+ // REQ-TRIGGER-SECRETS. The resolver runs INSIDE the 30-minute kill timer armed above, so it has
105
+ // to be abortable for the same reason runContainer does: a resolver blocking on an unreachable
106
+ // vault would otherwise hold its slot until its own timeout, and an abort landing mid-resolution
107
+ // would neither stop it nor keep the job from going on to mint, clone and reserve budget for a
108
+ // container that runContainer will refuse at entry anyway. Injected here, mirroring runContainer,
109
+ // because `signal` exists only in this scope. Omitted when unwired so a bare processor keeps
110
+ // runJob's own fail-closed default.
111
+ // `secretProfiles` is the OVERLAY half of the resolver table, read this job-start with the ten
112
+ // tunables above. It is bound here rather than at construction for the reason the overlay exists:
113
+ // an operator who declares a profile in the panel must not have to restart the worker.
114
+ ...(deps.resolveSecrets ? { resolveSecrets: (j) => deps.resolveSecrets(j, { signal, overlayProfiles: settings.secretProfiles ?? {} }) } : {}),
104
115
  // collectChain (INT-OUTBOX-CONTRACT) reads the completed parent's REAL BullMQ job: its `.id`
105
116
  // (the parent id children carry) and `.data` (kind/chainDepth). runJob's own `job` is the
106
117
  // effectiveJob -- a spread of job.data with no `.id`/`.data` -- so inject the real wrapper here,
@@ -114,6 +125,11 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
114
125
  // `queueJobId`, mirroring the collectChain injection above. Omitted when unwired so a bare
115
126
  // processor keeps runJob's plain (job, token) call.
116
127
  ...(deps.prepareWorkspace ? { prepareWorkspace: (j, t) => deps.prepareWorkspace(j, t, { queueJobId: job.id }) } : {}),
128
+ // The one-shot pre-spend check (issue #231) needs the REAL BullMQ job's `.id` to excuse this
129
+ // delivery's own earlier attempt -- runJob's effectiveJob has no `.id`, prepareWorkspace's
130
+ // own injection above states why, and this one mirrors it. Omitted when unwired so a bare
131
+ // processor keeps runJob's admit-everything default.
132
+ ...(deps.checkOnceSpent ? { checkOnceSpent: (j) => deps.checkOnceSpent(j, { queueJobId: job.id }) } : {}),
117
133
  });
118
134
  recordRun({ job, result, startedAt, endedAt: new Date().toISOString() });
119
135
  return result;
package/src/outbox.mjs CHANGED
@@ -176,6 +176,14 @@ export function makeCollectChain({ queue, enqueue = enqueueLocalJob, readFlowGat
176
176
  // same operator's flows and, without them, would look up a skill that is not there, write a
177
177
  // plausible report and exit 0. It is NOT part of chainedJobId, for the reason stated below.
178
178
  skillsDir: job.data?.skillsDir,
179
+ // `secrets`/`secretsProfile` are deliberately ABSENT, and the two lines above are exactly why
180
+ // this comment exists: their reasoning reads as though it should apply here too, and it must
181
+ // not. An image and a skills directory are toolchain; a resolved credential is a capability.
182
+ // A child's `task` is AGENT-AUTHORED (read off the request file below), so inheriting the
183
+ // binding would let a completed agent re-run itself against the operator's vault with a prompt
184
+ // it wrote for itself. The operator's grant was to the trigger they reviewed, not to whatever
185
+ // that job decides to queue next. A chained child that genuinely needs a secret gets it from a
186
+ // trigger of its own, which is an operator edit to a reviewed file.
179
187
  chainDepth: childDepth,
180
188
  parentJobId: job.id,
181
189
  // chainedJobId deliberately does NOT take the image: the child's identity is (parent, flow, task).
package/src/processor.mjs CHANGED
@@ -1,6 +1,7 @@
1
1
  import { lstatSync } from "node:fs";
2
2
  import { checkTokenCap, recordTokenSpend, releaseBudget, reserveBudget } from "./budget.mjs";
3
3
  import { configError } from "./config.mjs";
4
+ import { DEFAULT_SECRETS_PROFILE, secretsArmed } from "./secrets.mjs";
4
5
  import { EXIT_COMPLETED, EXIT_INFRA, EXIT_POLICY } from "./exit-code.mjs";
5
6
 
6
7
  /**
@@ -41,6 +42,11 @@ export async function runJob(job, deps) {
41
42
  // this job names is on this host (image-preflight.mjs). Default admits everything, so a wiring that
42
43
  // omits it behaves exactly as before -- the container's own failure stays the backstop.
43
44
  imagePreflight = async () => ({ ok: true }),
45
+ // (job) => { ok } | { refused, at, jobId }. The one-shot pre-spend check (issue #231,
46
+ // DES-ONE-SHOT-DISARM-IN-THE-FILE). Default admits everything -- an unwired processor behaves
47
+ // exactly as before, and the gate below only calls it for a job whose matched rule was a
48
+ // one-shot, so the default is never a probe running on every delivery.
49
+ checkOnceSpent = async () => ({ ok: true }),
44
50
  // REQ-EGRESS-ALLOWLIST. Default admits everything, so a wiring that omits it behaves exactly as a
45
51
  // deployment with no egress policy does -- which is also what the real factory returns when unarmed.
46
52
  egressPreflight = async () => ({ ok: true }),
@@ -70,6 +76,20 @@ export async function runJob(job, deps) {
70
76
  return false;
71
77
  }
72
78
  },
79
+ // (job) => { ok: true, secrets } | { profileUnknown } | { unresolved, ... }. The wiring binds the
80
+ // abort signal into it (index.mjs), the way it binds name+signal into runContainer.
81
+ // REQ-TRIGGER-SECRETS. Resolves this trigger's `run.secrets` references through the operator's own
82
+ // resolver, HOST-SIDE, before anything spends. Injected so the gate is testable without a real script.
83
+ //
84
+ // The default FAILS CLOSED, deliberately unlike imagePreflight's and egressPreflight's
85
+ // admit-everything defaults, and for the reason the sessionsDir default states above: an admitting
86
+ // default would let a job that armed run.secrets start with those variables UNSET under any wiring
87
+ // that omits this key. That is a false success which looks exactly like the feature working, and it
88
+ // is the inversion this whole gate exists to prevent. It can be UNCONDITIONALLY refusing because the
89
+ // gate below only calls it when the job is armed -- putting the arming test in the default instead
90
+ // would leave an INJECTED resolver running on every job, which is how a probe nobody wanted starts
91
+ // spawning a subprocess per delivery to learn nothing.
92
+ resolveSecrets = async (job) => ({ profileUnknown: job.secretsProfile ?? DEFAULT_SECRETS_PROFILE }),
73
93
  // (job) => scoped short-lived token. Takes the JOB, not the repo: which forge mints -- and therefore
74
94
  // which credential the container gets -- is a property of `job.kind`, and only the wiring knows the
75
95
  // map. Called for forge-backed jobs and for local jobs opted in via `github: true`; unflagged local
@@ -77,7 +97,8 @@ export async function runJob(job, deps) {
77
97
  mintToken,
78
98
  isDefaultBranchProtected, // (job, token) => boolean; same reason -- the forge is the job's, not the process's
79
99
  prepareWorkspace, // (job, token) => { workspaceDir, jobDir } (clone+materialise+prompt)
80
- // runContainer({ job, token, prepared, name, signal }) => { code, aborted, turns, tokens, session, usage }. It MUST honour
100
+ // runContainer({ job, token, prepared, secrets, name, signal }) => { code, aborted, turns, tokens, session, usage, context }.
101
+ // `secrets` is the resolved map from the gate above: values, already fetched, host-side. It MUST honour
81
102
  // `signal`: stop the container on abort, and reject/exit promptly if `signal.aborted` is already
82
103
  // true at entry (the timeout can fire during a slow prepare). The wiring injects name + signal.
83
104
  runContainer,
@@ -105,6 +126,27 @@ export async function runJob(job, deps) {
105
126
  let reserved = false;
106
127
 
107
128
  try {
129
+ // The one-shot pre-spend check (issue #231), FIRST on the ladder: one file read, cheaper than
130
+ // the docker inspect below, free, determinate, credential-less. Only a FOREIGN positive
131
+ // disarmed mark refuses -- the check excuses this queue job's own id, so a retry of the
132
+ // delivery that spent the trigger still runs (attempts:2 stays attempts:2) -- and anything
133
+ // unreadable or changed means "run": fail-open, because the disarm writer owns the loud
134
+ // refusals, and a broken read must never wedge every once job. In the compose topology the
135
+ // receiver reads a dead inode until restart, so this check is the once-enforcement layer
136
+ // there, not optional hardening.
137
+ if (job.trigger?.matched?.once === true) {
138
+ const spent = await checkOnceSpent(job);
139
+ if (spent.refused) {
140
+ // Commented like every sibling policy refusal: explainability is this refusal's whole
141
+ // purpose, and only a DISTINCT re-close reaches it past the GUID dedup, so the noise
142
+ // bound is the operator's own reopen-close rate. `at`/`jobId` are harness-written
143
+ // provenance, never payload text.
144
+ await comment(job, `Refused: this one-shot trigger was already spent${spent.at ? ` at ${spent.at}` : ""}${spent.jobId ? ` by job ${spent.jobId}` : ""}. The close that armed it has already produced a run; delete on.disarmed from the trigger entry to re-arm it. Not run.`);
145
+ log("refused_once_already_spent", { triggerIndex: job.trigger?.matched?.index ?? null });
146
+ return { outcome: "policy", reason: "once-already-spent", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
147
+ }
148
+ }
149
+
108
150
  // The job image must exist on THIS host before anything else happens. Free, determinate and
109
151
  // credential-less, so it precedes the mint, the clone and the reservation: a host that cannot run the
110
152
  // image refuses without minting a credential it will not use, cloning a repo it will not read, or
@@ -279,6 +321,90 @@ export async function runJob(job, deps) {
279
321
  return { outcome: "policy", reason: "skills-dir-missing", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
280
322
  }
281
323
 
324
+ // REQ-TRIGGER-SECRETS. A trigger that named secret references the worker cannot resolve would run its
325
+ // flow with those variables UNSET, get a 401 from whatever it was meant to reach, write a plausible
326
+ // report about why the integration is down, and exit 0.
327
+ //
328
+ // PLACEMENT. Strictly before the mint below, the clone in prepareWorkspace, the token-cap read and
329
+ // reserveBudget, so a refusal here costs nothing (CONST-BUDGET-BEFORE-TOKENS). But it is NOT one of the
330
+ // "free, determinate, credential-less and I/O-less" gates above, and this comment must not claim it is:
331
+ // resolving spawns a subprocess per reference against the operator's manager, which is none of those
332
+ // four things. The precedent it actually follows is one gate LATER -- prepareWorkspace already performs
333
+ // a full network clone before the budget is reserved. An I/O-bound pre-budget step is established here;
334
+ // a free one it is not. It sits last among the pre-mint gates because it is both the narrowest and the
335
+ // only one that can block, so every cheaper answer is already in hand when it runs.
336
+ //
337
+ // There is deliberately no cheap "would this resolve?" probe. The reference grammar belongs to the
338
+ // resolver (#206, #209: the seam is a command), so this project has nothing it could validate short of
339
+ // asking -- the same shape the branch-protection check takes, where the check IS the real call.
340
+ //
341
+ // Guarded by `secretsArmed` at the CALL SITE, not inside the resolver: an unflagged job must not reach
342
+ // it under ANY wiring, and a guard that lives in the default is a guard an injected resolver skips.
343
+ const resolved = secretsArmed(job) ? await resolveSecrets(job) : { ok: true, secrets: {} };
344
+ if (resolved.profileUnknown) {
345
+ await comment(job, "Refused: this trigger set `run.secrets`, and the resolver profile it names is not usable on this worker host. No profile of that name is declared, or its resolver is absent or not executable. The job would have started with those variables unset, and an agent that gets a 401 writes a plausible report and exits 0. Run `pi-dispatch doctor` on the worker to see which profiles it has. Not run.");
346
+ // The operator's own profile LABEL, and never a path, a reference, or a byte the resolver printed.
347
+ // `comment` posts publicly on the issue: a resolver path there publishes the operator's filesystem
348
+ // layout, and the DECLARED profile names would publish their vault topology -- which is why the
349
+ // message points at doctor for the list instead of enumerating it. The label itself is
350
+ // operator-authored trigger config, the same class as the image ref the job-image refusals name.
351
+ log("refused_secret_profile_unknown", { kind: job.kind ?? null, profile: resolved.profileUnknown });
352
+ return { outcome: "policy", reason: "secret-profile-unknown", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
353
+ }
354
+
355
+ if (resolved.ambiguous) {
356
+ // Two sources declared one profile name. Neither wins, deliberately: runtime-settings documents the
357
+ // overlay's precedence as overlay > env, so inverting it here would leave two rules disagreeing about
358
+ // what an overlay is, while honouring it would let a settings file in a world-writable default
359
+ // directory redirect a profile the operator wrote in .env. This project already refuses ambiguity
360
+ // rather than resolving it (PI_EGRESS: "a typo must never leave you believing you have a policy you
361
+ // do not"), and an operator who sees this fixes it in seconds.
362
+ await comment(job, "Refused: this trigger set `run.secrets`, and the resolver profile it names is declared twice on this worker host, once in the environment and once in the settings overlay. Neither wins, on purpose: the job would otherwise run against whichever one happened to be picked. Remove one of the two. Not run.");
363
+ log("refused_secret_profile_ambiguous", { kind: job.kind ?? null, profile: resolved.ambiguous });
364
+ return { outcome: "policy", reason: "secret-profile-ambiguous", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
365
+ }
366
+
367
+ if (resolved.reserved) {
368
+ // A key the worker itself writes, and one parseTriggers could not have caught: the provider
369
+ // credential's variable names depend on this job's resolved provider and on what this host has set,
370
+ // and PI_FORWARD_ENV is an operator env list. Both are deployment state, so this is the same
371
+ // load-time / pre-spend split run.resume makes against PI_SESSIONS_DIR.
372
+ //
373
+ // It matters most in the direction that is hardest to see: buildContainerEnv writes the provider
374
+ // credential BEFORE this feature's values, so a trigger binding ANTHROPIC_API_KEY would silently
375
+ // redirect which credential every job of that trigger spends.
376
+ await comment(job, `Refused: this trigger's \`run.secrets\` binds \`${resolved.reserved}\`, and the worker sets that variable itself for every job. The container would receive the worker's value rather than this trigger's, and the trigger would look like it worked. Rename it in the triggers file. Not run.`);
377
+ // The variable NAME only. It is the operator's own choice of name, not payload, and naming it is what
378
+ // makes the refusal actionable -- but the REFERENCE behind it never appears.
379
+ log("refused_secret_name_reserved", { kind: job.kind ?? null, name: resolved.reserved });
380
+ return { outcome: "policy", reason: "secret-name-reserved", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
381
+ }
382
+
383
+ if (resolved.unresolved) {
384
+ // DETERMINATE: the resolver said exit 2, printed nothing, overran the size cap, or returned a value
385
+ // with a NUL in it. Retrying cannot change any of those, so this RETURNS (CONST-RETRY-INFRA-ONLY).
386
+ const why = SECRET_FAILURES[resolved.failure] ?? "did not return a value";
387
+ await comment(job, `Refused: this trigger set \`run.secrets\`, and the resolver for \`${resolved.unresolved}\` ${why}. The job would have started with that variable unset, and an agent that gets a 401 writes a plausible report and exits 0. Run your profile's resolver by hand against that reference to see why: this comment carries neither the reference, nor the resolver's path, nor a byte of what it printed. Not run.`);
388
+ // The variable NAME, never a value -- the restraint refused_sessions_dir_unset keeps, and the name is
389
+ // the operator's own choice rather than payload. `failure` is OUR enum, `code` the script's small
390
+ // integer exit, `stderrBytes` a COUNT: never the resolver's words.
391
+ log("refused_secret_unresolved", { kind: job.kind ?? null, name: resolved.unresolved, failure: resolved.failure ?? null, code: resolved.code ?? null, stderrBytes: resolved.stderrBytes ?? 0 });
392
+ return { outcome: "policy", reason: "secret-unresolved", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
393
+ }
394
+
395
+ if (resolved.unreachable) {
396
+ // INDETERMINATE: exit 1 ("could not reach my manager"), an exit code we do not recognise, a spawn
397
+ // fault, or a timeout. THROWS, so BullMQ retries per `attempts` -- the determinate/indeterminate split
398
+ // the image and egress preflights already draw, decided here by the resolver's own exit code rather
399
+ // than by matching its stderr, which image-preflight.mjs forbids for good reason. Folding this into
400
+ // the refusal above would permanently burn a delivery over a twenty-second vault blip, and a webhook
401
+ // does not redeliver itself. Nothing has spent: `budgetReserved` computes false in the catch below
402
+ // because `reserved` is still false here.
403
+ log("secret_resolver_unreachable", { kind: job.kind ?? null, name: resolved.unreachable, failure: resolved.failure ?? null, code: resolved.code ?? null, stderrBytes: resolved.stderrBytes ?? 0 });
404
+ throw new InfraRetry(`secret resolver could not answer for ${resolved.unreachable}`, { reason: "secret-resolver-unreachable", provider: job.provider ?? null, model: job.model ?? null });
405
+ }
406
+ const secrets = resolved.secrets ?? {};
407
+
282
408
  if (wantsForgeToken) {
283
409
  token = await mintToken(job);
284
410
 
@@ -351,7 +477,7 @@ export async function runJob(job, deps) {
351
477
  return { outcome: "policy", reason: budget.reason, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: true }; // return => not retried
352
478
  }
353
479
 
354
- const { code, aborted, turns, tokens, session, usage, context } = await runContainer({ job, token, prepared });
480
+ const { code, aborted, turns, tokens, session, usage, context } = await runContainer({ job, token, prepared, secrets });
355
481
  log("container_exit", { exitCode: code, aborted });
356
482
 
357
483
  // Record token spend post-run (the check-AFTER half of the lagging token cap). The container ran,
@@ -492,6 +618,19 @@ function mergeSession(prepared, fromRunner, promoted = null) {
492
618
  };
493
619
  }
494
620
 
621
+ /**
622
+ * Our own words for why a resolver did not produce a value, turned into a phrase at the refusal site --
623
+ * the shape the egress refusal already uses for its `state` discriminator. The `??` fallback in the caller
624
+ * is not decoration: a failure code added in secrets.mjs must degrade to a generic sentence rather than
625
+ * print `undefined` on a public issue.
626
+ */
627
+ const SECRET_FAILURES = {
628
+ exit: "refused that reference",
629
+ empty: "printed nothing",
630
+ overflow: "printed more than the size cap allows",
631
+ nul: "printed a value containing a NUL byte, which cannot survive the container's argv",
632
+ };
633
+
495
634
  export class InfraRetry extends Error {
496
635
  constructor(message, { cause, reason, exitCode, turns, tokens, session, usage, provider, model, budgetReserved } = {}) {
497
636
  super(message, cause ? { cause } : undefined);