@edgehero/pi-dispatch 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +11 -0
- package/package.json +1 -1
- package/src/config.mjs +17 -0
- package/src/doctor.mjs +75 -3
- package/src/env-allowlist.mjs +31 -1
- package/src/forges.mjs +14 -0
- package/src/index.mjs +11 -0
- package/src/outbox.mjs +8 -0
- package/src/processor.mjs +115 -2
- package/src/queue.mjs +14 -2
- package/src/reserved-env.mjs +40 -0
- package/src/run-container.mjs +6 -2
- package/src/runtime-settings.mjs +40 -0
- package/src/schedules.mjs +1 -1
- package/src/secret-profiles.mjs +119 -0
- package/src/secrets.mjs +319 -0
- package/src/start.mjs +16 -1
- package/src/triggers.mjs +166 -6
package/.env.example
CHANGED
|
@@ -89,6 +89,17 @@ PI_JOB_IMAGE=pi-job:latest # the DEFAULT job image. Any trigger may nam
|
|
|
89
89
|
# PI_FORWARD_ENV= # comma-separated extra env var NAMES to forward into the container (e.g. a CUSTOM provider's key). Explicit allowlist, not a pass-through.
|
|
90
90
|
# GITHUB_TOKEN/GH_TOKEN are refused here -- the worker mints per-job tokens
|
|
91
91
|
|
|
92
|
+
# --- Per-trigger vault secrets (docs/secrets.md, issue #225) ---
|
|
93
|
+
# A trigger may name references ("secrets": { "STRIPE_KEY": "op://ci/stripe/api-key" }) and the profile that
|
|
94
|
+
# resolves them. The worker runs YOUR script once per reference, on the HOST, before the container starts.
|
|
95
|
+
# The job receives values; it never holds your manager's credential and cannot enumerate your vault.
|
|
96
|
+
# PI_SECRET_PROFILES= # name:/absolute/path pairs, comma separated (each entry splits on its FIRST colon, so a Windows C:\ path parses).
|
|
97
|
+
# A resolver is one line: `exec op read --no-newline "$1"`, `exec pass show "$1"`, `exec vault kv get -field=... "$1"`.
|
|
98
|
+
# Exit 2 if the reference is wrong, exit 1 if you could not reach your manager (that one retries). Unset = feature off.
|
|
99
|
+
# PI_SECRET_RESOLVER_ROOTS= # OS-path-delimited (; on Windows, : elsewhere) directories a PANEL-declared resolver may live in.
|
|
100
|
+
# Default empty = fail-closed: `/dispatch secrets add` can declare nothing, and only PI_SECRET_PROFILES above is honoured.
|
|
101
|
+
# PI_SECRET_RESOLVE_TIMEOUT_MS= # default 10000, per reference. Sits before a paid container and is multiplied by the reference count, so tighter than doctor's 30s.
|
|
102
|
+
|
|
92
103
|
PI_SCHEDULER_STALL_MAX=2 # tear down a scheduler after N consecutive stalls (money backstop)
|
|
93
104
|
|
|
94
105
|
# --- Egress policy: what a job container may reach on the network (docs/egress.md) ---
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@edgehero/pi-dispatch",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.3.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Self-hosted job harness for the pi coding agent: a BullMQ worker that drains the queue, mints scoped forge tokens, and runs one container per job — plus the pi-dispatch CLI (init, up, doctor, service).",
|
|
6
6
|
"keywords": [
|
package/src/config.mjs
CHANGED
|
@@ -9,6 +9,7 @@ import { existsSync } from "node:fs";
|
|
|
9
9
|
import { delimiter } from "node:path";
|
|
10
10
|
import { DEFAULT_EGRESS_PROXY, egressArmed } from "./egress.mjs";
|
|
11
11
|
import { MINTED_TOKEN_VARS } from "./forges.mjs";
|
|
12
|
+
import { parseSecretProfiles } from "./secret-profiles.mjs";
|
|
12
13
|
|
|
13
14
|
export function configError(message) {
|
|
14
15
|
const error = new Error(message);
|
|
@@ -278,6 +279,22 @@ export function loadConfig(env = process.env, { fileExists = existsSync } = {})
|
|
|
278
279
|
chainMaxPerJob: nonNegativeInt(env, "PI_CHAIN_MAX_PER_JOB", CHAIN_MAX_PER_JOB_DEFAULT), // INT-OUTBOX-CONTRACT: max request-<n>.json collected per parent
|
|
279
280
|
dispatchRunPerHour: nonNegativeInt(env, "PI_DISPATCH_RUN_PER_HOUR", 3), // DES-ADMIN-VIA-PI-EXTENSION; 0 = disable dispatch_run
|
|
280
281
|
dispatchRunRoots: delimitedList(env.PI_DISPATCH_RUN_ROOTS), // DES-AI-TRIGGER-FLOW-GATE: default [] fails closed — no folder passes, dispatch_run refuses everything
|
|
282
|
+
// REQ-TRIGGER-SECRETS. The operator's declared resolvers, `name:absolute-path` pairs, comma separated.
|
|
283
|
+
// Each entry splits on its FIRST colon so a Windows `C:\...` path survives -- the same drive-letter
|
|
284
|
+
// hazard `delimitedList` above exists for, arriving from the other side. Unset = the feature is off and
|
|
285
|
+
// any trigger naming secrets refuses pre-spend, which is why there is no default profile to fall into.
|
|
286
|
+
secretProfiles: parseSecretProfiles(env.PI_SECRET_PROFILES),
|
|
287
|
+
// The directories a resolver may live in. Default [] FAILS CLOSED exactly as dispatchRunRoots does, and
|
|
288
|
+
// for a sharper version of its reason: this bounds paths that can arrive from the settings overlay,
|
|
289
|
+
// which is not the reviewed artifact `triggers.json` is. Unset means the panel can declare no profile
|
|
290
|
+
// at all and only PI_SECRET_PROFILES above is honoured. Env-only, never the overlay and never the
|
|
291
|
+
// deployment pointer: `deployment-pointer.mjs` already refuses to carry PI_DISPATCH_RUN_ROOTS
|
|
292
|
+
// "because a pointer that could widen the AI-run folder allowlist would be a second, unreviewed door",
|
|
293
|
+
// and a bound that can be widened from the surface it bounds is not a bound.
|
|
294
|
+
secretResolverRoots: delimitedList(env.PI_SECRET_RESOLVER_ROOTS),
|
|
295
|
+
// Per-reference ceiling. Tighter than doctor's 30s on purpose: this runs before a paid container, is
|
|
296
|
+
// multiplied by the reference count, and holds a PI_CONCURRENCY slot while it waits.
|
|
297
|
+
secretResolveTimeoutMs: positiveInt(env, "PI_SECRET_RESOLVE_TIMEOUT_MS", 10000),
|
|
281
298
|
github: { ...loadGitHubAuth(env, fileExists), allowGhResume: env.PI_SESSIONS_ALLOW_GH_SOURCE === "1" },
|
|
282
299
|
gitlab: loadGitLabAuth(env),
|
|
283
300
|
forgejo: loadForgejoAuth(env),
|
package/src/doctor.mjs
CHANGED
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
*/
|
|
47
47
|
import { chmodSync, closeSync, existsSync, lstatSync, mkdirSync, mkdtempSync, openSync, readdirSync, readFileSync, readSync, rmSync, statSync } from "node:fs";
|
|
48
48
|
import { homedir, tmpdir } from "node:os";
|
|
49
|
-
import { dirname, join } from "node:path";
|
|
49
|
+
import { dirname, join, delimiter } from "node:path";
|
|
50
50
|
import { fileURLToPath } from "node:url";
|
|
51
51
|
import { spawn as nodeSpawn } from "node:child_process";
|
|
52
52
|
import { defaultSandboxDir, globalExtensionsEnabled } from "./config.mjs";
|
|
@@ -58,6 +58,7 @@ import { copySkillTree } from "./copy-tree.mjs";
|
|
|
58
58
|
import { SKILL_NAME_RE } from "./flow-gate.mjs";
|
|
59
59
|
import { DEFAULT_EGRESS_PROXY, egressArmed } from "./egress.mjs";
|
|
60
60
|
import { installedUnitPaths, readUnitSeam } from "./service.mjs";
|
|
61
|
+
import { parseSecretProfiles } from "./secret-profiles.mjs";
|
|
61
62
|
import { parseTriggers } from "./triggers.mjs";
|
|
62
63
|
|
|
63
64
|
const NODE_FLOOR = [22, 19]; // pi's engine floor (22.19.0)
|
|
@@ -263,7 +264,7 @@ export async function collectChecks(env, seams) {
|
|
|
263
264
|
// image checks just below, and `optingOut`/`requiring` colour the staged-packages lines further down.
|
|
264
265
|
// `optingOut` counts the only value that withholds the staged set; `requiring` counts an explicit
|
|
265
266
|
// run.packages: true, which arms nothing any more but is still an operator statement of intent.
|
|
266
|
-
const { requiring, optingOut, resuming, replicating, instructing, commands, images, skillsDirs, forges, repositories, flows, parseError, path: triggersFilePath } = readTriggerFacts(env, fileExists, cwd);
|
|
267
|
+
const { requiring, optingOut, resuming, replicating, instructing, commands, secreting, secretProfiles, localSecretFolders, images, skillsDirs, forges, repositories, flows, parseError, path: triggersFilePath } = readTriggerFacts(env, fileExists, cwd);
|
|
267
268
|
// FIRST, and fail rather than warn: every check below this line reads counts that a parse failure
|
|
268
269
|
// zeroed, so a green run here would be reporting on a file nobody could read. The receiver loads this
|
|
269
270
|
// file unconditionally and refuses to start without it, which is the consequence worth naming.
|
|
@@ -1096,6 +1097,52 @@ export async function collectChecks(env, seams) {
|
|
|
1096
1097
|
});
|
|
1097
1098
|
}
|
|
1098
1099
|
|
|
1100
|
+
// REQ-TRIGGER-SECRETS. Only reported when a trigger actually binds one, on the run.resume block's
|
|
1101
|
+
// reasoning below: a deployment that uses no secrets should not be told about a variable it has no
|
|
1102
|
+
// reason to set.
|
|
1103
|
+
if (secreting > 0) {
|
|
1104
|
+
const declared = parseSecretProfilesSafe(env.PI_SECRET_PROFILES);
|
|
1105
|
+
const names = Object.keys(declared).sort();
|
|
1106
|
+
if (declared.error) {
|
|
1107
|
+
checks.push({ ok: false, label: "PI_SECRET_PROFILES does not parse", fix: `${declared.error} -- the worker refuses to boot until this is fixed, rather than dropping the entry and leaving you a profile you believe is wired` });
|
|
1108
|
+
} else {
|
|
1109
|
+
// A HARD FAIL, not a warning, and worded like the run.resume/PI_SESSIONS_DIR check below for the
|
|
1110
|
+
// same reason: these jobs refuse pre-spend until it is set, deliberately, rather than running
|
|
1111
|
+
// without their secrets and looking like they worked.
|
|
1112
|
+
const missing = secretProfiles.filter((name) => !(name in declared));
|
|
1113
|
+
checks.push({
|
|
1114
|
+
ok: missing.length === 0,
|
|
1115
|
+
label: missing.length === 0 ? `${secreting} trigger(s) bind secrets, and every profile they name is declared` : `${secreting} trigger(s) bind secrets, but ${missing.length} named profile(s) are not declared: ${missing.join(", ")}`,
|
|
1116
|
+
fix: `declare them in PI_SECRET_PROFILES as name:/absolute/path pairs (a resolver is one line, e.g. \`exec op read --no-newline "$1"\`) -- these jobs refuse pre-spend until you do`,
|
|
1117
|
+
});
|
|
1118
|
+
// The declared table, so an operator sees what is wired without reading .env. NAMES and paths only:
|
|
1119
|
+
// this is doctor's own stdout on the operator's host, not a public issue comment.
|
|
1120
|
+
if (names.length > 0) {
|
|
1121
|
+
checks.push({ ok: true, label: `Secret resolver profiles declared: ${names.map((n) => `${n} -> ${declared[n]}`).join(", ")}` });
|
|
1122
|
+
}
|
|
1123
|
+
// The panel-authoring bound. Unset is the SAFE default rather than a defect, so this is a fact line
|
|
1124
|
+
// when closed and a disclosure when open.
|
|
1125
|
+
const roots = (env.PI_SECRET_RESOLVER_ROOTS ?? "").split(delimiter).map((r) => r.trim()).filter(Boolean);
|
|
1126
|
+
checks.push({
|
|
1127
|
+
ok: true,
|
|
1128
|
+
...(roots.length === 0
|
|
1129
|
+
? { label: "PI_SECRET_RESOLVER_ROOTS is unset, so only PI_SECRET_PROFILES declares resolvers (the panel can declare none)" }
|
|
1130
|
+
: { warn: true, label: `PI_SECRET_RESOLVER_ROOTS admits panel-declared resolvers under: ${roots.join(", ")}`, fix: "keep those directories writable by nobody but the account the worker runs as: whoever can write a resolver there can run code as the worker" }),
|
|
1131
|
+
});
|
|
1132
|
+
}
|
|
1133
|
+
// The local-workspace disclosure. Not a failure: a nightly deploy binding a secret is exactly what
|
|
1134
|
+
// this feature is for. But a local job's /workspace IS the folder, read-write and un-cloned, so an
|
|
1135
|
+
// agent that persists a credential to make its next command simpler writes it into a real repository.
|
|
1136
|
+
if (localSecretFolders.length > 0) {
|
|
1137
|
+
checks.push({
|
|
1138
|
+
ok: true,
|
|
1139
|
+
warn: true,
|
|
1140
|
+
label: `${localSecretFolders.length} local trigger(s) bind secrets and run IN the operator's own folder: ${localSecretFolders.join(", ")}`,
|
|
1141
|
+
fix: "a local job edits that folder in place, so a credential the agent writes to .env, .netrc or .git-credentials lands in your real repository (and survives in a retained sandbox for PI_SANDBOX_RETENTION_HOURS). Nothing scans for that: keep those folders out of anything you push",
|
|
1142
|
+
});
|
|
1143
|
+
}
|
|
1144
|
+
}
|
|
1145
|
+
|
|
1099
1146
|
// REQ-RESUMABLE-SESSION. Only reported when a trigger actually asked for it: a deployment that does
|
|
1100
1147
|
// not use resume should not be told about a directory it has no reason to create.
|
|
1101
1148
|
if (resuming > 0) {
|
|
@@ -1410,8 +1457,22 @@ async function repoFlowAtHead(spawn, folder, flow) {
|
|
|
1410
1457
|
return mode === "100644" && type === "blob" ? "present" : "absent";
|
|
1411
1458
|
}
|
|
1412
1459
|
|
|
1460
|
+
/**
|
|
1461
|
+
* `parseSecretProfiles`, but doctor never throws. A malformed PI_SECRET_PROFILES is a finding to REPORT,
|
|
1462
|
+
* not a reason for the diagnostic tool to die: the operator running doctor is very likely running it
|
|
1463
|
+
* BECAUSE the worker refused to boot on that exact line, and a stack trace instead of a check is the least
|
|
1464
|
+
* useful possible answer. Returns the table, or `{ error }` carrying the parser's own message.
|
|
1465
|
+
*/
|
|
1466
|
+
function parseSecretProfilesSafe(raw) {
|
|
1467
|
+
try {
|
|
1468
|
+
return parseSecretProfiles(raw);
|
|
1469
|
+
} catch (err) {
|
|
1470
|
+
return { error: err?.message ?? "unparseable" };
|
|
1471
|
+
}
|
|
1472
|
+
}
|
|
1473
|
+
|
|
1413
1474
|
function readTriggerFacts(env, fileExists, cwd) {
|
|
1414
|
-
const none = { requiring: 0, optingOut: 0, resuming: 0, replicating: 0, instructing: 0, commands: 0, images: [], skillsDirs: [], forges: [], repositories: [], flows: [], parseError: null, path: null };
|
|
1475
|
+
const none = { requiring: 0, optingOut: 0, resuming: 0, replicating: 0, instructing: 0, commands: 0, secreting: 0, secretProfiles: [], localSecretFolders: [], images: [], skillsDirs: [], forges: [], repositories: [], flows: [], parseError: null, path: null };
|
|
1415
1476
|
try {
|
|
1416
1477
|
// Unset falls back to ./triggers.json in cwd, MIRRORING the receiver's own default
|
|
1417
1478
|
// (receiver/src/config.mjs) -- the two must read the same file, or doctor preflights a deployment
|
|
@@ -1432,6 +1493,17 @@ function readTriggerFacts(env, fileExists, cwd) {
|
|
|
1432
1493
|
// tuple list already filters to `typeof f.flow === "string"`, so a command trigger drops out
|
|
1433
1494
|
// of the flow-tier probes naturally -- no exclusion needed there.
|
|
1434
1495
|
commands: triggers.filter((t) => typeof t.run.command === "string").length,
|
|
1496
|
+
// REQ-TRIGGER-SECRETS. Counted beside `instructing` for its reason: a per-trigger choice that
|
|
1497
|
+
// changes what every job of it can reach, and one that lives only in triggers.json.
|
|
1498
|
+
secreting: triggers.filter((t) => t.run.secrets !== undefined).length,
|
|
1499
|
+
// The distinct profile NAMES the file selects, deduped like `images`/`skillsDirs`: the checks below
|
|
1500
|
+
// cost a stat each, and two triggers naming one profile are one question. `default` is substituted
|
|
1501
|
+
// for an absent field so the table answers what the worker will actually look up.
|
|
1502
|
+
secretProfiles: [...new Set(triggers.filter((t) => t.run.secrets !== undefined).map((t) => t.run.secretsProfile ?? "default"))].sort(),
|
|
1503
|
+
// LOCAL triggers that bind secrets, by folder. A local job's /workspace IS this folder, bind-mounted
|
|
1504
|
+
// read-write with no clone, so a credential an agent writes into .env lands in the operator's real
|
|
1505
|
+
// repository rather than a temp dir that gets swept. Deduped for skillsDirs' reason.
|
|
1506
|
+
localSecretFolders: [...new Set(triggers.filter((t) => t.run.secrets !== undefined && t.run.kind === "local" && typeof t.run.folder === "string").map((t) => t.run.folder))].sort(),
|
|
1435
1507
|
optingOut: triggers.filter((t) => t.run.packages === false).length,
|
|
1436
1508
|
images: [...new Set(triggers.map((t) => t.run.image).filter((i) => typeof i === "string"))].sort(),
|
|
1437
1509
|
// REQ-PER-TRIGGER-SKILLS. The distinct host directories the file names, deduped like `images`,
|
package/src/env-allowlist.mjs
CHANGED
|
@@ -111,7 +111,7 @@ function resolveEnvName(provider, cred) {
|
|
|
111
111
|
* `allowGlobalExtensions` defaults to TRUE here, matching loadConfig's default (REQ-GLOBAL-PI-OVERLAY): a
|
|
112
112
|
* caller that says nothing gets the operator's staged setup, and only an explicit `false` withholds it.
|
|
113
113
|
*/
|
|
114
|
-
export function buildContainerEnv({ provider, model, maxTurns, maxTokens, jobId, githubToken, forgeKind, forgeHosts = {}, hostEnv, allowGlobalExtensions = true, packagePaths = [], forwardEnv = [], sessionFile = null, flow = null, command = null, authFromPi = false, egress = false, egressProxy, agentDir, readFile = readFileSync }) {
|
|
114
|
+
export function buildContainerEnv({ provider, model, maxTurns, maxTokens, jobId, githubToken, forgeKind, forgeHosts = {}, hostEnv, allowGlobalExtensions = true, packagePaths = [], forwardEnv = [], secrets = {}, sessionFile = null, flow = null, command = null, authFromPi = false, egress = false, egressProxy, agentDir, readFile = readFileSync }) {
|
|
115
115
|
// The provider credential(s), by pi's expected variable name(s) -- from the worker env, or (when
|
|
116
116
|
// PI_AUTH_FROM_PI is set and the env has none) host-side from pi's auth.json. Throws (config) if
|
|
117
117
|
// neither source yields one, which the processor turns into a pre-spend refusal.
|
|
@@ -182,6 +182,36 @@ export function buildContainerEnv({ provider, model, maxTurns, maxTokens, jobId,
|
|
|
182
182
|
if (hostEnv[name] !== undefined) env[name] = hostEnv[name];
|
|
183
183
|
}
|
|
184
184
|
|
|
185
|
+
// The trigger's own secrets (REQ-TRIGGER-SECRETS), resolved HOST-SIDE by the processor before anything
|
|
186
|
+
// spent and injected exactly the way the provider credential is -- never a vault credential handed to the
|
|
187
|
+
// container to fetch them itself. `docs/secrets.md`'s rule survives intact: what crosses the boundary is a
|
|
188
|
+
// value, and the thing that can FETCH values stays on the host.
|
|
189
|
+
//
|
|
190
|
+
// AFTER the PI_FORWARD_ENV loop, for that loop's own stated reason: a name on the operator's blanket host
|
|
191
|
+
// list must not silently outrank the specific reference this trigger declared.
|
|
192
|
+
//
|
|
193
|
+
// BEFORE the egress assign, and that direction is deliberate rather than incidental. A secret named
|
|
194
|
+
// HTTPS_PROXY that WON would point this job away from the proxy its --internal network was built around,
|
|
195
|
+
// while reading exactly like the control working -- an OUTAGE dressed as a policy. config.mjs refuses
|
|
196
|
+
// those names in PI_FORWARD_ENV outright while the policy is armed, for the same reason.
|
|
197
|
+
//
|
|
198
|
+
// BEFORE the mint below too, so the per-job scoped token still wins. A vault-supplied GITHUB_TOKEN overwriting
|
|
199
|
+
// the mint would hand every container a long-lived operator credential: CONST-TOKEN-SCOPED-PER-JOB
|
|
200
|
+
// defeated by a config line, which is the inversion forwardEnvList refuses at boot.
|
|
201
|
+
//
|
|
202
|
+
// Ordering is the BACKSTOP, not the gate. parseTriggers refuses every statically knowable one of these
|
|
203
|
+
// names at load, and the processor refuses the provider's own credential variables and the PI_FORWARD_ENV
|
|
204
|
+
// names pre-spend, where the resolved provider and the host env are in hand. This is the same division of
|
|
205
|
+
// labour the minted token already keeps ("and loadConfig refuses those names at load anyway").
|
|
206
|
+
//
|
|
207
|
+
// A LOOP rather than Object.assign, so a non-string or empty value becomes an ABSENT variable rather than
|
|
208
|
+
// `NAME=`: docker-run skips `undefined` but not `""`, and "never an empty string" is the rule PI_PACKAGES,
|
|
209
|
+
// PI_SESSION_FILE and PI_FLOW already keep. The resolver guarantees non-empty; this is the defense in
|
|
210
|
+
// depth at the DI seam that the empty-token guard keeps for the mint.
|
|
211
|
+
for (const [name, value] of Object.entries(secrets ?? {})) {
|
|
212
|
+
if (typeof value === "string" && value !== "") env[name] = value;
|
|
213
|
+
}
|
|
214
|
+
|
|
185
215
|
// The shipped egress policy's variables (REQ-EGRESS-ALLOWLIST), AFTER the PI_FORWARD_ENV loop so a
|
|
186
216
|
// forwarded name can never override them -- the same ordering, for the same reason, as the minted token
|
|
187
217
|
// below (and loadConfig refuses those names outright while the policy is armed anyway).
|
package/src/forges.mjs
CHANGED
|
@@ -135,6 +135,20 @@ export function forgeSpec(kind) {
|
|
|
135
135
|
*/
|
|
136
136
|
export const MINTED_TOKEN_VARS = new Set(FORGE_KINDS.flatMap((kind) => FORGES[kind].tokenVars));
|
|
137
137
|
|
|
138
|
+
/**
|
|
139
|
+
* Every environment variable name any forge's SELF-HOSTED INSTANCE URL can land in. A sibling of
|
|
140
|
+
* `MINTED_TOKEN_VARS` and derived the same way, from the table rather than by hand, so a forge added
|
|
141
|
+
* later cannot be missed here either.
|
|
142
|
+
*
|
|
143
|
+
* Separate from `MINTED_TOKEN_VARS` because `hostVar` is a separate column: the token set is
|
|
144
|
+
* `tokenVars` only, so a name like `GITLAB_HOST` is in NEITHER set until this one exists. That gap is
|
|
145
|
+
* not theoretical -- `buildContainerEnv` writes this variable after the mint, so a trigger field able
|
|
146
|
+
* to name it would be silently overwritten, and the job would run without the value it asked for on a
|
|
147
|
+
* clean exit 0. github's row is `null` and contributes nothing, which is why the filter is here and
|
|
148
|
+
* not at the call site.
|
|
149
|
+
*/
|
|
150
|
+
export const FORGE_HOST_VARS = new Set(FORGE_KINDS.map((kind) => FORGES[kind].hostVar).filter((name) => typeof name === "string"));
|
|
151
|
+
|
|
138
152
|
/**
|
|
139
153
|
* The separator between a repo label and a target number, for this forge and this target type: the
|
|
140
154
|
* forge's own notation for a pull/merge request, and `#` for an issue everywhere.
|
package/src/index.mjs
CHANGED
|
@@ -101,6 +101,17 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
|
|
|
101
101
|
tokenCap: settings.dailyTokenCap,
|
|
102
102
|
...deps,
|
|
103
103
|
runContainer: (ctx) => deps.runContainer({ ...ctx, name, signal }),
|
|
104
|
+
// REQ-TRIGGER-SECRETS. The resolver runs INSIDE the 30-minute kill timer armed above, so it has
|
|
105
|
+
// to be abortable for the same reason runContainer does: a resolver blocking on an unreachable
|
|
106
|
+
// vault would otherwise hold its slot until its own timeout, and an abort landing mid-resolution
|
|
107
|
+
// would neither stop it nor keep the job from going on to mint, clone and reserve budget for a
|
|
108
|
+
// container that runContainer will refuse at entry anyway. Injected here, mirroring runContainer,
|
|
109
|
+
// because `signal` exists only in this scope. Omitted when unwired so a bare processor keeps
|
|
110
|
+
// runJob's own fail-closed default.
|
|
111
|
+
// `secretProfiles` is the OVERLAY half of the resolver table, read this job-start with the ten
|
|
112
|
+
// tunables above. It is bound here rather than at construction for the reason the overlay exists:
|
|
113
|
+
// an operator who declares a profile in the panel must not have to restart the worker.
|
|
114
|
+
...(deps.resolveSecrets ? { resolveSecrets: (j) => deps.resolveSecrets(j, { signal, overlayProfiles: settings.secretProfiles ?? {} }) } : {}),
|
|
104
115
|
// collectChain (INT-OUTBOX-CONTRACT) reads the completed parent's REAL BullMQ job: its `.id`
|
|
105
116
|
// (the parent id children carry) and `.data` (kind/chainDepth). runJob's own `job` is the
|
|
106
117
|
// effectiveJob -- a spread of job.data with no `.id`/`.data` -- so inject the real wrapper here,
|
package/src/outbox.mjs
CHANGED
|
@@ -176,6 +176,14 @@ export function makeCollectChain({ queue, enqueue = enqueueLocalJob, readFlowGat
|
|
|
176
176
|
// same operator's flows and, without them, would look up a skill that is not there, write a
|
|
177
177
|
// plausible report and exit 0. It is NOT part of chainedJobId, for the reason stated below.
|
|
178
178
|
skillsDir: job.data?.skillsDir,
|
|
179
|
+
// `secrets`/`secretsProfile` are deliberately ABSENT, and the two lines above are exactly why
|
|
180
|
+
// this comment exists: their reasoning reads as though it should apply here too, and it must
|
|
181
|
+
// not. An image and a skills directory are toolchain; a resolved credential is a capability.
|
|
182
|
+
// A child's `task` is AGENT-AUTHORED (read off the request file below), so inheriting the
|
|
183
|
+
// binding would let a completed agent re-run itself against the operator's vault with a prompt
|
|
184
|
+
// it wrote for itself. The operator's grant was to the trigger they reviewed, not to whatever
|
|
185
|
+
// that job decides to queue next. A chained child that genuinely needs a secret gets it from a
|
|
186
|
+
// trigger of its own, which is an operator edit to a reviewed file.
|
|
179
187
|
chainDepth: childDepth,
|
|
180
188
|
parentJobId: job.id,
|
|
181
189
|
// chainedJobId deliberately does NOT take the image: the child's identity is (parent, flow, task).
|
package/src/processor.mjs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { lstatSync } from "node:fs";
|
|
2
2
|
import { checkTokenCap, recordTokenSpend, releaseBudget, reserveBudget } from "./budget.mjs";
|
|
3
3
|
import { configError } from "./config.mjs";
|
|
4
|
+
import { DEFAULT_SECRETS_PROFILE, secretsArmed } from "./secrets.mjs";
|
|
4
5
|
import { EXIT_COMPLETED, EXIT_INFRA, EXIT_POLICY } from "./exit-code.mjs";
|
|
5
6
|
|
|
6
7
|
/**
|
|
@@ -70,6 +71,20 @@ export async function runJob(job, deps) {
|
|
|
70
71
|
return false;
|
|
71
72
|
}
|
|
72
73
|
},
|
|
74
|
+
// (job) => { ok: true, secrets } | { profileUnknown } | { unresolved, ... }. The wiring binds the
|
|
75
|
+
// abort signal into it (index.mjs), the way it binds name+signal into runContainer.
|
|
76
|
+
// REQ-TRIGGER-SECRETS. Resolves this trigger's `run.secrets` references through the operator's own
|
|
77
|
+
// resolver, HOST-SIDE, before anything spends. Injected so the gate is testable without a real script.
|
|
78
|
+
//
|
|
79
|
+
// The default FAILS CLOSED, deliberately unlike imagePreflight's and egressPreflight's
|
|
80
|
+
// admit-everything defaults, and for the reason the sessionsDir default states above: an admitting
|
|
81
|
+
// default would let a job that armed run.secrets start with those variables UNSET under any wiring
|
|
82
|
+
// that omits this key. That is a false success which looks exactly like the feature working, and it
|
|
83
|
+
// is the inversion this whole gate exists to prevent. It can be UNCONDITIONALLY refusing because the
|
|
84
|
+
// gate below only calls it when the job is armed -- putting the arming test in the default instead
|
|
85
|
+
// would leave an INJECTED resolver running on every job, which is how a probe nobody wanted starts
|
|
86
|
+
// spawning a subprocess per delivery to learn nothing.
|
|
87
|
+
resolveSecrets = async (job) => ({ profileUnknown: job.secretsProfile ?? DEFAULT_SECRETS_PROFILE }),
|
|
73
88
|
// (job) => scoped short-lived token. Takes the JOB, not the repo: which forge mints -- and therefore
|
|
74
89
|
// which credential the container gets -- is a property of `job.kind`, and only the wiring knows the
|
|
75
90
|
// map. Called for forge-backed jobs and for local jobs opted in via `github: true`; unflagged local
|
|
@@ -77,7 +92,8 @@ export async function runJob(job, deps) {
|
|
|
77
92
|
mintToken,
|
|
78
93
|
isDefaultBranchProtected, // (job, token) => boolean; same reason -- the forge is the job's, not the process's
|
|
79
94
|
prepareWorkspace, // (job, token) => { workspaceDir, jobDir } (clone+materialise+prompt)
|
|
80
|
-
// runContainer({ job, token, prepared, name, signal }) => { code, aborted, turns, tokens, session, usage }.
|
|
95
|
+
// runContainer({ job, token, prepared, secrets, name, signal }) => { code, aborted, turns, tokens, session, usage, context }.
|
|
96
|
+
// `secrets` is the resolved map from the gate above: values, already fetched, host-side. It MUST honour
|
|
81
97
|
// `signal`: stop the container on abort, and reject/exit promptly if `signal.aborted` is already
|
|
82
98
|
// true at entry (the timeout can fire during a slow prepare). The wiring injects name + signal.
|
|
83
99
|
runContainer,
|
|
@@ -279,6 +295,90 @@ export async function runJob(job, deps) {
|
|
|
279
295
|
return { outcome: "policy", reason: "skills-dir-missing", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
280
296
|
}
|
|
281
297
|
|
|
298
|
+
// REQ-TRIGGER-SECRETS. A trigger that named secret references the worker cannot resolve would run its
|
|
299
|
+
// flow with those variables UNSET, get a 401 from whatever it was meant to reach, write a plausible
|
|
300
|
+
// report about why the integration is down, and exit 0.
|
|
301
|
+
//
|
|
302
|
+
// PLACEMENT. Strictly before the mint below, the clone in prepareWorkspace, the token-cap read and
|
|
303
|
+
// reserveBudget, so a refusal here costs nothing (CONST-BUDGET-BEFORE-TOKENS). But it is NOT one of the
|
|
304
|
+
// "free, determinate, credential-less and I/O-less" gates above, and this comment must not claim it is:
|
|
305
|
+
// resolving spawns a subprocess per reference against the operator's manager, which is none of those
|
|
306
|
+
// four things. The precedent it actually follows is one gate LATER -- prepareWorkspace already performs
|
|
307
|
+
// a full network clone before the budget is reserved. An I/O-bound pre-budget step is established here;
|
|
308
|
+
// a free one it is not. It sits last among the pre-mint gates because it is both the narrowest and the
|
|
309
|
+
// only one that can block, so every cheaper answer is already in hand when it runs.
|
|
310
|
+
//
|
|
311
|
+
// There is deliberately no cheap "would this resolve?" probe. The reference grammar belongs to the
|
|
312
|
+
// resolver (#206, #209: the seam is a command), so this project has nothing it could validate short of
|
|
313
|
+
// asking -- the same shape the branch-protection check takes, where the check IS the real call.
|
|
314
|
+
//
|
|
315
|
+
// Guarded by `secretsArmed` at the CALL SITE, not inside the resolver: an unflagged job must not reach
|
|
316
|
+
// it under ANY wiring, and a guard that lives in the default is a guard an injected resolver skips.
|
|
317
|
+
const resolved = secretsArmed(job) ? await resolveSecrets(job) : { ok: true, secrets: {} };
|
|
318
|
+
if (resolved.profileUnknown) {
|
|
319
|
+
await comment(job, "Refused: this trigger set `run.secrets`, and the resolver profile it names is not usable on this worker host. No profile of that name is declared, or its resolver is absent or not executable. The job would have started with those variables unset, and an agent that gets a 401 writes a plausible report and exits 0. Run `pi-dispatch doctor` on the worker to see which profiles it has. Not run.");
|
|
320
|
+
// The operator's own profile LABEL, and never a path, a reference, or a byte the resolver printed.
|
|
321
|
+
// `comment` posts publicly on the issue: a resolver path there publishes the operator's filesystem
|
|
322
|
+
// layout, and the DECLARED profile names would publish their vault topology -- which is why the
|
|
323
|
+
// message points at doctor for the list instead of enumerating it. The label itself is
|
|
324
|
+
// operator-authored trigger config, the same class as the image ref the job-image refusals name.
|
|
325
|
+
log("refused_secret_profile_unknown", { kind: job.kind ?? null, profile: resolved.profileUnknown });
|
|
326
|
+
return { outcome: "policy", reason: "secret-profile-unknown", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
if (resolved.ambiguous) {
|
|
330
|
+
// Two sources declared one profile name. Neither wins, deliberately: runtime-settings documents the
|
|
331
|
+
// overlay's precedence as overlay > env, so inverting it here would leave two rules disagreeing about
|
|
332
|
+
// what an overlay is, while honouring it would let a settings file in a world-writable default
|
|
333
|
+
// directory redirect a profile the operator wrote in .env. This project already refuses ambiguity
|
|
334
|
+
// rather than resolving it (PI_EGRESS: "a typo must never leave you believing you have a policy you
|
|
335
|
+
// do not"), and an operator who sees this fixes it in seconds.
|
|
336
|
+
await comment(job, "Refused: this trigger set `run.secrets`, and the resolver profile it names is declared twice on this worker host, once in the environment and once in the settings overlay. Neither wins, on purpose: the job would otherwise run against whichever one happened to be picked. Remove one of the two. Not run.");
|
|
337
|
+
log("refused_secret_profile_ambiguous", { kind: job.kind ?? null, profile: resolved.ambiguous });
|
|
338
|
+
return { outcome: "policy", reason: "secret-profile-ambiguous", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
if (resolved.reserved) {
|
|
342
|
+
// A key the worker itself writes, and one parseTriggers could not have caught: the provider
|
|
343
|
+
// credential's variable names depend on this job's resolved provider and on what this host has set,
|
|
344
|
+
// and PI_FORWARD_ENV is an operator env list. Both are deployment state, so this is the same
|
|
345
|
+
// load-time / pre-spend split run.resume makes against PI_SESSIONS_DIR.
|
|
346
|
+
//
|
|
347
|
+
// It matters most in the direction that is hardest to see: buildContainerEnv writes the provider
|
|
348
|
+
// credential BEFORE this feature's values, so a trigger binding ANTHROPIC_API_KEY would silently
|
|
349
|
+
// redirect which credential every job of that trigger spends.
|
|
350
|
+
await comment(job, `Refused: this trigger's \`run.secrets\` binds \`${resolved.reserved}\`, and the worker sets that variable itself for every job. The container would receive the worker's value rather than this trigger's, and the trigger would look like it worked. Rename it in the triggers file. Not run.`);
|
|
351
|
+
// The variable NAME only. It is the operator's own choice of name, not payload, and naming it is what
|
|
352
|
+
// makes the refusal actionable -- but the REFERENCE behind it never appears.
|
|
353
|
+
log("refused_secret_name_reserved", { kind: job.kind ?? null, name: resolved.reserved });
|
|
354
|
+
return { outcome: "policy", reason: "secret-name-reserved", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
if (resolved.unresolved) {
|
|
358
|
+
// DETERMINATE: the resolver said exit 2, printed nothing, overran the size cap, or returned a value
|
|
359
|
+
// with a NUL in it. Retrying cannot change any of those, so this RETURNS (CONST-RETRY-INFRA-ONLY).
|
|
360
|
+
const why = SECRET_FAILURES[resolved.failure] ?? "did not return a value";
|
|
361
|
+
await comment(job, `Refused: this trigger set \`run.secrets\`, and the resolver for \`${resolved.unresolved}\` ${why}. The job would have started with that variable unset, and an agent that gets a 401 writes a plausible report and exits 0. Run your profile's resolver by hand against that reference to see why: this comment carries neither the reference, nor the resolver's path, nor a byte of what it printed. Not run.`);
|
|
362
|
+
// The variable NAME, never a value -- the restraint refused_sessions_dir_unset keeps, and the name is
|
|
363
|
+
// the operator's own choice rather than payload. `failure` is OUR enum, `code` the script's small
|
|
364
|
+
// integer exit, `stderrBytes` a COUNT: never the resolver's words.
|
|
365
|
+
log("refused_secret_unresolved", { kind: job.kind ?? null, name: resolved.unresolved, failure: resolved.failure ?? null, code: resolved.code ?? null, stderrBytes: resolved.stderrBytes ?? 0 });
|
|
366
|
+
return { outcome: "policy", reason: "secret-unresolved", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
if (resolved.unreachable) {
|
|
370
|
+
// INDETERMINATE: exit 1 ("could not reach my manager"), an exit code we do not recognise, a spawn
|
|
371
|
+
// fault, or a timeout. THROWS, so BullMQ retries per `attempts` -- the determinate/indeterminate split
|
|
372
|
+
// the image and egress preflights already draw, decided here by the resolver's own exit code rather
|
|
373
|
+
// than by matching its stderr, which image-preflight.mjs forbids for good reason. Folding this into
|
|
374
|
+
// the refusal above would permanently burn a delivery over a twenty-second vault blip, and a webhook
|
|
375
|
+
// does not redeliver itself. Nothing has spent: `budgetReserved` computes false in the catch below
|
|
376
|
+
// because `reserved` is still false here.
|
|
377
|
+
log("secret_resolver_unreachable", { kind: job.kind ?? null, name: resolved.unreachable, failure: resolved.failure ?? null, code: resolved.code ?? null, stderrBytes: resolved.stderrBytes ?? 0 });
|
|
378
|
+
throw new InfraRetry(`secret resolver could not answer for ${resolved.unreachable}`, { reason: "secret-resolver-unreachable", provider: job.provider ?? null, model: job.model ?? null });
|
|
379
|
+
}
|
|
380
|
+
const secrets = resolved.secrets ?? {};
|
|
381
|
+
|
|
282
382
|
if (wantsForgeToken) {
|
|
283
383
|
token = await mintToken(job);
|
|
284
384
|
|
|
@@ -351,7 +451,7 @@ export async function runJob(job, deps) {
|
|
|
351
451
|
return { outcome: "policy", reason: budget.reason, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: true }; // return => not retried
|
|
352
452
|
}
|
|
353
453
|
|
|
354
|
-
const { code, aborted, turns, tokens, session, usage, context } = await runContainer({ job, token, prepared });
|
|
454
|
+
const { code, aborted, turns, tokens, session, usage, context } = await runContainer({ job, token, prepared, secrets });
|
|
355
455
|
log("container_exit", { exitCode: code, aborted });
|
|
356
456
|
|
|
357
457
|
// Record token spend post-run (the check-AFTER half of the lagging token cap). The container ran,
|
|
@@ -492,6 +592,19 @@ function mergeSession(prepared, fromRunner, promoted = null) {
|
|
|
492
592
|
};
|
|
493
593
|
}
|
|
494
594
|
|
|
595
|
+
/**
|
|
596
|
+
* Our own words for why a resolver did not produce a value, turned into a phrase at the refusal site --
|
|
597
|
+
* the shape the egress refusal already uses for its `state` discriminator. The `??` fallback in the caller
|
|
598
|
+
* is not decoration: a failure code added in secrets.mjs must degrade to a generic sentence rather than
|
|
599
|
+
* print `undefined` on a public issue.
|
|
600
|
+
*/
|
|
601
|
+
const SECRET_FAILURES = {
|
|
602
|
+
exit: "refused that reference",
|
|
603
|
+
empty: "printed nothing",
|
|
604
|
+
overflow: "printed more than the size cap allows",
|
|
605
|
+
nul: "printed a value containing a NUL byte, which cannot survive the container's argv",
|
|
606
|
+
};
|
|
607
|
+
|
|
495
608
|
export class InfraRetry extends Error {
|
|
496
609
|
constructor(message, { cause, reason, exitCode, turns, tokens, session, usage, provider, model, budgetReserved } = {}) {
|
|
497
610
|
super(message, cause ? { cause } : undefined);
|
package/src/queue.mjs
CHANGED
|
@@ -16,7 +16,7 @@ export function makeQueue(connection) {
|
|
|
16
16
|
* removeOnComplete keeps the dedup window ~= the retention. Unlike webhooks, local jobs are not
|
|
17
17
|
* redelivered, so a modest window is enough.
|
|
18
18
|
*/
|
|
19
|
-
export async function enqueueLocalJob(queue, { folder, flow, task, command, provider, model, maxTurns, image, skillsDir, chainDepth, parentJobId, jobId, now = new Date() }) {
|
|
19
|
+
export async function enqueueLocalJob(queue, { folder, flow, task, command, provider, model, maxTurns, image, skillsDir, secrets, secretsProfile, chainDepth, parentJobId, jobId, now = new Date() }) {
|
|
20
20
|
const minute = now.toISOString().slice(0, 16); // YYYY-MM-DDTHH:MM -- the dedup window
|
|
21
21
|
// A caller-supplied jobId (the outbox collector's retry-idempotent chainedJobId) wins; otherwise the
|
|
22
22
|
// minute-windowed localJobId is the dedup key. A command job (issue #189) fills the flow slot with
|
|
@@ -47,6 +47,11 @@ export async function enqueueLocalJob(queue, { folder, flow, task, command, prov
|
|
|
47
47
|
// than inside `trigger` because a worker-host path is an execution knob, not a fact about the
|
|
48
48
|
// delivery -- and `trigger` is the object copied into /job/event.json.
|
|
49
49
|
...(skillsDir !== undefined && { skillsDir }),
|
|
50
|
+
// REQ-TRIGGER-SECRETS, on the forge path's terms: references only, resolved by the worker at job
|
|
51
|
+
// start. A cron trigger may bind secrets (a nightly deploy is the obvious user), which is where
|
|
52
|
+
// this differs from `replicas` -- that one is refused on a local job and this one is not.
|
|
53
|
+
...(secrets !== undefined && { secrets }),
|
|
54
|
+
...(secretsProfile !== undefined && { secretsProfile }),
|
|
50
55
|
...(chainDepth !== undefined && { chainDepth }),
|
|
51
56
|
...(parentJobId !== undefined && { parentJobId }),
|
|
52
57
|
};
|
|
@@ -124,7 +129,7 @@ export async function enqueueGitLabJob(queue, fields) {
|
|
|
124
129
|
* window, replicas never coalesce against each other, and an unflagged job's dedup id is the same string it
|
|
125
130
|
* has always been.
|
|
126
131
|
*/
|
|
127
|
-
export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, target, flow, command, trigger, provider, model, maxTurns, packages, image, skillsDir, instructions, resume, replica, replicas }) {
|
|
132
|
+
export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, target, flow, command, trigger, provider, model, maxTurns, packages, image, skillsDir, instructions, resume, secrets, secretsProfile, replica, replicas }) {
|
|
128
133
|
const jobId = forgeDeliveryJobId(kind, trigger?.deliveryId, replica);
|
|
129
134
|
// `packages` (whether to load the operator-staged pi packages) and `image` (which container image to run)
|
|
130
135
|
// come off the MATCHED trigger (INT-TRIGGERS-FILE-CONTRACT / REQ-GLOBAL-PI-OVERLAY) and land on `data`
|
|
@@ -162,6 +167,13 @@ export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, tar
|
|
|
162
167
|
// delivery, and `trigger` is what /job/event.json is built from.
|
|
163
168
|
...(instructions !== undefined && { instructions }),
|
|
164
169
|
...(resume !== undefined && { resume }),
|
|
170
|
+
// REQ-TRIGGER-SECRETS. The env variables this trigger binds and the profile whose resolver reads
|
|
171
|
+
// them. REFERENCES only: no value is ever enqueued, because a queued job is durable and a resolved
|
|
172
|
+
// credential in Redis would outlive the container it was scoped to. The worker resolves them at job
|
|
173
|
+
// start, pre-spend. At JOB level rather than inside `trigger` for image/skillsDir's reason: `trigger`
|
|
174
|
+
// is copied verbatim into /job/event.json, which an agent reads.
|
|
175
|
+
...(secrets !== undefined && { secrets }),
|
|
176
|
+
...(secretsProfile !== undefined && { secretsProfile }),
|
|
165
177
|
// Conditional for the same reason packages/image/resume are: an unflagged job's data must keep
|
|
166
178
|
// exactly the keys it has today. `replica` is this job's 1-based index and `replicas` the set size;
|
|
167
179
|
// both are integers, so the run record they land in stays PII-free by construction.
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The environment variable names `buildContainerEnv` writes itself, spelled once (issue #225).
|
|
3
|
+
*
|
|
4
|
+
* This exists because `run.secrets` lets a trigger name env variables, and a trigger that names one the
|
|
5
|
+
* closed map already owns would either be silently overwritten (the job runs without the value it asked
|
|
6
|
+
* for, on a clean exit 0) or silently WIN (a trigger redirecting `PI_OFFLINE` or `PI_MAX_TURNS`). Both
|
|
7
|
+
* are the inversion `env-allowlist.mjs`'s header exists to prevent, arriving through a new door.
|
|
8
|
+
*
|
|
9
|
+
* A SEPARATE MODULE, and it has no imports at all, on purpose. `triggers.mjs` is the shared validator:
|
|
10
|
+
* it is pure and fs-free, the receiver loads it, and `admin/build.mjs` INLINES it into the published
|
|
11
|
+
* console. Importing `env-allowlist.mjs` to reach these names would drag `node:fs`, `node:os` and pi's
|
|
12
|
+
* compat shim into all three, to read a list of strings. So the list moves down here, where both can
|
|
13
|
+
* have it for free.
|
|
14
|
+
*
|
|
15
|
+
* Only the STATIC names live here. The rest of the closed map is deployment state and cannot be known
|
|
16
|
+
* from a triggers file at all: the provider credential's variable names come from `findEnvKeys(provider,
|
|
17
|
+
* hostEnv)`, and `PI_FORWARD_ENV` is an operator env list. Those two are refused PRE-SPEND, in the
|
|
18
|
+
* processor, where the resolved provider and the host env are both in hand. `MINTED_TOKEN_VARS` and
|
|
19
|
+
* `FORGE_HOST_VARS` (forges.mjs) and `EGRESS_ENV_VARS`/`WORKER_ONLY_SECRET_VARS` (config.mjs) stay in
|
|
20
|
+
* their own modules and are imported by the validator beside this one, never copied into it.
|
|
21
|
+
*
|
|
22
|
+
* `worker/test/env-allowlist.test.mjs` pins this set against what `buildContainerEnv` actually emits, so
|
|
23
|
+
* a variable added to the closed map and not to this list fails there rather than becoming a hole here.
|
|
24
|
+
*/
|
|
25
|
+
export const CONTAINER_ENV_NAMES = new Set([
|
|
26
|
+
"PI_PROVIDER",
|
|
27
|
+
"PI_MODEL",
|
|
28
|
+
"PI_MAX_TURNS",
|
|
29
|
+
"PI_MAX_TOKENS",
|
|
30
|
+
"PI_JOB_ID",
|
|
31
|
+
"PI_GLOBAL_ALLOW_EXTENSIONS",
|
|
32
|
+
"PI_PACKAGES",
|
|
33
|
+
"PI_SESSION_FILE",
|
|
34
|
+
"PI_FLOW",
|
|
35
|
+
"PI_COMMAND",
|
|
36
|
+
"PI_OFFLINE",
|
|
37
|
+
"PLAYWRIGHT_BROWSERS_PATH",
|
|
38
|
+
"PLAYWRIGHT_MCP_BROWSER",
|
|
39
|
+
"PLAYWRIGHT_MCP_SANDBOX",
|
|
40
|
+
]);
|
package/src/run-container.mjs
CHANGED
|
@@ -7,7 +7,7 @@ import { InfraRetry } from "./processor.mjs";
|
|
|
7
7
|
|
|
8
8
|
/**
|
|
9
9
|
* The real `runContainer` the processor injects. Launches one job container and returns
|
|
10
|
-
* `{ code, aborted, turns, tokens, session, usage }`, where `aborted` records whether the WORKER initiated the stop (docker stop on
|
|
10
|
+
* `{ code, aborted, turns, tokens, session, usage, context }`, where `aborted` records whether the WORKER initiated the stop (docker stop on
|
|
11
11
|
* the 30-min timeout or graceful shutdown), which the processor classifies as POLICY (no retry) per
|
|
12
12
|
* INT-RUNNER-EXIT-CODE-PROTOCOL. The numeric `code` alone cannot say this: a worker SIGKILL and a
|
|
13
13
|
* kernel OOM both surface as 137, so the abort FLAG -- not the code -- is the discriminator.
|
|
@@ -46,7 +46,7 @@ export function makeRunContainer({
|
|
|
46
46
|
}) {
|
|
47
47
|
// async so a synchronous throw (e.g. buildContainerEnv on an unconfigured provider) surfaces as
|
|
48
48
|
// a rejection, uniformly awaitable by the processor and by tests.
|
|
49
|
-
return async function runContainer({ job, token, prepared, name, signal }) {
|
|
49
|
+
return async function runContainer({ job, token, prepared, secrets = {}, name, signal }) {
|
|
50
50
|
if (signal?.aborted) return { code: 137, aborted: true, turns: null, tokens: null, session: null, usage: null, context: null }; // killed before it could start
|
|
51
51
|
|
|
52
52
|
// Closed env allowlist: only the provider key + the declared PI_* vars. Throws (config) if
|
|
@@ -87,6 +87,10 @@ export function makeRunContainer({
|
|
|
87
87
|
// mutually exclusive with it by parse -- a job carries one or the other, never both.
|
|
88
88
|
command: typeof job.command === "string" && job.command.trim() !== "" ? job.command : undefined,
|
|
89
89
|
authFromPi, // source the provider key from pi's auth.json when the env has none
|
|
90
|
+
// REQ-TRIGGER-SECRETS: this trigger's resolved secrets, fetched by the processor BEFORE anything
|
|
91
|
+
// spent. Off the call bag rather than off `job` or the closure: it is neither a per-job fact the
|
|
92
|
+
// record may carry nor a deployment setting, it is a live credential, and `token` is its precedent.
|
|
93
|
+
secrets,
|
|
90
94
|
});
|
|
91
95
|
|
|
92
96
|
// `-net` on this container's own name (egress.mjs). null when no policy is armed, and docker-run's
|
package/src/runtime-settings.mjs
CHANGED
|
@@ -27,6 +27,22 @@ import { defaultSettingsFile } from "./config.mjs";
|
|
|
27
27
|
|
|
28
28
|
export const KNOWN_KEYS = ["model", "provider", "maxTurns", "dailyCap", "weeklyCap", "monthlyCap", "maxTokens", "dailyTokenCap", "concurrency", "softHoldPct"];
|
|
29
29
|
|
|
30
|
+
/**
|
|
31
|
+
* Overlay keys the OVERLAY accepts but no model-callable tool may set. `secretProfiles` maps a profile name
|
|
32
|
+
* to an absolute path to a script the worker executes, which is plainly "a capability the model would
|
|
33
|
+
* GAIN" -- the test `admin/src/index.ts` already applies to `run.image` and `run.resume`.
|
|
34
|
+
*
|
|
35
|
+
* The mechanism is the OMISSION from KNOWN_KEYS above: `dispatch_set` narrows its open string parameter
|
|
36
|
+
* with `KNOWN_KEYS.includes(...)`, and the panel's generic settings dialog is a `ui.select` over the same
|
|
37
|
+
* array, so leaving the key out closes both doors at once. This constant exists so that omission has a NAME
|
|
38
|
+
* and a reason attached to it. Without one, `KNOWN_KEYS` silently stops meaning "keys the overlay accepts"
|
|
39
|
+
* and the next contributor tidies the gap away as an oversight.
|
|
40
|
+
*
|
|
41
|
+
* A profile is declared through the operator-typed `/dispatch` command instead, which pi reaches only from
|
|
42
|
+
* its own user-input path, and every declared path is bounded by PI_SECRET_RESOLVER_ROOTS in the worker.
|
|
43
|
+
*/
|
|
44
|
+
export const OPERATOR_ONLY_KEYS = ["secretProfiles"];
|
|
45
|
+
|
|
30
46
|
function isNonEmptyString(value) {
|
|
31
47
|
return typeof value === "string" && value.trim() !== "";
|
|
32
48
|
}
|
|
@@ -85,6 +101,30 @@ function validateOverlay(candidate, log) {
|
|
|
85
101
|
if (!isIntInRange(value, 1, 10)) return { invalid: "concurrency must be an integer 1-10" };
|
|
86
102
|
overlay[key] = value;
|
|
87
103
|
break;
|
|
104
|
+
case "secretProfiles": {
|
|
105
|
+
// REQ-TRIGGER-SECRETS. `{ [name]: "/abs/path" }`, the panel-authored half of the resolver
|
|
106
|
+
// table. The shape is checked here and the PATH BOUND is not: whether a path is allowed is
|
|
107
|
+
// PI_SECRET_RESOLVER_ROOTS' question, and it is asked in the worker at resolution time rather
|
|
108
|
+
// than here, because this same validator runs inside the admin extension where a "yes" would
|
|
109
|
+
// prove nothing about the host the job will run on.
|
|
110
|
+
//
|
|
111
|
+
// An invalid value fails the WHOLE overlay, which is this function's documented rule and is
|
|
112
|
+
// deliberate here too: a half-read profile table is a deployment that believes it has a
|
|
113
|
+
// resolver it does not.
|
|
114
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
115
|
+
return { invalid: "secretProfiles must be an object mapping profile names to resolver paths" };
|
|
116
|
+
}
|
|
117
|
+
for (const name of Object.keys(value)) {
|
|
118
|
+
if (!/^[A-Za-z0-9._-]+$/.test(name)) {
|
|
119
|
+
return { invalid: "secretProfiles names may use letters, digits, dot, dash and underscore only" };
|
|
120
|
+
}
|
|
121
|
+
if (typeof value[name] !== "string" || value[name].trim() === "") {
|
|
122
|
+
return { invalid: `secretProfiles.${name} must be a non-empty absolute path` };
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
overlay[key] = { ...value };
|
|
126
|
+
break;
|
|
127
|
+
}
|
|
88
128
|
case "softHoldPct":
|
|
89
129
|
// A percentage of each active cap; 100 would equal the hard wall (no band) and 0 has no meaning,
|
|
90
130
|
// so the enforced band is 1-99. Absence disables the soft-hold entirely.
|