@edgehero/pi-dispatch 2.1.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +41 -5
- package/README.md +11 -5
- package/deploy/docker-compose.yml +12 -0
- package/deploy/egress-proxy.conf +28 -3
- package/deploy/pi-dispatch-egress-proxy.container +8 -2
- package/package.json +9 -2
- package/src/allocation.mjs +731 -0
- package/src/backends.mjs +243 -0
- package/src/budget.mjs +40 -4
- package/src/cli.mjs +222 -11
- package/src/config.mjs +126 -5
- package/src/daemon-facts.mjs +3 -0
- package/src/deployment-venue.mjs +1 -0
- package/src/doctor.mjs +2280 -195
- package/src/dollar-budget.mjs +373 -0
- package/src/dollar-fingerprint.mjs +83 -0
- package/src/egress-cli.mjs +316 -0
- package/src/egress-proxy-state.mjs +35 -5
- package/src/egress.mjs +12 -0
- package/src/env-allowlist.mjs +125 -6
- package/src/env-file.mjs +194 -25
- package/src/envelope.mjs +413 -0
- package/src/exit-code.mjs +22 -0
- package/src/fleet-lease.mjs +85 -25
- package/src/get-token.mjs +16 -5
- package/src/git-dirty.mjs +67 -0
- package/src/github-app-setup.mjs +6 -3
- package/src/github-host.mjs +5 -3
- package/src/host-pi.mjs +1 -1
- package/src/identity.mjs +2 -1
- package/src/image-preflight.mjs +98 -24
- package/src/image-ref.mjs +37 -0
- package/src/import-pi.mjs +4 -2
- package/src/index.mjs +407 -62
- package/src/init.mjs +18 -0
- package/src/job-id.mjs +26 -3
- package/src/live-probes.mjs +24 -9
- package/src/model-catalog.mjs +297 -0
- package/src/model-endpoints.mjs +671 -0
- package/src/model-ref.mjs +151 -0
- package/src/models-json.mjs +268 -0
- package/src/money.mjs +144 -0
- package/src/octokit-log.mjs +65 -0
- package/src/outbox-plan.mjs +218 -0
- package/src/outbox.mjs +29 -9
- package/src/output-cap.mjs +157 -0
- package/src/pause-windows.mjs +81 -2
- package/src/pi-model-loader.mjs +77 -0
- package/src/podman-stack.mjs +16 -3
- package/src/portfolio-snapshot.mjs +304 -0
- package/src/prepare-local.mjs +247 -12
- package/src/prepare.mjs +35 -3
- package/src/priorities.mjs +569 -0
- package/src/processor.mjs +599 -170
- package/src/project-id.mjs +17 -0
- package/src/projects.mjs +238 -0
- package/src/provider-steering.mjs +179 -65
- package/src/queue.mjs +111 -6
- package/src/reserved-env.mjs +31 -0
- package/src/run-container.mjs +59 -5
- package/src/run-history.mjs +379 -24
- package/src/run-mirror.mjs +30 -0
- package/src/runtime-settings.mjs +104 -9
- package/src/schedules.mjs +33 -1
- package/src/scoped-limits.mjs +447 -27
- package/src/service.mjs +15 -4
- package/src/session-store.mjs +131 -6
- package/src/start.mjs +528 -40
- package/src/triggers-file.mjs +65 -4
- package/src/triggers.mjs +141 -11
- package/src/up.mjs +308 -34
- package/src/valkey-endpoint.mjs +3 -2
package/src/queue.mjs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
1
2
|
import { Queue } from "bullmq";
|
|
2
3
|
import { assertJudgedConnection, onValkeyError } from "./connection.mjs";
|
|
3
|
-
import { chainedJobId, localJobId, deliveryJobId, gitlabDeliveryJobId, forgeDeliveryJobId } from "./job-id.mjs";
|
|
4
|
+
import { chainedJobId, localJobId, deliveryJobId, gitlabDeliveryJobId, forgeDeliveryJobId, manualTriggerJobId } from "./job-id.mjs";
|
|
4
5
|
import { targetSeparator } from "./forges.mjs";
|
|
5
6
|
import { PR_CLOSE_ACTIONS } from "./triggers.mjs";
|
|
6
7
|
|
|
@@ -62,7 +63,7 @@ export function fleetQueueNames(hosts) {
|
|
|
62
63
|
/** The name charset, duplicated from `config.mjs` deliberately: this module imports nothing. */
|
|
63
64
|
const WORKER_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
|
|
64
65
|
|
|
65
|
-
export { chainedJobId, localJobId, deliveryJobId, gitlabDeliveryJobId, forgeDeliveryJobId };
|
|
66
|
+
export { chainedJobId, localJobId, deliveryJobId, gitlabDeliveryJobId, forgeDeliveryJobId, manualTriggerJobId };
|
|
66
67
|
|
|
67
68
|
/**
|
|
68
69
|
* A queue handle. `name` defaults to the shared queue, so every existing caller is unchanged and a
|
|
@@ -84,7 +85,98 @@ export function makeQueue(connection, { name = QUEUE } = {}) {
|
|
|
84
85
|
* removeOnComplete keeps the dedup window ~= the retention. Unlike webhooks, local jobs are not
|
|
85
86
|
* redelivered, so a modest window is enough.
|
|
86
87
|
*/
|
|
87
|
-
export async function enqueueLocalJob(queue,
|
|
88
|
+
export async function enqueueLocalJob(queue, fields) {
|
|
89
|
+
return (await addLocalJob(queue, fields)).id;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* `enqueueLocalJob` for a caller that must SAY whether anything was queued (issue #524): the operator's
|
|
94
|
+
* `pi-dispatch run`. Returns `{ id, existing }`, where `existing` is null for a job this call created, and
|
|
95
|
+
* `{ queuedAt, state }` when the one-minute dedup answered with a job an identical earlier call made.
|
|
96
|
+
*
|
|
97
|
+
* The CLI printed "queued <id>" either way, so a second `run` inside the same minute announced a job while
|
|
98
|
+
* running nothing, under the id of one that had already finished. The dedup itself is deliberate and stays.
|
|
99
|
+
*
|
|
100
|
+
* How a duplicate is told apart, and why not by asking first. BullMQ answers a jobId that already exists by
|
|
101
|
+
* returning the existing id inside a Job object it built LOCALLY for this call, so the return value says nothing
|
|
102
|
+
* about whether anything was stored. Each call therefore puts a random nonce of its own on the job's data
|
|
103
|
+
* (`enqueueNonce`) and reads the stored job back AFTER the add: the stored nonce is this call's exactly when
|
|
104
|
+
* this call's add is the one BullMQ kept. Exactly one of any number of concurrent identical calls finds its own.
|
|
105
|
+
* The first version compared the add's timestamp instead, and PR #528's review refuted it: calls in the same
|
|
106
|
+
* millisecond share a timestamp, and ten in parallel all reported "created" while one job existed. A lookup
|
|
107
|
+
* BEFORE the add has the same flaw by construction, since every concurrent caller can see nothing.
|
|
108
|
+
*
|
|
109
|
+
* The nonce is on job data, never inside `trigger`, which is the object copied into /job/event.json: it is the
|
|
110
|
+
* producer's bookkeeping and the agent has no use for it. Only the two reporting producers add it (this one and
|
|
111
|
+
* `enqueueTriggerRunReporting`), so a job from the outbox collector or a cron tick keeps exactly the data it had.
|
|
112
|
+
*
|
|
113
|
+
* A separate function rather than a change to `enqueueLocalJob`'s return, because the outbox collector consumes
|
|
114
|
+
* that as a bare id and prints nothing to a person. It also keeps its one Valkey round trip: only this function
|
|
115
|
+
* pays for the read back.
|
|
116
|
+
*
|
|
117
|
+
* `state` is BullMQ's own word for where the job is (`waiting`, `active`, `completed`, `failed`, `delayed`
|
|
118
|
+
* and the rest), or null when the state could not be read; the caller then says it is already queued or
|
|
119
|
+
* done rather than inventing one. A read that fails outright returns `existing: undefined`: whether this
|
|
120
|
+
* call queued anything is then unknown, and the caller says so instead of guessing either way.
|
|
121
|
+
*/
|
|
122
|
+
export async function enqueueLocalJobReporting(queue, fields) {
|
|
123
|
+
return addReporting(queue, (enqueueNonce) => addLocalJob(queue, { ...fields, enqueueNonce }));
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Fire one cron trigger by hand (`pi-dispatch run --trigger <id>`, issue #505), reporting like
|
|
128
|
+
* `enqueueLocalJobReporting` and through the same nonce and read back, so a second call in the same minute says
|
|
129
|
+
* the first job was already queued instead of announcing one it did not make.
|
|
130
|
+
*
|
|
131
|
+
* `schedule` is the trigger's own scheduler entry from `loadSchedules`, and its `data` is passed through WHOLE, with
|
|
132
|
+
* only the nonce added: the folder, the flow or command, the task, `github`, `packages`, `resume`, the model fields,
|
|
133
|
+
* `trigger: { id, pattern }` and a job-level `portfolio`. Field by field through `addLocalJob` was the other way, and
|
|
134
|
+
* it is the one that drops a field the next issue adds to the schedule. The options are the schedule's too, so the
|
|
135
|
+
* job is not retried, as a scheduled tick is not (a scheduler job carries no `attempts`). The id is
|
|
136
|
+
* `manualTriggerJobId`'s, so `localEventContext` reports `source: "cron"` with `scheduledFor: null`.
|
|
137
|
+
*/
|
|
138
|
+
export async function enqueueTriggerRunReporting(queue, schedule, { now = new Date() } = {}) {
|
|
139
|
+
const jobId = manualTriggerJobId({ triggerId: schedule.schedulerId, now });
|
|
140
|
+
return addReporting(queue, async (enqueueNonce) => {
|
|
141
|
+
await queue.add(schedule.name, { ...schedule.data, enqueueNonce }, { ...schedule.opts, jobId });
|
|
142
|
+
return { id: jobId };
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/** The add, then the read back that says whether THIS call's add is the one stored (see `enqueueLocalJobReporting`). */
|
|
147
|
+
async function addReporting(queue, add) {
|
|
148
|
+
const enqueueNonce = randomUUID();
|
|
149
|
+
const { id } = await add(enqueueNonce);
|
|
150
|
+
let stored;
|
|
151
|
+
try {
|
|
152
|
+
stored = await queue.getJob(id);
|
|
153
|
+
} catch {
|
|
154
|
+
return { id, existing: undefined };
|
|
155
|
+
}
|
|
156
|
+
// No stored job: it was removed between the add and this read (a `cancel` racing it). Nothing says it was
|
|
157
|
+
// a duplicate, so it reads as created, which is what the add itself answered.
|
|
158
|
+
if (!stored || stored.data?.enqueueNonce === enqueueNonce) return { id, existing: null };
|
|
159
|
+
let state = null;
|
|
160
|
+
try {
|
|
161
|
+
const s = await stored.getState();
|
|
162
|
+
state = typeof s === "string" && s !== "unknown" ? s : null;
|
|
163
|
+
} catch {}
|
|
164
|
+
return { id, existing: { queuedAt: Number.isFinite(stored.timestamp) ? stored.timestamp : null, state } };
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* The one sentence for a swallowed local enqueue (issue #524), shared by `pi-dispatch run` and the admin's
|
|
169
|
+
* `/dispatch run` so the two producers an operator types cannot drift apart. `existing` is
|
|
170
|
+
* `enqueueLocalJobReporting`'s: the time is the first job's own, in this process's local time, and a state the
|
|
171
|
+
* queue could not give is said as "already queued or done" rather than guessed.
|
|
172
|
+
*/
|
|
173
|
+
export function swallowedRunSentence(jobId, existing) {
|
|
174
|
+
const at = Number.isFinite(existing?.queuedAt) ? new Date(existing.queuedAt) : null;
|
|
175
|
+
const hhmm = at ? ` at ${String(at.getHours()).padStart(2, "0")}:${String(at.getMinutes()).padStart(2, "0")}` : "";
|
|
176
|
+
return `an identical run was queued${hhmm} as ${jobId} (${existing?.state ?? "already queued or done"}); nothing new was queued.`;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
async function addLocalJob(queue, { folder, flow, task, command, provider, model, maxTurns, models, maxCostUsd, image, backend, excludeTools, skillsDir, secrets, secretsProfile, chainDepth, parentJobId, jobId, enqueueNonce, now = new Date() }) {
|
|
88
180
|
const minute = now.toISOString().slice(0, 16); // YYYY-MM-DDTHH:MM -- the dedup window
|
|
89
181
|
// A caller-supplied jobId (the outbox collector's retry-idempotent chainedJobId) wins; otherwise the
|
|
90
182
|
// minute-windowed localJobId is the dedup key. A command job (issue #189) fills the flow slot with
|
|
@@ -93,7 +185,7 @@ export async function enqueueLocalJob(queue, { folder, flow, task, command, prov
|
|
|
93
185
|
// vanish silently -- and the `cmd:` prefix keeps a command named X from colliding with a flow named X
|
|
94
186
|
// (`:` is outside the skill-name charset, so no real flow can spell the prefixed form). A flow job's
|
|
95
187
|
// key is byte-identical to before the feature.
|
|
96
|
-
const id = jobId ?? localJobId({ folder, flow: command !== undefined ? `cmd:${command}` : flow, task, minute });
|
|
188
|
+
const id = jobId ?? localJobId({ folder, flow: command !== undefined ? `cmd:${command}` : flow, task, minute, provider, model, models });
|
|
97
189
|
// image/chainDepth/parentJobId land on `data` only when present, so a plain non-chained job's data is
|
|
98
190
|
// byte-identical. `image` is the container image this job runs in (INT-TRIGGERS-FILE-CONTRACT); absent
|
|
99
191
|
// resolves the deployment default at job start, never a value frozen here.
|
|
@@ -109,6 +201,11 @@ export async function enqueueLocalJob(queue, { folder, flow, task, command, prov
|
|
|
109
201
|
provider,
|
|
110
202
|
model,
|
|
111
203
|
maxTurns,
|
|
204
|
+
// Issue #502: the allowed-model list, conditional like `image`, so a job whose trigger named none keeps its
|
|
205
|
+
// data byte-identical. An outbox child carries its parent's (outbox.mjs), never one its request file names.
|
|
206
|
+
...(models !== undefined && { models }),
|
|
207
|
+
// Issue #501: the per-job dollar cap, passed through for the outbox child that inherits its parent's.
|
|
208
|
+
...(maxCostUsd !== undefined && { maxCostUsd }),
|
|
112
209
|
...(image !== undefined && { image }),
|
|
113
210
|
// #227. WHERE this job's container is built. Conditional like `image`, so a trigger that named no
|
|
114
211
|
// venue produces byte-identical job data -- and at JOB level, never inside `trigger`, for the reason
|
|
@@ -132,6 +229,8 @@ export async function enqueueLocalJob(queue, { folder, flow, task, command, prov
|
|
|
132
229
|
...(secretsProfile !== undefined && { secretsProfile }),
|
|
133
230
|
...(chainDepth !== undefined && { chainDepth }),
|
|
134
231
|
...(parentJobId !== undefined && { parentJobId }),
|
|
232
|
+
// Issue #524: `enqueueLocalJobReporting`'s own mark, so it can tell whether ITS add is the one stored.
|
|
233
|
+
...(enqueueNonce !== undefined && { enqueueNonce }),
|
|
135
234
|
};
|
|
136
235
|
await queue.add("local", data, {
|
|
137
236
|
jobId: id,
|
|
@@ -140,7 +239,7 @@ export async function enqueueLocalJob(queue, { folder, flow, task, command, prov
|
|
|
140
239
|
removeOnComplete: { age: 24 * 3600 },
|
|
141
240
|
removeOnFail: { age: 7 * 24 * 3600 },
|
|
142
241
|
});
|
|
143
|
-
return id;
|
|
242
|
+
return { id };
|
|
144
243
|
}
|
|
145
244
|
|
|
146
245
|
// Coalesces rapid re-label spam; the GUID jobId + 31d retention handle exact redelivery, so this
|
|
@@ -211,7 +310,7 @@ export async function enqueueGitLabJob(queue, fields) {
|
|
|
211
310
|
* window, replicas never coalesce against each other, and an unflagged job's dedup id is the same string it
|
|
212
311
|
* has always been.
|
|
213
312
|
*/
|
|
214
|
-
export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, target, flow, command, trigger, provider, model, maxTurns, packages, image, backend, excludeTools, skillsDir, instructions, resume, secrets, secretsProfile, waitFor, replica, replicas }) {
|
|
313
|
+
export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, target, flow, command, trigger, provider, model, maxTurns, models, maxCostUsd, packages, image, backend, excludeTools, skillsDir, instructions, resume, secrets, secretsProfile, waitFor, replica, replicas }) {
|
|
215
314
|
const jobId = forgeDeliveryJobId(kind, trigger?.deliveryId, replica);
|
|
216
315
|
// `packages` (whether to load the operator-staged pi packages) and `image` (which container image to run)
|
|
217
316
|
// come off the MATCHED trigger (INT-TRIGGERS-FILE-CONTRACT / REQ-GLOBAL-PI-OVERLAY) and land on `data`
|
|
@@ -237,6 +336,9 @@ export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, tar
|
|
|
237
336
|
provider,
|
|
238
337
|
model,
|
|
239
338
|
maxTurns,
|
|
339
|
+
// Issue #502: the allowed-model list, conditional like `packages` below so an unflagged trigger's job data is
|
|
340
|
+
// byte-identical, and at JOB level, never inside `trigger` (copied verbatim into /job/event.json).
|
|
341
|
+
...(models !== undefined && { models }),
|
|
240
342
|
...(packages !== undefined && { packages }),
|
|
241
343
|
...(image !== undefined && { image }),
|
|
242
344
|
// #227. WHERE this job's container is built. Conditional like `image`, so a trigger that named no
|
|
@@ -249,6 +351,9 @@ export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, tar
|
|
|
249
351
|
// same reason: `trigger` is copied verbatim into /job/event.json, and a permission boundary is the
|
|
250
352
|
// worker's business, never agent-visible input to reason about.
|
|
251
353
|
...(excludeTools !== undefined && { excludeTools }),
|
|
354
|
+
// #501. The trigger's per-job dollar cap, as written. Conditional like `excludeTools`, at JOB level for its
|
|
355
|
+
// reason, and a narrowing like it: the worker runs the job under the smaller of this and the deployment cap.
|
|
356
|
+
...(maxCostUsd !== undefined && { maxCostUsd }),
|
|
252
357
|
// The host directory of operator-authored skills this trigger injects (REQ-PER-TRIGGER-SKILLS).
|
|
253
358
|
// Conditional like `image`, so an unflagged job's data stays byte-identical, and at JOB level rather
|
|
254
359
|
// than inside `trigger` because a worker-host path is an execution knob, not a fact about the
|
package/src/reserved-env.mjs
CHANGED
|
@@ -1,3 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Issue #503: the variable a keyless provider's `models.json` key names (`"apiKey": "$PI_DISPATCH_KEYLESS"`), so pi
|
|
3
|
+
* composes the provider. `buildContainerEnv` writes it, with the fixed, non-secret value `keyless`, only when the
|
|
4
|
+
* credential gate passed the job's provider keyless (env-allowlist.mjs, `resolveProviderCredential`): a provider pi does
|
|
5
|
+
* not know, served by keyless model endpoints alone. Reserved in the set below, so neither `run.secrets` nor
|
|
6
|
+
* `PI_FORWARD_ENV` can set it.
|
|
7
|
+
*/
|
|
8
|
+
export const KEYLESS_ENV_NAME = "PI_DISPATCH_KEYLESS";
|
|
9
|
+
|
|
1
10
|
/**
|
|
2
11
|
* The environment variable names `buildContainerEnv` writes itself, spelled once (issue #225).
|
|
3
12
|
*
|
|
@@ -34,6 +43,14 @@ export const CONTAINER_ENV_NAMES = new Set([
|
|
|
34
43
|
"PI_FLOW",
|
|
35
44
|
"PI_COMMAND",
|
|
36
45
|
"PI_EXCLUDE_TOOLS",
|
|
46
|
+
// Issues #501, #502: the per-job dollar cap (written by the worker since #501, from the trigger's and the
|
|
47
|
+
// deployment's caps) and the allowed-model list (reserved until #502 writes it). Neither can be bound through
|
|
48
|
+
// `run.secrets`: a trigger setting its own cap, or its own list, would be the policy choosing itself.
|
|
49
|
+
"PI_MAX_COST_MICROS",
|
|
50
|
+
"PI_ALLOWED_MODELS",
|
|
51
|
+
// Issue #545: tells the runner the exit line's key waits on stdin. A trigger setting it would block its own runner
|
|
52
|
+
// on a stdin no one writes, or turn the signed line off.
|
|
53
|
+
"PI_EXIT_AUTH",
|
|
37
54
|
"PI_OFFLINE",
|
|
38
55
|
"PLAYWRIGHT_BROWSERS_PATH",
|
|
39
56
|
"PLAYWRIGHT_MCP_BROWSER",
|
|
@@ -41,4 +58,18 @@ export const CONTAINER_ENV_NAMES = new Set([
|
|
|
41
58
|
// Issue #341: set beside `--user` so a uid with no passwd entry has a writable home. A new reservation, so a
|
|
42
59
|
// triggers file binding a secret named HOME is now refused at parse (worker, receiver and admin alike).
|
|
43
60
|
"HOME",
|
|
61
|
+
// Issue #503: written only on the keyless branch of the credential gate (see KEYLESS_ENV_NAME at the top).
|
|
62
|
+
KEYLESS_ENV_NAME,
|
|
44
63
|
]);
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Issue #500: the names the RUNNER sets in its own environment inside the container, for its descendants: the child
|
|
67
|
+
* ledger directory and the runner's pid (image/runner/src/usage-meter.mjs, openChildLedger), and the hash of the price
|
|
68
|
+
* table the runner hands its children (publishPriceTable, issue #587's review). The worker never writes
|
|
69
|
+
* them, so they are not in the set above (which a test pins to what buildContainerEnv emits). They are reserved all the
|
|
70
|
+
* same: a trigger binding one through `run.secrets`, or a host value forwarded through `PI_FORWARD_ENV`, would arrive in
|
|
71
|
+
* the runner's environment before the runner sets its own, and a value the runner then failed to replace would point
|
|
72
|
+
* every pi child's ledger at a directory the agent chose. Refused at load in both lists, and deleted by
|
|
73
|
+
* buildContainerEnv after both loops as the backstop.
|
|
74
|
+
*/
|
|
75
|
+
export const RUNNER_ENV_NAMES = new Set(["PI_DISPATCH_CHILD_LEDGER", "PI_DISPATCH_PRICE_TABLE", "PI_DISPATCH_RUNNER_PID"]);
|
package/src/run-container.mjs
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
|
+
import { randomBytes } from "node:crypto";
|
|
2
3
|
import { readFileSync, rmSync } from "node:fs";
|
|
3
4
|
import { DOCKER_NEVER_STARTED_EXITS } from "./backends.mjs";
|
|
4
5
|
import { CONTAINER_HOME } from "./container-spec.mjs";
|
|
@@ -61,14 +62,24 @@ export function makeRunContainer({
|
|
|
61
62
|
// `log` names a teardown the gate refused, which leaves the job's network for the boot reaper.
|
|
62
63
|
teardownRuntime = null,
|
|
63
64
|
log = () => {},
|
|
65
|
+
// Issue #545: the per-job exit-line key, 32 random bytes as hex. A seam so a test can name the key it verifies with.
|
|
66
|
+
mintExitKey = () => randomBytes(32).toString("hex"),
|
|
64
67
|
}) {
|
|
65
68
|
// async so a synchronous throw (e.g. buildContainerEnv on an unconfigured provider) surfaces as
|
|
66
69
|
// a rejection, uniformly awaitable by the processor and by tests.
|
|
67
70
|
// `relabel` (issue #355) is the processor's, off the same job-user answer as `user`: true where the daemon confines
|
|
68
71
|
// containers with SELinux, so the worker's own per-job mounts carry `:Z`. Defaults off, so a caller that predates it
|
|
69
72
|
// builds exactly the argv it always did.
|
|
70
|
-
|
|
73
|
+
// `modelEndpoints` (issue #503) is the pickup's endpoint snapshot, the one the free credential gate decided on, so a
|
|
74
|
+
// keyless provider's PI_DISPATCH_KEYLESS is written from the same declaration and never from a second read.
|
|
75
|
+
// `exitAuth` (issue #545) is the processor's, off the image preflight: true when the job image declares `exitAuth`, so
|
|
76
|
+
// its runner reads a key from stdin and signs its exit line with it. Defaults off, so a caller that predates it, and
|
|
77
|
+
// every image that does not declare it, runs exactly as before and is read exactly as before.
|
|
78
|
+
return async function runContainer({ job, token, prepared, secrets = {}, name, signal, user = null, home = null, relabel = false, modelEndpoints = null, exitAuth = false }) {
|
|
71
79
|
if (signal?.aborted) return { code: 137, aborted: true, turns: null, tokens: null, session: null, usage: null, context: null, exitReason: null }; // killed before it could start
|
|
80
|
+
// Minted per attempt, held in this closure and the sink's, and handed to the container on its stdin only: never in
|
|
81
|
+
// the argv (a host `ps` shows it), never in the env (the container's /proc/1/environ shows it), never logged.
|
|
82
|
+
const exitKey = exitAuth === true ? mintExitKey() : null;
|
|
72
83
|
// Issue #341. `user` and `home` travel as a PAIR: a uid with no passwd entry in the image gets `HOME=/` from
|
|
73
84
|
// Docker and `HOME=/workspace` from Podman (measured), so a `--user` without this HOME is refused here rather
|
|
74
85
|
// than started. The builder does not insist, because `doctor --live`'s probes run `--user` with no environment.
|
|
@@ -87,6 +98,7 @@ export function makeRunContainer({
|
|
|
87
98
|
model: job.model,
|
|
88
99
|
maxTurns: job.maxTurns,
|
|
89
100
|
maxTokens: job.maxTokens, // optional per-job token budget (issue #25); undefined => runner meter only
|
|
101
|
+
maxCostMicros: job.maxCostMicros ?? null, // issue #501: effectiveJobOf's min of trigger and deployment; null => no cap
|
|
90
102
|
jobId: name,
|
|
91
103
|
githubToken: token ?? undefined,
|
|
92
104
|
// Which forge minted it, so the token lands in that forge's own variable names and no other.
|
|
@@ -121,12 +133,17 @@ export function makeRunContainer({
|
|
|
121
133
|
// non-empty validated array; the guard is the same defensive shape `flow` above wears, so a
|
|
122
134
|
// hand-built job with junk in the field emits no variable rather than an empty one.
|
|
123
135
|
excludeTools: Array.isArray(job.excludeTools) && job.excludeTools.length > 0 ? job.excludeTools : undefined,
|
|
136
|
+
// Issue #502: the effective allowed-model list, off `job` like excludeTools. index.mjs `effectiveJobOf` folds
|
|
137
|
+
// the deployment's PI_ALLOWED_MODELS in under the trigger's own, so `job.models` is already the one list.
|
|
138
|
+
allowedModels: Array.isArray(job.models) && job.models.length > 0 ? job.models : null,
|
|
124
139
|
authFromPi, // source the provider key from pi's auth.json when the env has none
|
|
125
140
|
// REQ-TRIGGER-SECRETS: this trigger's resolved secrets, fetched by the processor BEFORE anything
|
|
126
141
|
// spent. Off the call bag rather than off `job` or the closure: it is neither a per-job fact the
|
|
127
142
|
// record may carry nor a deployment setting, it is a live credential, and `token` is its precedent.
|
|
128
143
|
secrets,
|
|
129
144
|
home: user !== null ? home : null, // issue #341: HOME only beside --user, assigned after the forward loops
|
|
145
|
+
modelEndpoints, // issue #503: the keyless branch of the credential, from the gate's own snapshot
|
|
146
|
+
exitAuth: exitKey !== null, // issue #545: PI_EXIT_AUTH=stdin, only when a key is actually handed over below
|
|
130
147
|
});
|
|
131
148
|
|
|
132
149
|
// `-net` on this container's own name (egress.mjs). null when no policy is armed, and docker-run's
|
|
@@ -168,6 +185,9 @@ export function makeRunContainer({
|
|
|
168
185
|
// Both facts, the kind the preparers branch on AND containment in this job's own directory, so a kind added later
|
|
169
186
|
// that works on a folder in place fails closed rather than relabelling it.
|
|
170
187
|
workspaceOwned: job?.kind !== "local" && insideDir(prepared.jobDir, prepared.workspace),
|
|
188
|
+
// Issue #545: `-i`, so the container's stdin is the pipe the key is written to. Absent otherwise, and the argv is
|
|
189
|
+
// byte-identical to one built before the signed exit line.
|
|
190
|
+
...(exitKey !== null ? { extraFlags: ["-i"] } : {}),
|
|
171
191
|
});
|
|
172
192
|
|
|
173
193
|
// REQ-EGRESS-ALLOWLIST. This job's own --internal network, created here rather than at boot because
|
|
@@ -184,10 +204,17 @@ export function makeRunContainer({
|
|
|
184
204
|
|
|
185
205
|
// Host-side per-job log sink, teed off `onOutput`. `name` is `pi-job-<jobId>`; the sink
|
|
186
206
|
// sanitizes internally. No container mount, no env var -- the sink lives on this side only.
|
|
187
|
-
const sink = openJobLog(name);
|
|
207
|
+
const sink = openJobLog(name, { exitKey });
|
|
188
208
|
|
|
189
209
|
const run = new Promise((resolve, reject) => {
|
|
190
|
-
const child = spawnFn(bin, args, { stdio: ["ignore", "pipe", "pipe"] });
|
|
210
|
+
const child = spawnFn(bin, args, { stdio: [exitKey !== null ? "pipe" : "ignore", "pipe", "pipe"] });
|
|
211
|
+
if (exitKey !== null) {
|
|
212
|
+
// Written and closed at once: the runner drains stdin to EOF before anything else runs, so the key is out of
|
|
213
|
+
// the pipe before any tool exists. A CLI that died first makes this EPIPE, which is the run failing, reported
|
|
214
|
+
// by `error` or `close` below; the listener only keeps the write's own error from crashing the worker.
|
|
215
|
+
child.stdin?.on("error", () => {});
|
|
216
|
+
child.stdin?.end(`${exitKey}\n`);
|
|
217
|
+
}
|
|
191
218
|
// A throwing sink.write is swallowed so a misbehaving sink cannot break the tee or hang the run.
|
|
192
219
|
const tee = (chunk) => {
|
|
193
220
|
onOutput(chunk);
|
|
@@ -216,12 +243,19 @@ export function makeRunContainer({
|
|
|
216
243
|
// RUNNER_POLICY_REASONS set and to a line that itself said code 2. The processor still decides
|
|
217
244
|
// the retry class from `code` alone; this only picks the label inside exit 2.
|
|
218
245
|
let exitReason = null;
|
|
246
|
+
// Issue #501 (PR #542's review, round 3): the LAST exit line's own `code`, which the dollar settlement
|
|
247
|
+
// compares with the container's real exit code before it trusts that line's cost.
|
|
248
|
+
let exitLineCode = null;
|
|
249
|
+
// Issue #507: the cost guard's rule on a `cost-cap` line, already filtered by parseExitWhy to COST_CAP_WHYS.
|
|
250
|
+
let exitWhy = null;
|
|
251
|
+
// Issue #545: null (no key issued), "verified" or "unverified" (a key, and no exit line carried it).
|
|
252
|
+
let exitAuthResult = null;
|
|
219
253
|
try {
|
|
220
254
|
// `context = null` is a DEFAULT rather than a plain destructure: an injected sink that
|
|
221
255
|
// predates the field returns no such key, and `undefined` would then reach the record's
|
|
222
256
|
// shape where every other absence is spelled `null`.
|
|
223
257
|
// `exitReason` defaults the same way, for the same reason.
|
|
224
|
-
({ turns, tokens, session, usage, context = null, exitReason = null } = await sink.close());
|
|
258
|
+
({ turns, tokens, session, usage, context = null, exitReason = null, exitLineCode = null, exitWhy = null, exitAuth: exitAuthResult = null } = await sink.close());
|
|
225
259
|
} catch {
|
|
226
260
|
turns = null;
|
|
227
261
|
tokens = null;
|
|
@@ -229,8 +263,28 @@ export function makeRunContainer({
|
|
|
229
263
|
usage = null;
|
|
230
264
|
context = null;
|
|
231
265
|
exitReason = null;
|
|
266
|
+
exitLineCode = null;
|
|
267
|
+
exitWhy = null;
|
|
268
|
+
// The sink could not say, and a key was issued: nothing it returned was verified.
|
|
269
|
+
exitAuthResult = exitKey !== null ? "unverified" : null;
|
|
270
|
+
}
|
|
271
|
+
// A sink that predates the field returned none; with a key issued, that is still nothing verified.
|
|
272
|
+
if (exitKey !== null && exitAuthResult !== "verified") {
|
|
273
|
+
exitAuthResult = "unverified";
|
|
274
|
+
turns = null;
|
|
275
|
+
tokens = null;
|
|
276
|
+
session = null;
|
|
277
|
+
usage = null;
|
|
278
|
+
context = null;
|
|
279
|
+
exitReason = null;
|
|
280
|
+
exitLineCode = null;
|
|
281
|
+
exitWhy = null;
|
|
232
282
|
}
|
|
233
|
-
|
|
283
|
+
// Spread only when a key was issued, so a run without one resolves the very object it always did.
|
|
284
|
+
const auth = exitKey !== null ? { exitAuth: exitAuthResult } : {};
|
|
285
|
+
// `exitWhy` only when the line named one, so every other run resolves the object it always did.
|
|
286
|
+
const why = exitWhy !== null ? { exitWhy } : {};
|
|
287
|
+
resolve(aborted ? { code: code ?? 137, aborted: true, turns, tokens, session, usage, context, exitReason, exitLineCode, ...why, ...auth } : { code: code ?? 1, aborted: false, turns, tokens, session, usage, context, exitReason, exitLineCode, ...why, ...auth });
|
|
234
288
|
});
|
|
235
289
|
});
|
|
236
290
|
|