@edgehero/pi-dispatch 2.1.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/.env.example +41 -5
  2. package/README.md +11 -5
  3. package/deploy/docker-compose.yml +12 -0
  4. package/deploy/egress-proxy.conf +28 -3
  5. package/deploy/pi-dispatch-egress-proxy.container +8 -2
  6. package/package.json +8 -1
  7. package/src/allocation.mjs +731 -0
  8. package/src/backends.mjs +243 -0
  9. package/src/budget.mjs +40 -4
  10. package/src/cli.mjs +222 -11
  11. package/src/config.mjs +126 -5
  12. package/src/daemon-facts.mjs +3 -0
  13. package/src/deployment-venue.mjs +1 -0
  14. package/src/doctor.mjs +2261 -203
  15. package/src/dollar-budget.mjs +373 -0
  16. package/src/dollar-fingerprint.mjs +83 -0
  17. package/src/egress-cli.mjs +316 -0
  18. package/src/egress-proxy-state.mjs +35 -5
  19. package/src/egress.mjs +12 -0
  20. package/src/env-allowlist.mjs +107 -6
  21. package/src/env-file.mjs +194 -25
  22. package/src/envelope.mjs +413 -0
  23. package/src/exit-code.mjs +22 -0
  24. package/src/fleet-lease.mjs +85 -25
  25. package/src/get-token.mjs +16 -5
  26. package/src/git-dirty.mjs +67 -0
  27. package/src/github-app-setup.mjs +6 -3
  28. package/src/github-host.mjs +5 -3
  29. package/src/identity.mjs +2 -1
  30. package/src/image-preflight.mjs +98 -24
  31. package/src/image-ref.mjs +37 -0
  32. package/src/import-pi.mjs +4 -2
  33. package/src/index.mjs +407 -62
  34. package/src/init.mjs +18 -0
  35. package/src/job-id.mjs +26 -3
  36. package/src/live-probes.mjs +24 -9
  37. package/src/model-catalog.mjs +297 -0
  38. package/src/model-endpoints.mjs +649 -0
  39. package/src/model-ref.mjs +151 -0
  40. package/src/models-json.mjs +262 -0
  41. package/src/money.mjs +144 -0
  42. package/src/octokit-log.mjs +65 -0
  43. package/src/outbox-plan.mjs +218 -0
  44. package/src/outbox.mjs +29 -9
  45. package/src/output-cap.mjs +157 -0
  46. package/src/pause-windows.mjs +81 -2
  47. package/src/pi-model-loader.mjs +77 -0
  48. package/src/podman-stack.mjs +16 -3
  49. package/src/portfolio-snapshot.mjs +304 -0
  50. package/src/prepare-local.mjs +247 -12
  51. package/src/prepare.mjs +35 -3
  52. package/src/priorities.mjs +569 -0
  53. package/src/processor.mjs +599 -170
  54. package/src/project-id.mjs +17 -0
  55. package/src/projects.mjs +238 -0
  56. package/src/provider-steering.mjs +179 -65
  57. package/src/queue.mjs +111 -6
  58. package/src/reserved-env.mjs +30 -0
  59. package/src/run-container.mjs +59 -5
  60. package/src/run-history.mjs +379 -24
  61. package/src/run-mirror.mjs +30 -0
  62. package/src/runtime-settings.mjs +104 -9
  63. package/src/schedules.mjs +33 -1
  64. package/src/scoped-limits.mjs +447 -27
  65. package/src/service.mjs +15 -4
  66. package/src/session-store.mjs +131 -6
  67. package/src/start.mjs +528 -40
  68. package/src/triggers-file.mjs +65 -4
  69. package/src/triggers.mjs +135 -7
  70. package/src/up.mjs +308 -34
  71. package/src/valkey-endpoint.mjs +3 -2
package/src/cli.mjs CHANGED
@@ -1,12 +1,13 @@
1
1
  #!/usr/bin/env node
2
- import { existsSync } from "node:fs";
3
- import { resolve } from "node:path";
2
+ import { existsSync, readFileSync } from "node:fs";
3
+ import { join, resolve } from "node:path";
4
4
  import { parseArgs } from "node:util";
5
5
  import { loadConfig } from "./config.mjs";
6
- import { EXIT_POLICY, installRejectionPrinter } from "./exit-code.mjs";
6
+ import { EXIT_POLICY, installRejectionPrinter, installStdoutPipeGuard } from "./exit-code.mjs";
7
7
  import { isEntryModule } from "./entry.mjs";
8
- import { gitDirty } from "./git-dirty.mjs";
8
+ import { gitDirty, localRepoProblem } from "./git-dirty.mjs";
9
9
  import { imageRefProblem } from "./image-ref.mjs";
10
+ import { resolveServiceEnv, serviceEnvFileOf, serviceEnvLoader } from "./service-env.mjs";
10
11
 
11
12
  /** How long the kill switch waits on the host registry before acting on the shared queue alone. */
12
13
  const FLEET_READ_TIMEOUT_MS = 2_000;
@@ -27,6 +28,12 @@ const USAGE = `pi-dispatch — run pi coding-agent flows on your own folders
27
28
 
28
29
  pi-dispatch run <folder> --task "<what to do>" [--flow <name>]
29
30
  [--provider <p>] [--model <m>] [--max-turns <n>] [--image <ref>] [--force]
31
+ pi-dispatch run --trigger <cron id>
32
+ fire one cron trigger now, once, exactly as its schedule would: its folder, flow,
33
+ task and every other field come from the triggers file (PI_TRIGGERS_FILE, else
34
+ ./triggers.json here); a second call in the same minute queues nothing.
35
+ Both run forms take PI_WORKER_NAME (and --trigger PI_TRIGGERS_FILE and
36
+ PI_MAX_COST_USD) from this shell, else from ./.env here, refusing a disagreement
30
37
  pi-dispatch sandbox <jobId>
31
38
  re-open a finished run's sandbox as a shell — same image, same workspace,
32
39
  no credentials [--publish <port>[:<containerPort>]] [--pin]
@@ -39,6 +46,9 @@ const USAGE = `pi-dispatch — run pi coding-agent flows on your own folders
39
46
  run the worker (or --receiver) as an OS service — the deploy/ templates
40
47
  rendered with this host's real paths, installed user-level;
41
48
  \`service restart --drain\` lets the in-flight job finish first
49
+ pi-dispatch egress render
50
+ write model-endpoints.conf here from model-endpoints.json (in place), then print
51
+ the command that reloads the egress proxy; it never reloads the proxy itself
42
52
  pi-dispatch pause stop taking new jobs (durable; survives worker restart)
43
53
  pi-dispatch resume resume taking jobs
44
54
  pi-dispatch status show paused state + job counts
@@ -54,7 +64,7 @@ with a consent per action: pi install npm:@edgehero/pi-dispatch-admin`;
54
64
  // injects a collector instead of reassigning `process.stdout.write`. That matters because `node --test`
55
65
  // runs each file in a child process that serialises its own results over that same stdout, so a test
56
66
  // holding a replacement across an `await` swallows the runner's result frames (issue #266).
57
- export async function main(argv = process.argv.slice(2), env = process.env, { write = (chunk) => process.stdout.write(chunk), valkeyRefusal = valkeyRefusalAtStart } = {}) {
67
+ export async function main(argv = process.argv.slice(2), env = process.env, { write = (chunk) => process.stdout.write(chunk), valkeyRefusal = valkeyRefusalAtStart, now = () => new Date() } = {}) {
58
68
  const cmd = argv[0];
59
69
 
60
70
  if (cmd === "init") {
@@ -107,6 +117,11 @@ export async function main(argv = process.argv.slice(2), env = process.env, { wr
107
117
  return runService(argv.slice(1), { env });
108
118
  }
109
119
 
120
+ if (cmd === "egress") {
121
+ const { runEgress } = await import("./egress-cli.mjs");
122
+ return runEgress(argv.slice(1), { env, out: write });
123
+ }
124
+
110
125
  if (cmd === "run") {
111
126
  const { values, positionals } = parseArgs({
112
127
  args: argv.slice(1),
@@ -119,8 +134,10 @@ export async function main(argv = process.argv.slice(2), env = process.env, { wr
119
134
  "max-turns": { type: "string" },
120
135
  image: { type: "string" }, // the container image for this one job; blank/absent = PI_JOB_IMAGE
121
136
  force: { type: "boolean", default: false },
137
+ trigger: { type: "string" }, // issue #505: fire this cron trigger once, by hand
122
138
  },
123
139
  });
140
+ if (values.trigger !== undefined) return runTrigger(values, positionals, env, { write, valkeyRefusal, now });
124
141
  const folder = positionals[0] && resolve(positionals[0]);
125
142
  if (!folder || !existsSync(folder)) return fail(`folder not found: ${positionals[0] ?? "(none given)"}`);
126
143
  if (!values.task) return fail("a --task is required");
@@ -131,21 +148,31 @@ export async function main(argv = process.argv.slice(2), env = process.env, { wr
131
148
  if (problem) return fail(`--image ${problem.reason} (got ${JSON.stringify(values.image)})`);
132
149
  }
133
150
 
151
+ // The worker's folder rule, before anything is queued (issue #524): `.git` at the folder itself and a commit at
152
+ // HEAD. A folder that fails it used to be queued anyway and refused at pickup as a generic `config-refused`.
153
+ // Not waved away by --force: that flag accepts uncommitted work, and the worker refuses this either way.
154
+ const notARepo = localRepoProblem(folder);
155
+ if (notARepo) return fail(`${notARepo} Nothing was queued.`);
156
+
134
157
  // A local job edits the folder IN PLACE with no undo (SECURITY.md). Refuse a dirty working
135
158
  // tree unless --force, so a bad run cannot mix with uncommitted work the operator can't
136
- // cleanly separate. A non-git folder is caught later by prepare (v1 requires a git repo).
137
- if (existsSync(`${folder}/.git`) && !values.force) {
159
+ // cleanly separate.
160
+ if (!values.force) {
138
161
  const dirty = gitDirty(folder);
139
162
  if (dirty === null) return fail(`${folder} is not a usable git repository`);
140
163
  if (dirty) return fail(`${folder} has uncommitted changes. Commit or stash them, or pass --force.`);
141
164
  }
142
165
 
143
- const config = loadConfig(env);
166
+ // Which host queue (review of PR #575): PI_WORKER_NAME by the deployment's rule, as VALKEY_URL below is, so `run`
167
+ // from the deployment folder queues where that folder's worker drains even when this shell does not export it.
168
+ const deployment = cliDeploymentEnv(env, ["PI_WORKER_NAME"]);
169
+ if (deployment.problem) return fail(`${deployment.problem}. Nothing was queued.`);
170
+ const config = loadConfig(deployment.env);
144
171
  const { cliValkeyUrl, parseConnection } = await import("./connection.mjs");
145
172
  // PR #475's review: VALKEY_URL as the password is read, this shell's else the deployment .env's (a disagreement
146
173
  // named), not the shell's alone: from the folder of a Valkey on another port, `run` dialled 6379.
147
174
  const valkeyUrl = cliValkeyUrl(env);
148
- const { makeQueue, enqueueLocalJob, hostQueueName } = await import("./queue.mjs");
175
+ const { makeQueue, enqueueLocalJobReporting, hostQueueName, swallowedRunSentence } = await import("./queue.mjs");
149
176
  // failFast: a one-shot enqueue must not hang forever if Valkey is down -- error clearly.
150
177
  // Onto THIS host's queue when the deployment declares a name (issue #57). The folder was checked
151
178
  // against this machine's filesystem a few lines up, so this machine is the only one that can run it;
@@ -159,7 +186,7 @@ export async function main(argv = process.argv.slice(2), env = process.env, { wr
159
186
  try {
160
187
  // Absent flags stay absent (undefined) so the value resolves at job start against the
161
188
  // settings overlay/env, not a default frozen here (INT-CONFIG-OVERLAY-CONTRACT).
162
- const jobId = await enqueueLocalJob(queue, {
189
+ const { id: jobId, existing } = await enqueueLocalJobReporting(queue, {
163
190
  folder,
164
191
  task: values.task,
165
192
  flow: values.flow,
@@ -169,8 +196,12 @@ export async function main(argv = process.argv.slice(2), env = process.env, { wr
169
196
  // || not the raw value: `--image ""` must collapse to absent rather than becoming a falsy string that
170
197
  // throws inside buildDockerRunArgs after a budget slot is reserved.
171
198
  image: values.image || undefined,
199
+ now: now(),
172
200
  });
173
- write(`queued ${jobId} — folder ${folder}\nrun \`pi-dispatch worker\` to process it.\n`);
201
+ // Issue #530: the closing line used to say "run `pi-dispatch worker` to process it" while workers ran. It now
202
+ // says what this queue shows, asked on the connection the enqueue already holds, and only when a job was queued.
203
+ const hint = existing ? null : await workerHint(queue);
204
+ write(runQueuedLine({ jobId, existing, folder, hint }, { swallowedRunSentence }));
174
205
  } catch (error) {
175
206
  return fail(error?.valkeyRefused ? error.message : `could not reach Valkey at ${(await import("./connection.mjs")).urlShown(valkeyUrl)}: ${(await import("./valkey-auth.mjs")).valkeyDownHint(valkeyUrl)}\n ${error.message}`);
176
207
  } finally {
@@ -241,6 +272,166 @@ async function killSwitchUrls(args, env) {
241
272
  return { ...picked, positionals: parsed.positionals };
242
273
  }
243
274
 
275
+ /**
276
+ * `pi-dispatch run --trigger <cron id>` (issue #505): fire one cron trigger now, once, as its schedule would. Typed by
277
+ * an operator only; no tool calls it. It is the way to run a `run.portfolio` job without waiting for its schedule
278
+ * (`pi-dispatch run <folder>` makes a manual job, which never carries the flag).
279
+ *
280
+ * WHICH FILE: `PI_TRIGGERS_FILE`, else `./triggers.json` in this directory, doctor's rule (`triggersPath`) and the
281
+ * one-shot rule the worker's own live checks read by, so the command and the worker's check of the portfolio flag
282
+ * read one file. Not `config.triggersFile`, whose null means "the worker schedules no cron": a trigger fired by hand
283
+ * does not need the worker's scheduler.
284
+ *
285
+ * WHAT IS QUEUED: the trigger's own scheduler entry, from `loadSchedules` over that file (so a file the worker would
286
+ * refuse is refused here, with the loader's message), and its data passed through whole. Nothing on the command line
287
+ * can change it, so every other `run` flag is refused beside `--trigger`. The job id is `manual:<id>:<minute>`,
288
+ * deduplicated through the same read back as `pi-dispatch run`.
289
+ *
290
+ * WHERE: on the queue the worker of this host schedules the trigger on (its own host queue when the deployment
291
+ * declares a name, else the shared one), because the trigger's folder is on this machine. A trigger whose folder is
292
+ * not here belongs to another host, and is refused with the command to run there: queued here, no worker could run it.
293
+ */
294
+ async function runTrigger(values, positionals, env, { write, valkeyRefusal, now }) {
295
+ const id = values.trigger;
296
+ const extra = ["task", "flow", "provider", "model", "max-turns", "image"].filter((k) => values[k] !== undefined);
297
+ if (values.force) extra.push("force");
298
+ if (positionals.length > 0 || extra.length > 0) {
299
+ return fail(`--trigger takes no folder and no other flag (got ${[...positionals.map((p) => JSON.stringify(p)), ...extra.map((k) => `--${k}`)].join(" ")}): the trigger's own fields are what runs. Nothing was queued.`);
300
+ }
301
+ // PI_TRIGGERS_FILE and PI_WORKER_NAME by the deployment's rule (review of PR #575), the one VALKEY_URL is read by
302
+ // below: which file and which host queue are the deployment's facts, and a shell that does not export them must not
303
+ // read ./triggers.json or queue on the shared queue while the service uses the `.env`'s. PI_MAX_COST_USD too: it is
304
+ // the one other config value the worker's trigger loader (`loadSchedules`) accepts or refuses a file by (a trigger's
305
+ // `run.maxCostUsd` above it), so this command refuses exactly the files the worker refuses. `fleet` is the declared
306
+ // PI_WORKER_NAME, already here. Keep this list equal to the loader's config inputs.
307
+ const deployment = cliDeploymentEnv(env, TRIGGER_LOADER_ENV_KEYS);
308
+ if (deployment.problem) return fail(`${deployment.problem}. Nothing was queued.`);
309
+ const { triggersPath } = await import("./doctor.mjs");
310
+ const path = triggersPath(deployment.env, process.cwd());
311
+ if (path === "" || !existsSync(path)) return fail(`no triggers file at ${path === "" ? "(PI_TRIGGERS_FILE is empty)" : path}. Set PI_TRIGGERS_FILE or run this from the deployment folder. Nothing was queued.`);
312
+
313
+ const config = loadConfig(deployment.env);
314
+ const { loadSchedules } = await import("./schedules.mjs");
315
+ let schedule;
316
+ try {
317
+ // One read, validated whole by the shared loader (so a file the worker would refuse is refused here, with its
318
+ // message). The cron schedule is looked up FIRST: only when no cron trigger has this id is the raw file asked
319
+ // whether a webhook entry carries it, so a webhook entry spelling the same id never hides the cron trigger.
320
+ const text = readFileSync(path, "utf8");
321
+ schedule = loadSchedules({ ...config, triggersFile: path }, { readFileSync: () => text, fleet: config.workerNameDeclared }).find((s) => s.schedulerId === id);
322
+ if (!schedule) {
323
+ const named = JSON.parse(text).triggers.find((t) => t?.on?.id === id && t.on.type !== "cron");
324
+ if (named) return fail(`trigger "${id}" is a ${String(named.on.type)} trigger: only a cron trigger can be fired by hand. Nothing was queued.`);
325
+ }
326
+ } catch (error) {
327
+ return fail(`${error?.message ?? error}. Nothing was queued.`);
328
+ }
329
+ if (!schedule) return fail(`no cron trigger with id "${id}" in ${path}. Nothing was queued.`);
330
+ if (schedule.unserved) return fail(`cron trigger "${id}" runs on another host: its folder is not on this machine. Run this command on the host that has the folder. Nothing was queued.`);
331
+ // The worker's folder rule, `run <folder>`'s check (issue #524): `.git` at the folder and a commit at HEAD, or the job
332
+ // is refused at pickup. Uncommitted changes stay allowed, as they are for every scheduled tick.
333
+ const notARepo = localRepoProblem(schedule.data.folder);
334
+ if (notARepo) return fail(`cron trigger "${id}": ${notARepo} Nothing was queued.`);
335
+
336
+ const { cliValkeyUrl, parseConnection } = await import("./connection.mjs");
337
+ const valkeyUrl = cliValkeyUrl(env);
338
+ const { makeQueue, enqueueTriggerRunReporting, hostQueueName, swallowedRunSentence } = await import("./queue.mjs");
339
+ // The queue the worker of this host schedules this trigger on (start.mjs: `cronQueue`).
340
+ const hq = config.workerNameDeclared ? hostQueueName(config.workerName) : null;
341
+ const refused = await valkeyRefusal(valkeyUrl, env);
342
+ if (refused) return fail(refused);
343
+ const queue = makeQueue(parseConnection(valkeyUrl, { failFast: true }), { ...(hq ? { name: hq } : {}) });
344
+ try {
345
+ const { id: jobId, existing } = await enqueueTriggerRunReporting(queue, schedule, { now: now() });
346
+ const hint = existing ? null : await workerHint(queue);
347
+ write(runQueuedLine({ jobId, existing, trigger: id, hint }, { swallowedRunSentence }));
348
+ } catch (error) {
349
+ return fail(error?.valkeyRefused ? error.message : `could not reach Valkey at ${(await import("./connection.mjs")).urlShown(valkeyUrl)}: ${(await import("./valkey-auth.mjs")).valkeyDownHint(valkeyUrl)}\n ${error.message}`);
350
+ } finally {
351
+ await queue.close().catch(() => {});
352
+ }
353
+ return 0;
354
+ }
355
+
356
+ /**
357
+ * The env keys `run --trigger` resolves from the deployment: the file, the host, and every config value the worker's
358
+ * trigger loader (`loadSchedules`: `triggersFile`, `maxCostUsd`, and `fleet` from a declared worker name) judges a file by.
359
+ */
360
+ export const TRIGGER_LOADER_ENV_KEYS = Object.freeze(["PI_TRIGGERS_FILE", "PI_WORKER_NAME", "PI_MAX_COST_USD"]);
361
+
362
+ /**
363
+ * Deployment keys for a CLI verb that acts for the deployment (review of PR #575): `keys` resolved by issue #471's rule
364
+ * (`resolveServiceEnv`, service-env.mjs), the one doctor and `up` read the service's keys by. This shell's value where
365
+ * it sets the key, else the `.env` in `cwd` from a line the service's loader reads as written. Where the two disagree,
366
+ * or a line naming the key cannot be read as the loader reads it, or the file cannot be read and this shell sets none,
367
+ * the answer is unknown and the verb refuses (`problem`), as `up` refuses on a PI_JOB_IMAGE disagreement: a job queued
368
+ * on a queue no worker drains, or from a triggers file the worker does not read, is a silent no-op. Returns
369
+ * `{ env }` (this shell's with the file's values filled in) or `{ problem }`.
370
+ */
371
+ export function cliDeploymentEnv(env, keys, { cwd = process.cwd(), platform = process.platform, readFile = (p) => readFileSync(p) } = {}) {
372
+ const envPath = join(cwd, ".env");
373
+ let file;
374
+ try {
375
+ file = serviceEnvFileOf(readFile(envPath), envPath, serviceEnvLoader(platform));
376
+ } catch (error) {
377
+ const unset = keys.filter((k) => typeof env[k] !== "string");
378
+ if (error?.code !== "ENOENT" && unset.length > 0) return { problem: `${envPath} could not be read (${error?.code ?? "error"}), so ${unset.join(" and ")} cannot be told` };
379
+ return { env };
380
+ }
381
+ const read = resolveServiceEnv({ env, file, keys });
382
+ const [d] = read.disagreements;
383
+ if (d) return { problem: `${d.key} is ${JSON.stringify(d.shell)} in this shell and ${JSON.stringify(d.file)} in ${envPath}: make them agree (the service runs the file's)` };
384
+ const unread = [...read.unread.map((u) => u.key), ...read.hazardSkipped];
385
+ if (unread.length > 0) return { problem: `${envPath} has a line for ${unread.join(" and ")} that the service's loader may read differently, so it cannot be told (pi-dispatch doctor names the line)` };
386
+ return { env: read.env };
387
+ }
388
+
389
+ /** `workerHint`'s line when the queue cannot say, true whether or not a worker runs. */
390
+ export const WORKER_HINT_UNKNOWN = "a worker picks it up; start one with `pi-dispatch worker` if none is running.";
391
+
392
+ /**
393
+ * `run`'s closing line (issue #530): true with or without a running worker.
394
+ *
395
+ * It asks the queue the job went to, on the connection the enqueue already opened: `getWorkers()` (one CLIENT LIST,
396
+ * matched on the client names BullMQ gives a Worker of THIS queue, so a named host queue counts only that host's
397
+ * workers) and `isPaused()`. Measured on bullmq 5.80.4: 0 with no Worker, 1 with one, back to 0 once it closed, and
398
+ * the Queue's own client is never counted. A paused queue is said first, since a connected worker does not take a
399
+ * job from it. Any doubt falls back to a line that is true either way:
400
+ * - either call fails or takes over `timeoutMs` (a Valkey whose ACL refuses CLIENT LIST, one that went away);
401
+ * - an answer that is not what bullmq returns;
402
+ * - bullmq's own marker for a server with no CLIENT LIST, a fake row that would read as "1 worker".
403
+ * A server that ignores CLIENT SETNAME lists no worker at all, which is why zero says "shows as connected" and names
404
+ * the command rather than claiming none runs.
405
+ *
406
+ * CLIENT LIST spans every database, and bullmq matches its rows by client NAME alone, so a Worker of a same-named
407
+ * queue on database 8 was counted for a job queued on database 9 (measured in PR #531's review: "1 worker is
408
+ * connected ... will pick it up", and no worker ever would). Only rows whose `db` is the queue client's own count; a
409
+ * row with no `db` field, or a client whose database cannot be read, is a doubt.
410
+ */
411
+ export async function workerHint(queue, { timeoutMs = 2000 } = {}) {
412
+ let timer;
413
+ try {
414
+ const answer = Promise.all([queue.getWorkers(), queue.isPaused(), Promise.resolve(queue.client).then((c) => c?.options?.db ?? 0)]);
415
+ answer.catch(() => {});
416
+ const late = new Promise((resolve) => {
417
+ timer = setTimeout(resolve, timeoutMs, null);
418
+ });
419
+ const got = await Promise.race([answer, late]);
420
+ if (got === null) return WORKER_HINT_UNKNOWN;
421
+ const [workers, paused, db] = got;
422
+ if (!Array.isArray(workers) || typeof paused !== "boolean" || !Number.isInteger(Number(db))) return WORKER_HINT_UNKNOWN;
423
+ if (workers.some((w) => w?.name === "GCP does not support client list" || typeof w?.db !== "string")) return WORKER_HINT_UNKNOWN;
424
+ if (paused) return "the queue is paused: no worker takes it until `pi-dispatch resume`.";
425
+ const n = workers.filter((w) => w.db === String(db)).length;
426
+ if (n === 0) return "no worker shows as connected to this queue: start one with `pi-dispatch worker`.";
427
+ return `${n} ${n === 1 ? "worker is" : "workers are"} connected to this queue and will pick it up.`;
428
+ } catch {
429
+ return WORKER_HINT_UNKNOWN;
430
+ } finally {
431
+ clearTimeout(timer);
432
+ }
433
+ }
434
+
244
435
  /** One Valkey's pause, resume or status: every queue the deployment drains there (issue #57). Returns the exit code. */
245
436
  async function killSwitch(cmd, url, { env, write, label, urlShown, valkeyRefusal }) {
246
437
  const { parseConnection, makeRedisClient } = await import("./connection.mjs");
@@ -324,6 +515,24 @@ async function killSwitch(cmd, url, { env, write, label, urlShown, valkeyRefusal
324
515
  return 0;
325
516
  }
326
517
 
518
+ /**
519
+ * What `run` prints once the queue has answered (issue #524). Three answers, because the queue gives three.
520
+ *
521
+ * A local job's id is derived from the folder, the flow, the task, the model fields and the minute (`localJobId`), so the same `run`
522
+ * twice inside one minute is the same id and the queue keeps the first. That dedup is deliberate (a hasty second
523
+ * Enter must not pay twice) and stays; what changes is that it is said. The time is the first job's own, in this
524
+ * terminal's local time, and the state is the queue's word for it, or "already queued or done" when it could not be
525
+ * read. No flag is offered to force a second job, because none exists: a later minute is a different id.
526
+ * The sentence itself is `queue.mjs`'s, shared with the admin's `/dispatch run`, and handed in because this module
527
+ * imports the queue lazily. `hint` is `workerHint`'s closing line (issue #530); without one, the line true either way.
528
+ */
529
+ export function runQueuedLine({ jobId, existing, folder, trigger, hint = WORKER_HINT_UNKNOWN }, { swallowedRunSentence }) {
530
+ // `trigger` (issue #505) is `run --trigger`'s cron id, said in place of the folder.
531
+ if (existing) return `${swallowedRunSentence(jobId, existing)}\n${trigger !== undefined ? "the same trigger queues" : "the same folder and task queue"} a new run from the next minute on.\n`;
532
+ const unknown = existing === undefined ? "could not check whether an identical run from this minute already held this id.\n" : "";
533
+ return `queued ${jobId} for ${trigger !== undefined ? `cron trigger ${trigger}` : `folder ${folder}`}\n${unknown}${hint}\n`;
534
+ }
535
+
327
536
  function fail(message) {
328
537
  process.stderr.write(`error: ${message}\n`);
329
538
  return 1;
@@ -343,6 +552,8 @@ if (isEntryModule(import.meta.url)) {
343
552
  // A promise nobody handled is printed as its message alone (PR #475's review): Node's own print shows the whole
344
553
  // reason, which for a Valkey client's error could carry what it sent.
345
554
  installRejectionPrinter();
555
+ // A reader that closes early (`| head -1`) ends the verb quietly instead of with an uncaught EPIPE.
556
+ installStdoutPipeGuard();
346
557
  main()
347
558
  .then((code) => {
348
559
  if (code) process.exitCode = code;
package/src/config.mjs CHANGED
@@ -15,6 +15,20 @@ import { SWEEP_INTERVAL_HOURS, SWEEP_INTERVAL_MAX_HOURS } from "./retention-swee
15
15
  import { parseSecretProfiles } from "./secret-profiles.mjs";
16
16
  import { WAIT_AFTER_MAX_DEFAULT_MS, WAIT_INTERVAL_FLOOR_MS, parseWaitProfiles } from "./wait-for.mjs";
17
17
  import { imageRefProblem } from "./image-ref.mjs";
18
+ import { CONTAINER_ENV_NAMES, KEYLESS_ENV_NAME, RUNNER_ENV_NAMES } from "./reserved-env.mjs";
19
+ import { modelListProblem } from "./model-ref.mjs";
20
+ import { DOLLAR_ENV_NAMES, DOLLAR_WINDOW_KEYS, checkDollarInvariant, optionalUsdMicros } from "./money.mjs";
21
+
22
+ /**
23
+ * The VALKEY_URL the worker uses when none is set. ONE constant (issue #503's review): doctor judges the model endpoints
24
+ * file against this port too, and a doctor that read an unset VALKEY_URL as "no queue port" passed an endpoint on 6379
25
+ * that the worker refuses. valkey-endpoint.mjs re-exports it.
26
+ */
27
+ export const DEFAULT_VALKEY_URL = "redis://127.0.0.1:6379";
28
+ /** The model a job runs on when nothing names one (`PI_MODEL` unset): one copy, for `loadConfig` and doctor. */
29
+ export const DEFAULT_MODEL = "claude-sonnet-4-5-20250929";
30
+ /** The provider a job runs on when nothing names one (`PI_PROVIDER` unset). */
31
+ export const DEFAULT_PROVIDER = "anthropic";
18
32
 
19
33
  export function configError(message) {
20
34
  const error = new Error(message);
@@ -86,7 +100,7 @@ function optionalBoundedInt(env, name, min, max) {
86
100
  // Split a PATH-style list on the OS path delimiter (`;` on Windows, `:` elsewhere) so a Windows
87
101
  // drive-letter colon is not mistaken for a separator. Trims, drops empties. Entries are stored
88
102
  // verbatim; downstream (task 3.1) realpaths them, so no posix normalisation happens here.
89
- function delimitedList(raw) {
103
+ export function delimitedList(raw) {
90
104
  return (raw ?? "")
91
105
  .split(delimiter)
92
106
  .map((s) => s.trim())
@@ -165,6 +179,28 @@ function forwardEnvList(raw, egressArmed = false) {
165
179
  `PI_FORWARD_ENV must not forward ${workerOnly.join(", ")} -- ${workerOnly.map((n) => WORKER_ONLY_WHY[n]).join("; ")}, and a job container is the last place it belongs (CONST-TOKEN-SCOPED-PER-JOB)`,
166
180
  );
167
181
  }
182
+ // Issue #503: the keyless marker is the worker's to set, and only for a job whose provider is served by keyless
183
+ // endpoints alone. Forwarded, it would make any provider whose models.json key is `$PI_DISPATCH_KEYLESS` look
184
+ // configured, so it is refused whatever the egress policy.
185
+ if (names.includes(KEYLESS_ENV_NAME)) {
186
+ throw configError(`PI_FORWARD_ENV must not forward ${KEYLESS_ENV_NAME} -- the worker sets it itself, only for a job whose provider has every model on a keyless model endpoint (model-endpoints.json)`);
187
+ }
188
+ // Issue #502 (PR #536's review): every name the worker writes into a job's container itself. The forward loop runs
189
+ // AFTER that write, so a forwarded host value REPLACES the per-job one: `PI_MODEL` or `PI_PROVIDER` would run a model
190
+ // the pre-spend gates never checked, `PI_ALLOWED_MODELS` would swap a trigger's narrower list for the deployment's,
191
+ // `PI_MAX_TURNS` would lift the turn limit. One rule over the whole closed set rather than a list of the ones that
192
+ // happen to matter today. HOME is the one exception, and an old one: forwarding it is supported, the job-user path
193
+ // overrides it beside `--user` and says so at boot (`forward_env_home_overridden`).
194
+ const owned = names.filter((n) => CONTAINER_ENV_NAMES.has(n) && n !== "HOME" && n !== KEYLESS_ENV_NAME);
195
+ if (owned.length > 0) {
196
+ throw configError(`PI_FORWARD_ENV must not forward ${owned.join(", ")} -- the worker writes ${owned.length === 1 ? "it" : "them"} into every job's container itself, and a forwarded host value would replace the per-job one (a forwarded PI_MODEL runs a model the pre-spend checks never saw)`);
197
+ }
198
+ // Issue #500: the two names the runner sets in its own environment for its descendants (RUNNER_ENV_NAMES). A forwarded
199
+ // host value would sit in the runner's environment before the runner sets its own.
200
+ const runnerOwned = names.filter((n) => RUNNER_ENV_NAMES.has(n));
201
+ if (runnerOwned.length > 0) {
202
+ throw configError(`PI_FORWARD_ENV must not forward ${runnerOwned.join(", ")} -- the job's runner sets ${runnerOwned.length === 1 ? "it" : "them"} inside the container for its own child processes, and a forwarded value would point their usage ledger at a directory nobody reads`);
203
+ }
168
204
  const egress = egressArmed ? names.filter((n) => EGRESS_ENV_VARS.has(n)) : [];
169
205
  if (egress.length > 0) {
170
206
  throw configError(
@@ -174,6 +210,32 @@ function forwardEnvList(raw, egressArmed = false) {
174
210
  return names;
175
211
  }
176
212
 
213
+ /**
214
+ * PI_ALLOWED_MODELS (issue #502): the deployment's allowed-model list, comma-separated `provider/model`, the
215
+ * grammar a trigger's `run.models` has (`modelListProblem`, model-ref.mjs), so a list means the same thing in
216
+ * either place. A job's effective list is its trigger's, else this one, else none (unrestricted).
217
+ *
218
+ * ENV ONLY, and never a settings-overlay key, for the reason `run.image` resolves against env only: the overlay is
219
+ * writable by a model-callable tool (`dispatch_set`), and a list a tool could widen is not a policy. Refused at boot
220
+ * when malformed, like every other knob here. Unset or empty is unrestricted: an empty env var is how an `.env`
221
+ * template says "not set", unlike an empty `run.models`, which a reviewed file only holds by mistake.
222
+ *
223
+ * NO WHITESPACE anywhere in the value (PR #536's lab review). The launchd and other non-systemd wrappers SOURCE the
224
+ * `.env` with a shell, which reads `PI_ALLOWED_MODELS=a/b, c/d` as a one-off assignment followed by a command named
225
+ * `c/d`: the variable is never set and every job runs unrestricted, with nothing failing. A worker that does see the
226
+ * spaced value (systemd reads it whole) therefore refuses it rather than trimming it, so the same line cannot mean a
227
+ * list on one host and no list on another. An empty segment between two commas is refused too, never skipped.
228
+ */
229
+ export function allowedModelsFrom(env) {
230
+ const raw = env.PI_ALLOWED_MODELS;
231
+ if (raw === undefined || raw === "") return null;
232
+ if (/\s/.test(raw)) throw configError("invalid PI_ALLOWED_MODELS: write the list with no spaces (a/b,c/d) -- a shell that sources .env cuts an unquoted value at the first space and leaves the variable unset, so jobs would run with no list");
233
+ const list = raw.split(",");
234
+ const problem = modelListProblem(list);
235
+ if (problem !== null) throw configError(`invalid PI_ALLOWED_MODELS: the list ${problem}`);
236
+ return list;
237
+ }
238
+
177
239
  /**
178
240
  * PI_EGRESS (REQ-EGRESS-ALLOWLIST). The parse itself lives in egress.mjs, because `doctor` and `up` read
179
241
  * the environment directly and three copies of one default is two chances to flip it in the wrong number
@@ -244,10 +306,15 @@ export function jobImageFrom(env) {
244
306
 
245
307
  // The operator's global pi overlay dir (REQ-GLOBAL-PI-OVERLAY). Unset/empty = feature off. When set it
246
308
  // must EXIST at boot -- a typo pointing at nothing would silently drop the operator's whole setup on
247
- // every job, so fail loud like every other config error rather than degrade to nothing.
309
+ // every job, so fail loud like every other config error rather than degrade to nothing. And it must be
310
+ // ABSOLUTE (PR #553's review): the value becomes the source of the job's `-v <dir>:/opt/pi-global:ro`, and a
311
+ // relative one is resolved differently by the worker and the container runtime (the worker against its own
312
+ // working directory; the runtime against the client's, or a bare name as a named volume or an error), so the
313
+ // job could mount another folder, or none, than the one the worker judged. Every relative value is refused.
248
314
  function resolveGlobalPiDir(env, fileExists) {
249
315
  const dir = env.PI_GLOBAL_PI_DIR;
250
316
  if (dir === undefined || dir === "") return null;
317
+ if (!isAbsolute(dir)) throw configError(`PI_GLOBAL_PI_DIR must be an absolute path (got ${JSON.stringify(dir)}): a relative value is resolved differently by the worker and the container runtime, so a job could mount another folder than the one the worker checks`);
251
318
  if (!fileExists(dir)) throw configError(`PI_GLOBAL_PI_DIR does not exist: ${dir}`);
252
319
  return dir;
253
320
  }
@@ -294,14 +361,14 @@ export function globalExtensionsEnabled(env) {
294
361
  * spend caps above, the per-job token scoping, or the admin-extension recursion block.
295
362
  */
296
363
  export function loadConfig(env = process.env, { fileExists = existsSync } = {}) {
297
- const model = env.PI_MODEL ?? "claude-sonnet-4-5-20250929"; // dated snapshot; deterministic per CONST-PI-VERSION-PINNED
364
+ const model = env.PI_MODEL ?? DEFAULT_MODEL; // dated snapshot; deterministic per CONST-PI-VERSION-PINNED
298
365
  // #227. Hoisted above the object because `defaultBackend` INDEXES `backends`, and a property cannot read
299
366
  // a sibling of the literal it is in. (Calling the parser twice would be harmless -- `egressEnabled` is
300
367
  // called twice a few properties down for the same reason -- so this is about the index, not the throw.)
301
368
  const backends = backendSet(env);
302
369
  const backendFloor = backendFloorOf(env);
303
370
  const config = {
304
- valkeyUrl: env.VALKEY_URL ?? "redis://127.0.0.1:6379",
371
+ valkeyUrl: env.VALKEY_URL ?? DEFAULT_VALKEY_URL,
305
372
  // Issue #57. What this machine calls itself: the key of its registry row, the `host` on every log
306
373
  // line and run record, and the BullMQ worker name. Always populated -- a deployment that declares
307
374
  // nothing still has an identity, which is what lets a fleet of two be TOLD APART before anyone has
@@ -315,11 +382,20 @@ export function loadConfig(env = process.env, { fileExists = existsSync } = {})
315
382
  weeklyCap: optionalBoundedInt(env, "PI_WEEKLY_CAP", 1), // REQ-SPEND-CAPS-MULTI-WINDOW; null = weekly window disabled
316
383
  monthlyCap: optionalBoundedInt(env, "PI_MONTHLY_CAP", 1), // null = monthly window disabled
317
384
  softHoldPct: optionalBoundedInt(env, "PI_SOFT_HOLD_PCT", 1, 99), // null = soft-hold band disabled
318
- provider: env.PI_PROVIDER ?? "anthropic",
385
+ provider: env.PI_PROVIDER ?? DEFAULT_PROVIDER,
319
386
  model,
320
387
  maxTurns: positiveInt(env, "PI_MAX_TURNS", 30), // pi has no turn limit; we impose one
388
+ allowedModels: allowedModelsFrom(env), // issue #502: the deployment's allowed-model list; null = unrestricted. ENV ONLY, never the settings overlay
321
389
  maxTokens: optionalBoundedInt(env, "PI_MAX_TOKENS", 1), // issue #25; null = per-job token budget disabled (lagging in-run backstop)
322
390
  dailyTokenCap: optionalBoundedInt(env, "PI_DAILY_TOKEN_CAP", 1), // issue #25; null = daily token counter disabled (check-AFTER, host-side)
391
+ // Issue #501. The dollar settings, each an operator's decimal (`"2.50"`) kept AS WRITTEN once it parses, so
392
+ // the overlay and env carry one kind of value and `effectiveJobOf` converts both to micro-dollars the same
393
+ // way. Unset or empty is null: no per-job cap, no window. The cross-key rule is below, after the object is
394
+ // built.
395
+ maxCostUsd: usdSetting(env, "PI_MAX_COST_USD"),
396
+ dailyCostUsd: usdSetting(env, "PI_DAILY_COST_USD"),
397
+ weeklyCostUsd: usdSetting(env, "PI_WEEKLY_COST_USD"),
398
+ monthlyCostUsd: usdSetting(env, "PI_MONTHLY_COST_USD"),
323
399
  jobImage: jobImageFrom(env), // || (not ??) so an empty string falls back; "" is falsy and would throw inside buildDockerRunArgs AFTER a budget slot was reserved
324
400
  globalPiDir: resolveGlobalPiDir(env, fileExists), // REQ-GLOBAL-PI-OVERLAY: operator's ~/.pi/agent subset, :ro-mounted; null = off
325
401
  allowGlobalExtensions: globalExtensionsEnabled(env), // REQ-GLOBAL-PI-OVERLAY: ON unless PI_GLOBAL_ALLOW_EXTENSIONS=0
@@ -352,6 +428,9 @@ export function loadConfig(env = process.env, { fileExists = existsSync } = {})
352
428
  sandboxIdleMinutes: nonNegativeInt(env, "PI_SANDBOX_IDLE_MINUTES", 30), // bash's own TMOUT inside a sandbox; 0 = no idle logout
353
429
  triggersFile: env.PI_TRIGGERS_FILE ?? null, // DES-CRON-VIA-BULLMQ-SCHEDULER: unified triggers file; null = cron disabled for the worker (it selects on.type:"cron")
354
430
  pauseWindowsFile: pauseWindowsFilePath(env), // REQ-SCOPED-PAUSE-WINDOWS: per-folder/repo timed pause; null = no scoped pauses
431
+ modelEndpointsFile: modelEndpointsFilePath(env), // issue #503: the declared model endpoints (INT-MODEL-ENDPOINTS-FILE-CONTRACT); null = model-endpoints.json in the deployment folder, and a missing default file declares none
432
+ envelopeFile: envelopeFilePath(env), // issue #504: the allocation envelope (INT-ENVELOPE-FILE-CONTRACT); null = no envelope and no delegation anywhere
433
+ projectsFile: projectsFilePath(env), // issue #499: named groups of repos and folders (INT-PROJECTS-FILE-CONTRACT); null = no projects, and every record's project is null
355
434
  scopedLimitsFile: scopedLimitsFilePath(env), // issue #242: per-scope run caps + concurrency (INT-SCOPED-LIMITS-FILE-CONTRACT); null = none. The one-job-per-folder mutex for local jobs is code, not configuration, and holds regardless
356
435
  schedulerStallMax: positiveInt(env, "PI_SCHEDULER_STALL_MAX", 2), // CONST-RETRY-INFRA-ONLY: per-scheduler stall backstop; positiveInt rejects <1 so a 0 threshold fails closed
357
436
  logsDir: logsDirPath(env), // || (not ??) inside logsDirPath, so an empty string falls back to the default
@@ -464,10 +543,34 @@ export function loadConfig(env = process.env, { fileExists = existsSync } = {})
464
543
  // #227, and it runs AFTER the object is built rather than inside it: the refusal reads `egress` as well
465
544
  // as the two backend fields, and a check woven between properties would depend on key order.
466
545
  refuseBackendShortfall(config);
546
+ // Issue #501, after the object for the same reason: both rules read more than one key.
547
+ refuseDollarSettings(config);
467
548
 
468
549
  return config;
469
550
  }
470
551
 
552
+ // An optional dollar amount from env (issue #501): unset or empty is null, anything else must parse as money
553
+ // (`parseUsdMicros`) and is kept as the string the operator wrote. The refusal names the variable, never the
554
+ // value, which is `parseUsdMicros`' own rule.
555
+ function usdSetting(env, name) {
556
+ const raw = env[name];
557
+ if (raw === undefined || raw === "") return null;
558
+ optionalUsdMicros(raw, name);
559
+ return raw;
560
+ }
561
+
562
+ // The env half of the dollar rule (issue #501). The overlay half runs on MERGED values per job (start.mjs);
563
+ // this one runs on env alone at boot, where an operator is present to read the refusal: a window without a
564
+ // per-job cap (`checkDollarInvariant`), because a window reserves each job's per-job cap and without one there
565
+ // is no amount to reserve. The windows themselves are enforced (dollar-budget.mjs, processor.mjs).
566
+ function refuseDollarSettings(config) {
567
+ const broken = checkDollarInvariant(config);
568
+ if (broken) {
569
+ const window = DOLLAR_WINDOW_KEYS.find((key) => config[key] !== null);
570
+ throw configError(`${DOLLAR_ENV_NAMES[window]} needs PI_MAX_COST_USD: a dollar window reserves each job's per-job cost cap before it starts, so it cannot be set without one`);
571
+ }
572
+ }
573
+
471
574
  /**
472
575
  * Normalise an inline App private key, or refuse it. Returns `null` for absent/blank (an empty
473
576
  * `GITHUB_APP_PRIVATE_KEY=` line in a scaffolded .env means "unset", never "a key that is empty").
@@ -863,6 +966,24 @@ export function scopedLimitsFilePath(env = process.env) {
863
966
  return env.PI_SCOPED_LIMITS_FILE ?? null;
864
967
  }
865
968
 
969
+ /** Issue #499. `??` like the two above: an empty value is a value, so the boot load refuses it rather than reading it
970
+ * as "no projects". */
971
+ export function projectsFilePath(env = process.env) {
972
+ return env.PI_PROJECTS_FILE ?? null;
973
+ }
974
+
975
+ /** Issue #504. `??` like the projects key: an empty value is a value, so the boot load refuses it rather than reading
976
+ * it as "no envelope". Unset turns delegation off everywhere. */
977
+ export function envelopeFilePath(env = process.env) {
978
+ return env.PI_ENVELOPE_FILE ?? null;
979
+ }
980
+
981
+ /** Issue #503. `??` like the two above, so an empty value is a value, which the loader refuses rather than reading the
982
+ * default file in its place. Unlike them, null does not turn anything off: it means the deployment folder's file. */
983
+ export function modelEndpointsFilePath(env = process.env) {
984
+ return env.PI_MODEL_ENDPOINTS_FILE ?? null;
985
+ }
986
+
866
987
  export function logsDirPath(env = process.env, home) {
867
988
  return env.PI_LOGS_DIR || defaultLogsDir(env, home);
868
989
  }
@@ -141,6 +141,9 @@ export function parsePodmanInfo(stdout) {
141
141
  const controllers = Array.isArray(host.cgroupControllers) ? host.cgroupControllers.filter((c) => typeof c === "string" && /^[a-z][a-z0-9_]{0,31}$/.test(c)) : null;
142
142
  return {
143
143
  rootless: bool(host.security?.rootless),
144
+ // Issue #503: the rootless network helper Podman 5 names (`host.rootlessNetworkCmd`, measured `pasta` on 5.8.1;
145
+ // 4.9.3 has no such field). One of the two known words, lowercased, else no fact.
146
+ rootlessNetworkCmd: typeof host.rootlessNetworkCmd === "string" && /^(?:pasta|slirp4netns)$/i.test(host.rootlessNetworkCmd) ? host.rootlessNetworkCmd.toLowerCase() : null,
144
147
  serviceIsRemote: bool(host.serviceIsRemote),
145
148
  selinux: bool(host.security?.selinuxEnabled),
146
149
  // "v2" or "v1"; anything else is no fact rather than a string an operator's terminal is handed.
@@ -14,6 +14,7 @@ const COMMAND_SAYS = Object.freeze({
14
14
  up: "up would drive the shell's venue",
15
15
  init: "init's next steps would be for the shell's venue",
16
16
  doctor: "doctor would judge the shell's venue",
17
+ egress: "the render would name the shell's venue's reload",
17
18
  });
18
19
 
19
20
  /**