@edgehero/pi-dispatch 5.1.0 → 5.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -49,6 +49,11 @@ const API_ROOT = "https://api.github.com";
49
49
  const USER_AGENT = "pi-dispatch-setup-github";
50
50
  // The repo homepage, shown on the App's about page — informational, not a trust anchor.
51
51
  const HOMEPAGE = "https://github.com/edgehero/pi-dispatch";
52
+ // The webhook URL of an App made with --no-webhook (issue #874). GitHub requires one even for an inactive hook. A
53
+ // `.invalid` name (RFC 6761) never resolves, so nothing GitHub sends there reaches any host: not the ping an inactive
54
+ // hook still got at creation (in the measurement that `buildManifest`'s comment cites), and not an event if someone
55
+ // ticks Active later without changing the URL. A real host would get both, and a domain can change owner.
56
+ export const NO_WEBHOOK_URL = "https://no-webhook.invalid/";
52
57
  // GitHub app names are GLOBALLY unique and capped at 34 characters.
53
58
  const NAME_MAX = 34;
54
59
 
@@ -363,7 +368,11 @@ function printSummary(say, summary, { noWebhook }) {
363
368
  say("\nNext:\n");
364
369
  if (noWebhook) {
365
370
  say(" the App was created with its webhook INACTIVE (--no-webhook) — the polling-ready shape.\n");
366
- say(" Polling delivery is a follow-up; until it lands, forge triggers for this App will not fire on their own.\n");
371
+ say(" start the poller so events reach the queue: `pi-dispatch-receiver poll`. Under GITHUB_AUTH_SOURCE=app it\n");
372
+ say(" watches the repositories of the installation GITHUB_APP_INSTALLATION_ID names; POLL_REPOS names a list of\n");
373
+ say(" your own instead. See docs/polling.md.\n");
374
+ say(` the App's webhook URL is a placeholder, ${NO_WEBHOOK_URL}, which nothing can receive. Moving to webhooks\n`);
375
+ say(" later takes a webhook secret and an order of steps: docs/polling.md, \"Moving to webhooks later\".\n");
367
376
  } else {
368
377
  say(" start the receiver so deliveries have somewhere to land: `pi-dispatch-receiver`\n");
369
378
  say(" (or, where the folder holds deploy/docker-compose.yml, its receiver profile), reachable at the --webhook-url you gave GitHub.\n");
@@ -402,8 +411,12 @@ export function sanitizeAppName(raw) {
402
411
  * The manifest GitHub converts into the App — the narrowest shape this system needs
403
412
  * (SECURITY.md's auth-source ladder): write on exactly the three surfaces jobs touch, metadata read
404
413
  * because the API requires it, private, and only the event families the triggers file can name.
405
- * `hook_attributes` carries the receiver URL, or `active: false` under --no-webhook — the
406
- * no-public-URL shape the polling transport will consume.
414
+ * `hook_attributes` carries the receiver URL, or under --no-webhook an INACTIVE hook at `NO_WEBHOOK_URL`. GitHub
415
+ * refuses a manifest whose `hook_attributes` has no `url`, inactive or not ("url" wasn't supplied, issue #874), and one
416
+ * that leaves `hook_attributes` out while it subscribes events ("Hook url cannot be blank"). What it accepts was
417
+ * measured against github.com on 2026-11-09 by another project's throwaway Apps: a `.invalid` URL with
418
+ * `active: false` was accepted, only `localhost` and `127.0.0.1` were refused as not reachable over the public
419
+ * Internet, and the App with the inactive hook recorded one delivery, its creation's ping, and no event after it.
407
420
  *
408
421
  * `pull_request_review` (issue #66) is subscribed for every new App, armed trigger or not. That is the
409
422
  * posture `pull_request` already has for a label-only deployment: an unsubscribed event cannot be turned
@@ -415,7 +428,7 @@ export function buildManifest({ name, redirectUrl, webhookUrl }) {
415
428
  name,
416
429
  url: HOMEPAGE,
417
430
  redirect_url: redirectUrl,
418
- hook_attributes: webhookUrl ? { url: webhookUrl } : { active: false },
431
+ hook_attributes: webhookUrl ? { url: webhookUrl } : { url: NO_WEBHOOK_URL, active: false },
419
432
  public: false,
420
433
  default_permissions: { contents: "write", pull_requests: "write", issues: "write", metadata: "read" },
421
434
  default_events: ["issues", "issue_comment", "pull_request", "pull_request_review"],
@@ -14,9 +14,11 @@
14
14
  * charset, on the way in AND on the way out, since another host wrote what a reader parses. An overlay extension's first
15
15
  * path segment and whether it is a file or a folder; a skill's folder name; a server's name, transport, where it is
16
16
  * (`mcpServerWhere`: a url's host and port, or a command's base name, never a header, an argument or a value), exposure,
17
- * whether it is on, and the variable NAMES it references; a package's name and version; a trigger's position, flow name and
18
- * the tier it resolved in. No path, no value and no tool name: what a server or an extension offers is known only once it
19
- * runs, and the run's own record says that. These are the same facts the viewer's Connections page shows; a model's
17
+ * whether it is on, and the variable NAMES it references; a package's name and version; a trigger's position, flow name,
18
+ * the tier it resolved in and a short hex digest of the trigger fields that answer was judged from (`flowBasisOf`). No path,
19
+ * no value and no tool name: what a server or an extension offers is known only once it runs, and the run's own record says
20
+ * that. The digest holds a path (`run.folder`, `run.skillsDir`) only as a hash cut to 8 hex digits, which names nothing a
21
+ * reader can read back. These are the same facts the viewer's Connections page shows; a model's
20
22
  * projection (`modelConnections`) carries counts in place of `where` and the key names, and one of these rows gets the
21
23
  * same treatment when a tool reads it.
22
24
  *
@@ -27,6 +29,7 @@
27
29
  * folders on the host and runs nothing: a frontmatter `name:` rename is invisible, and what a run really loaded is its
28
30
  * own record's `loaded`. Where a read cannot say, it says so (`unread`), never an empty list that reads as none.
29
31
  */
32
+ import { createHash } from "node:crypto";
30
33
  import * as nodeFs from "node:fs";
31
34
  import { join, resolve, sep } from "node:path";
32
35
  import { DOCTOR_HOSTS, DOCTOR_HOSTS_MAX, DOCTOR_RESULTS_TTL_MS } from "./doctor-results.mjs";
@@ -73,6 +76,52 @@ const KEY_NAME_RE = /^[A-Z_][A-Z0-9_]{0,63}$/;
73
76
  const WHERE_RE = /^[A-Za-z0-9._:[\]-]{1,255}$/;
74
77
  const INDEX_MAX = 1_000_000;
75
78
 
79
+ /**
80
+ * The trigger fields `flowTierOf` reads, by the names its first argument gives them (issue #869). A flow row's `basis` is a
81
+ * digest of exactly these, so a test holds this list to that argument's own names: a field `flowTierOf` came to read that
82
+ * this list left out would let an edit to it keep the console answering for the file as it was.
83
+ */
84
+ export const FLOW_INPUT_FIELDS = Object.freeze(["flow", "folder", "skillsDir", "packages", "excludeExtensions"]);
85
+ /**
86
+ * How long a basis is, in hex digits: 32 bits. A collision costs one stale answer read as current, which is what every edit
87
+ * cost before the basis, and the triggers file has no author but the operator, so nobody gains by aiming for one. Per edit the
88
+ * odds are one in about four billion. REJECTED: a longer digest, which a flow row carries 128 times over in a record that already
89
+ * trims its flow rows at its worst (`FIT_ORDER`), so each digit would put a trigger's answer out of the record on such a host.
90
+ */
91
+ export const FLOW_BASIS_HEX = 8;
92
+ const FLOW_BASIS_RE = new RegExp(`^[0-9a-f]{${FLOW_BASIS_HEX}}$`);
93
+
94
+ /**
95
+ * What `flowTierOf` is asked of a trigger as the loader parsed it (`parseTriggers`' entry): doctor's flow lines and its published
96
+ * rows ask it this, and nothing else does. A field that is not there reads as `flowTierOf`'s own default. A disarmed one-shot is
97
+ * the loader's sentinel, whose `run` is empty, so it names no flow and doctor publishes no row for it.
98
+ */
99
+ export function flowInputOf(trigger) {
100
+ const run = isObject(trigger?.run) ? trigger.run : {};
101
+ return {
102
+ flow: typeof run.flow === "string" ? run.flow : null,
103
+ folder: typeof run.folder === "string" ? run.folder : null,
104
+ skillsDir: typeof run.skillsDir === "string" ? run.skillsDir : null,
105
+ packages: run.packages !== false,
106
+ excludeExtensions: Array.isArray(run.excludeExtensions) ? run.excludeExtensions.filter((name) => typeof name === "string") : [],
107
+ };
108
+ }
109
+
110
+ /**
111
+ * The basis of a flow row (issue #869): a digest of what `flowTierOf` was asked of this trigger, from the trigger as the loader
112
+ * parsed it. Doctor writes it beside each answer, and the console computes it from its own parse of the file it reads, through
113
+ * this one function, so the two cannot compute it two ways. Where they differ, the answer was judged for another version of the file.
114
+ * `flowTierOf` only asks whether a name is in `excludeExtensions`, so the list's order and a repeat change nothing here either.
115
+ * REJECTED: leaving the list out under `"packages": false`, where `flowTierOf` does not look at it. An edit there would then keep
116
+ * the answer, at the price of a copy of `flowTierOf`'s branches here that would have to move with them, and the cost of not doing
117
+ * it is an answer read as unknown until doctor runs on the file as it is, the safe direction.
118
+ */
119
+ export function flowBasisOf(trigger) {
120
+ const input = flowInputOf(trigger);
121
+ const canonical = FLOW_INPUT_FIELDS.map((field) => (field === "excludeExtensions" ? [...new Set(input[field])].sort() : input[field]));
122
+ return createHash("sha256").update(JSON.stringify(canonical)).digest("hex").slice(0, FLOW_BASIS_HEX);
123
+ }
124
+
76
125
  /** A first path segment: the shape a run's record holds, less the two a folder listing never returns and a path could. */
77
126
  const extensionOf = (row) => (isObject(row) && typeof row.name === "string" && EXTENSION_NAME_RE.test(row.name) && row.name !== "." && row.name !== ".." && LOADS_EXTENSION_KINDS.includes(row.kind) ? { name: row.name, kind: row.kind } : null);
78
127
  const skillOf = (name) => (typeof name === "string" && SKILL_NAME_RE.test(name) ? name : null);
@@ -83,7 +132,10 @@ const flowOf = (row) => {
83
132
  if (flow === null && row.flow !== null) return null;
84
133
  const pkg = row.package === null ? null : isPackageName(row.package) ? row.package : undefined;
85
134
  if (pkg === undefined || (pkg !== null && row.tier !== "package")) return null;
86
- return { trigger: row.trigger, flow, tier: row.tier, package: pkg, repo: row.repo };
135
+ // Absent is a row from a release before issue #869, which a reader answers from as it did. Any other shape is refused, never
136
+ // read as absent: that would put a row of unknown basis back on the older rule, which answers whatever the file now says.
137
+ if (row.basis !== undefined && (typeof row.basis !== "string" || !FLOW_BASIS_RE.test(row.basis))) return null;
138
+ return { trigger: row.trigger, flow, tier: row.tier, package: pkg, repo: row.repo, ...(row.basis === undefined ? {} : { basis: row.basis }) };
87
139
  };
88
140
  /** One server's row, or null, with how many of its keys its cap left out. */
89
141
  const serverOf = (row) => {
@@ -346,6 +398,8 @@ export function hostsAgree(rows) {
346
398
  * resolved, checked, unknown the words of doctor's own line, which name folders and so are never published
347
399
  *
348
400
  * `repoFlowAtHead(folder, flow)` answers `present`, `absent` or anything else for not known. `staged` is `readStagedSkills`.
401
+ * The first argument is `flowInputOf`'s, and its names are `FLOW_INPUT_FIELDS`: a new one goes in both, or a flow row's basis
402
+ * would not change when it does (`worker/test/host-loads.test.mjs` reads them off this signature).
349
403
  */
350
404
  export async function flowTierOf({ flow, folder = null, skillsDir = null, packages = true, excludeExtensions = [] }, { repoFlowAtHead, fileExists, overlay = null, staged = { skills: [], unenumerable: [] } }) {
351
405
  let tier = null;
package/src/job-size.mjs CHANGED
@@ -106,6 +106,33 @@ export function codemodeWorkersOf(memMiB) {
106
106
  return Math.max(1, Math.floor(memMiB / (2 * CODEMODE_WORKER_HEAP_MIB)));
107
107
  }
108
108
 
109
+ /**
110
+ * Issue #865: a size's CPUs as a whole count, rounded UP (`1.5` is 2, `0.25` is 1), for a tool's `-j` or worker flag,
111
+ * which takes no fraction. Up, not down, so no size reads as 0 and a size between two counts gets the larger one: it is
112
+ * a pool size under a CPU WEIGHT, and the job may already use idle cores beyond its size (the `--cpus` ceiling above).
113
+ */
114
+ export function wholeCpusOf(cpuCenti) {
115
+ return Math.ceil(cpuCenti / 100);
116
+ }
117
+
118
+ /**
119
+ * Issue #865: the job's size as its own processes can read it, `{ PI_JOB_SIZE_CPUS, PI_JOB_SIZE_MEMORY_MIB }`, both
120
+ * decimal strings. A job's CPU size is a weight that nothing inside the container can see: `os.availableParallelism()`
121
+ * reads the cgroup's `cpu.max`, which is the host ceiling (`--cpus`, the same for every job), and `nproc` reads the
122
+ * affinity mask, which is the whole runtime. So a tool that starts one worker per CPU it sees (jest and vitest by
123
+ * default) starts one per CEILING CPU, all inside the size's memory, which is the hard bound. These two values are what
124
+ * a flow, an image or the agent's commands size such a pool from instead.
125
+ *
126
+ * INFORMATION, NOT A BOUND: the size is resolved at pickup and enforced by the container's flags; nothing reads these
127
+ * back. One function, so the job's closed env map (`env-allowlist.mjs`) and the operator's sandbox shell
128
+ * (`sandbox.mjs`) cannot spell them two ways. Not `PI_JOB_CPUS` and `PI_JOB_MEMORY`: those name the DEPLOYMENT's default
129
+ * size in `.env`, and a project row can give a job another one.
130
+ */
131
+ export function jobSizeEnv(size) {
132
+ assertJobSize(size);
133
+ return { PI_JOB_SIZE_CPUS: String(wholeCpusOf(size.cpuCenti)), PI_JOB_SIZE_MEMORY_MIB: String(size.memMiB) };
134
+ }
135
+
109
136
  /** The scope prefix of a project row, as `scoped-limits.mjs` spells it (`PROJECT_SCOPE_PREFIX`; a test holds them equal). */
110
137
  const PROJECT_ROW_PREFIX = "project:";
111
138
 
@@ -22,7 +22,8 @@
22
22
  * FIRST, IT RUNS ONLY ON TEXT `JSON.parse` HAS ALREADY ACCEPTED. Every caller parses first and scans
23
23
  * second. So the scanner never has to decide whether malformed input is malformed, never has to recover
24
24
  * from anything, and every branch it does not have is a branch that cannot be wrong. Feeding it text that
25
- * did not parse is a caller error, not an input case: the result is unspecified rather than a refusal.
25
+ * did not parse is a caller error, not an input case: the result is unspecified rather than a refusal, and
26
+ * since issue #868 it can be a `SyntaxError`, thrown when a key with an escape is decoded.
26
27
  *
27
28
  * SECOND, IT IS CHECKED AGAINST A GENERATOR THAT KNOWS THE ANSWER. `worker/test/json-duplicates.test.mjs`
28
29
  * builds thousands of documents with adversarial key characters -- quotes, backslashes, newlines, braces,
@@ -33,14 +34,19 @@
33
34
  * a key written plainly and the same key written with a `u00`-style escape are ONE key to `JSON.parse`,
34
35
  * and a byte comparison sees two.
35
36
  *
37
+ * SINCE ISSUE #868 NO KEY IS DECODED BY HAND. A key with no backslash is a slice of the text, and a key
38
+ * with one is handed to `JSON.parse` as its own string literal, so the decoder is the same code whose
39
+ * reading of the file the check defends. The hand-written decoder before it appended one character at a
40
+ * time, which took about 30 bytes of heap per character of a key: a 4MB key cost 123MB, and a 16MB key
41
+ * ended a process with a 256MB heap while it read its own configuration. That decoder agreed with
42
+ * `JSON.parse` on every escape, so no verdict moves. What is still written by hand about a string is
43
+ * where it ends, the one rule the value skip has always relied on, and the generator above checks it.
44
+ *
36
45
  * IMPORT-FREE, like `reserved-env.mjs` and `provider-key.mjs` beside it: `triggers.mjs` is the shared
37
46
  * validator, the receiver loads it and `admin/build.mjs` inlines it into the published console, so
38
47
  * anything it reaches has to be as cheap as a list of strings.
39
48
  */
40
49
 
41
- /** The escape characters JSON defines, other than `\u`. */
42
- const SIMPLE_ESCAPES = { '"': '"', "\\": "\\", "/": "/", b: "\b", f: "\f", n: "\n", r: "\r", t: "\t" };
43
-
44
50
  /**
45
51
  * The first duplicate key in `text`, as `{ key, at }`, or `null` when there is none.
46
52
  *
@@ -73,54 +79,53 @@ export function findDuplicateKey(text) {
73
79
  };
74
80
 
75
81
  /**
76
- * Walk past the string starting at `s[i] === '"'` WITHOUT building it.
82
+ * Walk past the string starting at `s[i] === '"'` WITHOUT building it, and say whether it held an escape.
83
+ *
84
+ * Building a string here one character at a time (`out += ch`) costs about 30 bytes of heap per source
85
+ * character, because each append is a new rope node that lives until the string is flattened. A value's
86
+ * text is discarded, so it is never built: a single 64MB value cost 2GB of garbage that way, and a
87
+ * trigger's `run.task` and `run.instructions` are free text. A key is taken from the text by `readKey`
88
+ * with the end this finds (issue #868). A worker that dies at boot reading its own configuration is a
89
+ * worse failure than anything this file refuses.
77
90
  *
78
- * Value position only, and it is not a micro-optimisation: `readString` accumulates about 32 bytes of
79
- * heap per source character, so a single 64MB string value cost 2GB of garbage to decode a value this
80
- * function then discards. A trigger's `run.task` and `run.instructions` are free text, and a worker
81
- * that dies at boot reading its own configuration is a worse failure than anything this file refuses.
91
+ * Character codes, as `skipWhitespace` reads them, because comparing numbers measured a little faster
92
+ * than comparing one-character strings on an ordinary triggers file. Either is correct.
82
93
  */
83
94
  const skipString = () => {
84
95
  i++; // the opening quote
96
+ let escaped = false;
85
97
  while (i < n) {
86
- const ch = s[i];
87
- if (ch === '"') {
98
+ const c = s.charCodeAt(i);
99
+ if (c === 0x22) {
88
100
  i++;
89
- return;
101
+ return escaped;
90
102
  }
91
103
  // An escape consumes its own next character, which is what stops `\"` ending the string. The
92
104
  // `\uXXXX` digits need no special case here: none of them is a quote or a backslash.
93
- i += ch === "\\" ? 2 : 1;
94
- }
95
- };
96
-
97
- /** Read the string starting at `s[i] === '"'` and return its DECODED value. Keys only. */
98
- const readString = () => {
99
- i++; // the opening quote
100
- let out = "";
101
- while (i < n) {
102
- const ch = s[i];
103
- if (ch === '"') {
104
- i++;
105
- return out;
106
- }
107
- if (ch === "\\") {
108
- const esc = s[i + 1];
105
+ if (c === 0x5c) {
106
+ escaped = true;
109
107
  i += 2;
110
- if (esc === "u") {
111
- // A lone surrogate is preserved as itself, which is what JSON.parse does too, so a pair
112
- // written as two escapes still compares equal to the same pair written literally.
113
- out += String.fromCharCode(Number.parseInt(s.slice(i, i + 4), 16));
114
- i += 4;
115
- } else {
116
- out += SIMPLE_ESCAPES[esc] ?? esc;
117
- }
118
- continue;
108
+ } else {
109
+ i++;
119
110
  }
120
- out += ch;
121
- i++;
122
111
  }
123
- return out; // unreachable on text that parsed
112
+ return escaped; // unreachable on text that parsed
113
+ };
114
+
115
+ /**
116
+ * Read the key starting at `s[i] === '"'` and return its DECODED value.
117
+ *
118
+ * A key with no backslash is its own text, so it is a slice, which V8 keeps as a view of `s` rather than
119
+ * a copy from 13 characters up. A key with one is decoded by `JSON.parse` of its literal, quotes
120
+ * included. That literal is valid on its own because the whole text already parsed and `skipString`
121
+ * ends a string where the parse did. Decoding every key with `JSON.parse` would give the same answers;
122
+ * the slice is there because nearly every key has no escape and a slice copies nothing.
123
+ * `json-duplicates.test.mjs` holds every `\u` code unit, in both cases of hex, and every simple escape
124
+ * against the key `JSON.parse` makes, and the hand-written decoder this replaced passed it too.
125
+ */
126
+ const readKey = () => {
127
+ const start = i;
128
+ return skipString() ? JSON.parse(s.slice(start, i)) : s.slice(start + 1, i - 1);
124
129
  };
125
130
 
126
131
  // True when the next string is a KEY rather than a value: just after `{`, or after a `,` inside one.
@@ -177,8 +182,14 @@ export function findDuplicateKey(text) {
177
182
  if (ch === '"') {
178
183
  const top = stack[stack.length - 1];
179
184
  if (expectKey && top && top.keys !== null) {
180
- const key = readString();
181
- if (top.keys.has(key)) return { key, at: [...path, key].join(".") };
185
+ const key = readKey();
186
+ // The key is copied on the way out, once: a slice is a view that keeps the whole text alive for as long
187
+ // as a caller holds the answer, and `secret-bundles.mjs` keeps `key` past the call while its text is a
188
+ // resolver's output, which holds secret values. JSON's own round trip keeps a lone surrogate as itself.
189
+ if (top.keys.has(key)) {
190
+ const own = JSON.parse(JSON.stringify(key));
191
+ return { key: own, at: [...path, own].join(".") };
192
+ }
182
193
  top.keys.add(key);
183
194
  path.push(key);
184
195
  top.named = true;
@@ -158,7 +158,7 @@ export function makeMcpCheck({ globalPiDir, forwardEnv = [], hostEnv = {}, egres
158
158
  // writes under exactly that name, which is the reason every other server is refused it.
159
159
  for (const name of overlayNames) {
160
160
  for (const ref of servers[name].refs) {
161
- if (own.has(ref)) return { refused: "mcp-secret-reserved", name, why: `it references ${ref}, which this deployment uses for the job's own credentials` };
161
+ if (own.has(ref)) return { refused: "mcp-secret-reserved", name, why: `it references ${ref}, which this deployment uses for the job's own credentials or settings` };
162
162
  }
163
163
  }
164
164
  for (const name of overlayNames) {
@@ -276,7 +276,7 @@ export const MCP_REFUSAL_COMMENT = Object.freeze({
276
276
  "mcp-list-invalid": "the list this job carries is not one the triggers file's loader writes (a service in this deployment may be at another version)",
277
277
  "mcp-config-invalid": "this deployment's list of MCP servers (mcp.json in the overlay) cannot be used",
278
278
  "mcp-server-unknown": "this deployment does not offer that server",
279
- "mcp-secret-reserved": "that server would be handed a credential this deployment keeps for the job itself",
279
+ "mcp-secret-reserved": "that server would be handed a variable this deployment keeps for the job itself",
280
280
  "mcp-secret-unmapped": "a credential that server needs is not passed by this trigger",
281
281
  "mcp-excluded": "this trigger's run.excludeTools removes every way the job could reach that server",
282
282
  "mcp-egress-unreachable": "the egress policy does not let a job reach that server",
@@ -83,8 +83,13 @@ export function pauseWindowsVersionRefusal(doc, { supported = PAUSE_WINDOWS_VERS
83
83
  export function heldPauseWindows(text, path) {
84
84
  if ((text ?? "").trim() === "") return { doc: null, windows: [] };
85
85
  const { value, problem } = heldDocument(text, "windows");
86
- const refused = problem ?? (findDuplicateKey(text) ? "has a key written twice" : null);
87
- if (refused) return { doc: value ?? null, problem: `the pause-windows file ${path} ${refused}, so its windows cannot be kept` };
86
+ // Whenever the text parsed, a problem with its shape included (`"windows": {}`), as the loader judges the duplicate first.
87
+ const doubled = value !== undefined && findDuplicateKey(text) !== null;
88
+ const refused = doubled ? "has a key written twice" : problem ?? null;
89
+ // A key written twice hands back no document (issue #846): the callers ask the document whether a newer release wrote
90
+ // it before they read `problem`, and `{"version":1,"version":2}` would send the operator to upgrade to a release that
91
+ // refuses the file too. Broken, never newer, as the loader reads it and as the triggers writer does.
92
+ if (refused) return { doc: doubled ? null : value ?? null, problem: `the pause-windows file ${path} ${refused}, so its windows cannot be kept` };
88
93
  return { doc: value, windows: Array.isArray(value.windows) ? value.windows : [] };
89
94
  }
90
95
 
@@ -158,7 +163,7 @@ function validateTz(tz, at, path) {
158
163
  * (unknown fields dropped; `fromMin`/`toMin` precomputed). Throws `configError` (fail-loud) on any malformed
159
164
  * entry. `path` is for messages only.
160
165
  *
161
- * The file's `version` is judged FIRST, before the windows (issue #761): a newer file's windows may use a shape this
166
+ * The file's `version` is judged right after the duplicate-key check (issue #846) and before the windows (issue #761): a newer file's windows may use a shape this
162
167
  * build would call malformed, and the true message is "upgrade". A newer file throws a configError with
163
168
  * `.newer === true` (the worker defers jobs rather than misread a pause); a broken `version`, or a window using a field
164
169
  * or a value that needs a higher version than the file declares, throws the ordinary configError. `supported`, `table`
@@ -171,6 +176,17 @@ export function parsePauseWindows(text, path, { supported = PAUSE_WINDOWS_VERSIO
171
176
  } catch (error) {
172
177
  throw configError(`pause-windows file is not valid JSON: ${path} (${error.message})`);
173
178
  }
179
+ // Issue #846: a key written twice is REFUSED, after the parse and before anything else, as the triggers loader refuses
180
+ // one (`triggers.mjs`, issue #313). `JSON.parse` keeps the LAST value, so `{"windows":[...],"windows":[]}` loaded no
181
+ // window while a person reading the file sees two, and a reload swapped a live list down to none with nothing said:
182
+ // quiet hours stopped applying, which costs money in exactly the hours someone believes are paused. Refused, the boot
183
+ // does not start and a reload keeps the last good windows and logs why (`reloadPauseWindows`). The writers have
184
+ // refused such a file since #841 (`heldPauseWindows`), so only a hand edit makes one. BEFORE the version, the triggers
185
+ // loader's order: a doubled `version` is broken, never newer, and a newer release refuses the duplicate as well.
186
+ const duplicate = findDuplicateKey(text);
187
+ if (duplicate) {
188
+ throw configError(`pause-windows file has a duplicate key ${JSON.stringify(duplicate.key)} at ${duplicate.at}: JSON keeps the LAST value, so the file a person reads and the file that applies are not the same: ${path}`);
189
+ }
174
190
  const problem = pauseWindowsVersionRefusal(parsed, { supported });
175
191
  if (problem) {
176
192
  const error = configError(`${problem.message}: ${path}`);
@@ -274,7 +274,7 @@ const ROWS = [
274
274
  ["mcp-config-invalid", "Refused: the list of MCP servers could not be used: the overlay `mcp.json` before the job started, or the list the worker wrote for the job inside its container. Not run.", "Fix the overlay `mcp.json` (`pi-dispatch doctor` checks it), and run a worker and a job image of the same release."],
275
275
  ["mcp-server-unknown", mcpRefusalComment("mcp-server-unknown", null), "Declare the server in the overlay `mcp.json`, or fix the name in `run.mcp`."],
276
276
  ["mcp-secret-unmapped", mcpRefusalComment("mcp-secret-unmapped", null), "Pass the credential the server needs in the trigger's `run.secrets`."],
277
- ["mcp-secret-reserved", mcpRefusalComment("mcp-secret-reserved", null), "Give the server's credential a variable of its own."],
277
+ ["mcp-secret-reserved", mcpRefusalComment("mcp-secret-reserved", null), "Give the server a variable of its own, not one the job keeps."],
278
278
  ["mcp-excluded", mcpRefusalComment("mcp-excluded", null), "Remove the server from `run.mcp`, or keep a tool that reaches it out of `run.excludeTools`."],
279
279
  ["mcp-egress-unreachable", mcpRefusalComment("mcp-egress-unreachable", null), "Add the server's host to the egress allowlist, or drop the server from `run.mcp`."],
280
280
  ["mcp-package-runner", mcpRefusalComment("mcp-package-runner", null), "Allow the package registry in the egress allowlist, or install the server in the job image."],
@@ -135,6 +135,10 @@ export function makeRedactor({ values = [], extraPatterns = [] } = {}) {
135
135
  export const NON_SECRET_ENV_KEYS = new Set([
136
136
  // Issue #693: a memory-derived concurrency count, never a credential; keep it in the structural venue env.
137
137
  "PI_CODEMODE_MAX_WORKERS",
138
+ // Issue #865: the job's size, two integers, never a secret. Listed, or OpenShell would put each in the job's credential
139
+ // provider and the agent would read a placeholder where a tool's worker count needs the number.
140
+ "PI_JOB_SIZE_CPUS",
141
+ "PI_JOB_SIZE_MEMORY_MIB",
138
142
  "PI_PROVIDER",
139
143
  "PI_MODEL",
140
144
  "PI_FORK_ENTRY",
@@ -37,6 +37,10 @@ export const CONTAINER_ENV_NAMES = new Set([
37
37
  "PI_RUNNER_OBSERVABLE",
38
38
  // Issue #693: derived from the resolved job memory, never chosen by a trigger or forwarded host value.
39
39
  "PI_CODEMODE_MAX_WORKERS",
40
+ // Issue #865: the job's size as the job reads it (job-size.mjs `jobSizeEnv`). A trigger binding one through
41
+ // run.secrets, or a forwarded host value, would tell the job's tools a size the container was not given.
42
+ "PI_JOB_SIZE_CPUS",
43
+ "PI_JOB_SIZE_MEMORY_MIB",
40
44
  "PI_PROVIDER",
41
45
  "PI_MODEL",
42
46
  "PI_FORK_ENTRY",
@@ -61,7 +61,9 @@ export async function runSandbox(argv = [], { env = process.env, deps = {} } = {
61
61
 
62
62
  if (values.deployment) {
63
63
  const { sandboxDeploymentEnv } = await import("./sandbox-deployment.mjs");
64
- const loaded = await (deps.deploymentEnv ?? sandboxDeploymentEnv)({ dir: values.deployment, env, expectedHost: values["expected-host"] });
64
+ // Only `--new` reads the preparation's settings (#863), so only `--new` loads and compares them: an opener's open or
65
+ // attach must not refuse over a shell's PI_JOB_MEMORY it would never read.
66
+ const loaded = await (deps.deploymentEnv ?? sandboxDeploymentEnv)({ dir: values.deployment, env, expectedHost: values["expected-host"], preparing: values.new !== undefined });
65
67
  if (loaded.refused) return fail(err, loaded.message);
66
68
  // Readers and runtime helpers use their process's environment. Re-exec in the clean environment so none
67
69
  // can accidentally consult the SSH shell's DOCKER_HOST, egress or rootless store (#771).
@@ -209,9 +211,13 @@ export async function runSandbox(argv = [], { env = process.env, deps = {} } = {
209
211
  return result.code ?? 0;
210
212
  }
211
213
 
212
- function runDeploymentSandbox(argv, env, dir) {
214
+ // The child runs IN the deployment folder, as the service does (its unit's WorkingDirectory), and that is load-bearing
215
+ // (#863): a relative PI_PROJECTS_FILE, PI_SCOPED_LIMITS_FILE or PI_TRIGGERS_FILE, and an unset PI_TRIGGERS_FILE's
216
+ // `triggers.json`, then name the files the deployment's worker reads. Resolving them here instead would be a second
217
+ // rule for the same paths, which drifts.
218
+ export function runDeploymentSandbox(argv, env, dir, { spawnFn = spawn } = {}) {
213
219
  return new Promise((resolve) => {
214
- const child = spawn(process.execPath, [fileURLToPath(new URL("./cli.mjs", import.meta.url)), "sandbox", ...argv], { env, ...(dir ? { cwd: dir } : {}), stdio: "inherit" });
220
+ const child = spawnFn(process.execPath, [fileURLToPath(new URL("./cli.mjs", import.meta.url)), "sandbox", ...argv], { env, ...(dir ? { cwd: dir } : {}), stdio: "inherit" });
215
221
  child.once("error", () => resolve(1));
216
222
  child.once("exit", (code) => resolve(code ?? 1));
217
223
  });
@@ -1,4 +1,7 @@
1
- /** An explicit deployment handoff is stronger than a generic deployment pointer: it loads sandbox authority only. */
1
+ /**
2
+ * An explicit deployment handoff is stronger than a generic deployment pointer: it loads sandbox authority only, plus,
3
+ * for `--new` alone, what a sandbox's preparation reads. It never loads a forge credential.
4
+ */
2
5
  import * as nodeFs from "node:fs";
3
6
  import { isAbsolute, join, win32 } from "node:path";
4
7
  import { homedir } from "node:os";
@@ -10,15 +13,38 @@ export const SANDBOX_DEPLOYMENT_KEYS = Object.freeze([
10
13
  "PI_BACKENDS", "PI_BACKEND_FLOOR", "PI_EGRESS", "PI_EGRESS_PROXY", "DOCKER_HOST", "DOCKER_CONTEXT", "DOCKER_CONFIG", "DOCKER_TLS_VERIFY", "DOCKER_CERT_PATH",
11
14
  "VALKEY_URL", "VALKEY_PASSWORD", "PI_VALKEY_SHARED", "PATH", "XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME", "CONTAINERS_STORAGE_CONF", "CONTAINERS_CONF", "CONTAINERS_CONF_OVERRIDE", "CONTAINERS_REGISTRIES_CONF", "CONTAINER_HOST", "CONTAINER_CONNECTION",
12
15
  ]);
16
+
17
+ /**
18
+ * Issue #863: what `--new` reads to plan a preparation (`planSandboxPreparation`) beyond the list above. Without them a
19
+ * sandbox prepared through `--deployment` got no project, the default size and the default image, so its plan was not
20
+ * the one the deployment's own worker and the console's preview make. A relative file path names a file in the
21
+ * deployment folder, because the re-exec runs there as the service does (`runDeploymentSandbox`).
22
+ *
23
+ * A LIST OF ITS OWN, read only when preparing. Opening, attaching and `--list` never read these, and folding them into
24
+ * the list above made every opener command refuse over a shell's `PI_JOB_MEMORY` or a `.env` line such as
25
+ * `PI_PROJECTS_FILE=$HOME/p.json` that only the service's shell can expand. When preparing they get the same strict rule
26
+ * as the rest, named: a value the shell sets that the deployment does not, or sets otherwise, refuses, and so does a line
27
+ * that cannot be read as the service reads it. Letting the deployment's value win quietly would prepare with a setting
28
+ * the person did not see.
29
+ *
30
+ * The forge's credential and address are left out on purpose (#771): with none, a GitHub trigger clones with this
31
+ * account's own `gh` login, and a trigger on another forge cannot be prepared this way (the console's preparation runs
32
+ * in the worker, with its credential). sandbox-deployment-plan.test.mjs records the plan's inputs from the real preparer,
33
+ * and fails on a new one until it is carried in one of these lists or named there with its reason.
34
+ */
35
+ export const SANDBOX_PREPARATION_KEYS = Object.freeze(["PI_TRIGGERS_FILE", "PI_PROJECTS_FILE", "PI_SCOPED_LIMITS_FILE", "PI_JOB_IMAGE", "PI_JOB_MEMORY", "PI_JOB_CPUS"]);
13
36
  const OS_KEYS = Object.freeze(["HOME", "USERPROFILE", "SystemRoot", "WINDIR", "TEMP", "TMP", "TMPDIR", "TERM", "LANG", "LC_ALL", "PATH"]);
14
- export function sandboxDeploymentEnv({ dir, env = process.env, expectedHost = null, fs = nodeFs, platform = process.platform, uid = process.geteuid?.(), home = homedir(), read = deploymentServiceEnv } = {}) {
37
+ export function sandboxDeploymentEnv({ dir, env = process.env, expectedHost = null, preparing = false, fs = nodeFs, platform = process.platform, uid = process.geteuid?.(), home = homedir(), read = deploymentServiceEnv } = {}) {
15
38
  const refuse = (message) => ({ refused: "sandbox-deployment", message });
16
39
  if (typeof dir !== "string" || !isAbsolute(dir) || /[\x00-\x1f\x7f]/.test(dir)) return refuse("--deployment must name an absolute deployment directory");
17
40
  // One descriptor-backed read. Ambient values are compared afterwards, never used as defaults for authority.
18
- const resolved = read({ dir, env: {}, keys: SANDBOX_DEPLOYMENT_KEYS, platform, fs, uid });
41
+ const resolved = read({ dir, env: {}, keys: preparing ? [...SANDBOX_DEPLOYMENT_KEYS, ...SANDBOX_PREPARATION_KEYS] : SANDBOX_DEPLOYMENT_KEYS, platform, fs, uid });
19
42
  if (resolved.unreadable || resolved.untrusted || resolved.hazard) return refuse("the deployment .env cannot be trusted or read as its service reads it; run pi-dispatch doctor");
20
43
  if (!resolved.path || (resolved.fromFile && Object.keys(resolved.fromFile).length === 0 && resolved.env && !fs.existsSync(resolved.path))) return refuse("the deployment .env is missing; point --deployment at the installed deployment");
21
- if (resolved.unread?.length || resolved.hazardSkipped?.length) return refuse("a sandbox setting cannot be read the way the deployment service reads it; run pi-dispatch doctor");
44
+ const unread = [...(resolved.unread ?? []).map((u) => u.key), ...(resolved.hazardSkipped ?? [])];
45
+ if (unread.some((key) => SANDBOX_DEPLOYMENT_KEYS.includes(key))) return refuse("a sandbox setting cannot be read the way the deployment service reads it; run pi-dispatch doctor");
46
+ const unreadPlan = unread.find((key) => SANDBOX_PREPARATION_KEYS.includes(key));
47
+ if (unreadPlan) return refuse(`the deployment's ${unreadPlan} line cannot be read the way its service reads it; run pi-dispatch doctor before preparing`);
22
48
  const clean = { ...resolved.env };
23
49
  for (const key of SANDBOX_DEPLOYMENT_KEYS) {
24
50
  if (typeof env[key] !== "string") continue;
@@ -38,11 +64,18 @@ export function sandboxDeploymentEnv({ dir, env = process.env, expectedHost = nu
38
64
  if (key === "XDG_CONFIG_HOME" && env[key] === join(home, ".config")) { clean[key] = env[key]; continue; }
39
65
  return refuse(`the shell sets sandbox authority absent from the deployment: ${key}; unset that variable before opening`);
40
66
  }
67
+ if (preparing) {
68
+ for (const key of SANDBOX_PREPARATION_KEYS) {
69
+ if (typeof env[key] !== "string") continue;
70
+ if (typeof clean[key] !== "string") return refuse(`the shell sets ${key}, which the deployment's .env does not set; unset that variable before preparing`);
71
+ if (clean[key] !== env[key]) return refuse(`the shell disagrees with the deployment for ${key}; unset that variable before preparing`);
72
+ }
73
+ }
41
74
  const result = {};
42
75
  for (const key of OS_KEYS) if (typeof env[key] === "string") result[key] = env[key];
43
76
  // env-internal HOME: compare the launching account's home to the OS identity before opening a rootless store.
44
77
  if (platform !== "win32" && env.HOME && env.HOME !== home) return refuse("HOME is not this account's home; open the sandbox as the worker's account");
45
- for (const key of SANDBOX_DEPLOYMENT_KEYS) if (typeof clean[key] === "string") result[key] = clean[key];
78
+ for (const key of preparing ? [...SANDBOX_DEPLOYMENT_KEYS, ...SANDBOX_PREPARATION_KEYS] : SANDBOX_DEPLOYMENT_KEYS) if (typeof clean[key] === "string") result[key] = clean[key];
46
79
  if (typeof result.PATH === "string" && (/[\x00-\x1f\x7f]/.test(result.PATH) || result.PATH.split(platform === "win32" ? ";" : ":").some((part) => !part || !(platform === "win32" ? win32.isAbsolute(part) : isAbsolute(part))))) return refuse("the sandbox runtime PATH must contain only absolute tool directories; correct PATH in the deployment .env");
47
80
  try {
48
81
  if (expectedHost !== null && workerName(result) !== expectedHost) return refuse("the deployment worker name does not match the opener host; check Fleet and the local opener configuration");
@@ -21,6 +21,13 @@ const PREPARATION_LOCK_MAX_BYTES = 16 * 1024;
21
21
  const hash = (value) => createHash("sha256").update(value).digest("hex");
22
22
  const refused = (reason, message) => ({ refused: reason, message });
23
23
  const configRefusal = () => refused("sandbox-config-invalid", "the host's sandbox configuration could not be read; run pi-dispatch doctor on that host");
24
+ /**
25
+ * What a preparation needs set to clone from each forge whose address is configuration (#863), as `config.mjs` reads
26
+ * them (`loadGitLabAuth`, `loadForgejoAuth`, `loadAzureAuth`: no block at all without the token). GitHub's address is
27
+ * fixed. Named in the refusal so it says what to set, in words true both in a worker serving the console and in a
28
+ * terminal; sandbox-prepare.test.mjs holds each name to `loadConfig`.
29
+ */
30
+ export const SANDBOX_FORGE_SETTINGS = Object.freeze({ gitlab: "GitLab (GITLAB_TOKEN, and GITLAB_URL when it is not gitlab.com)", forgejo: "Forgejo (FORGEJO_TOKEN and FORGEJO_URL)", azure: "Azure DevOps (AZURE_TOKEN and AZURE_ORG_URL)" });
24
31
 
25
32
  export function sandboxTriggerDigest(text, index) {
26
33
  const rows = JSON.parse(text)?.triggers;
@@ -123,11 +130,18 @@ export function createSandboxPreparer({ env = process.env, config = loadConfig(e
123
130
  catch (error) { if (error?.piDispatchConfig || error?.code === "ENOENT") return configRefusal(); throw new Error("sandbox preparation configuration read failed", { cause: error }); }
124
131
  if (planned.refused) return planned;
125
132
  const { plan } = planned;
133
+ // A forge's settings are configuration, so a preparation without usable ones is a determinate refusal, said here
134
+ // before any directory, lock, credential or clone. Inside the directories' catch below it read as infrastructure: the
135
+ // CLI said "try again" and the control queue retried a request that can never succeed (#863,
136
+ // CONST-RETRY-INFRA-ONLY). The plan has already refused a malformed repository (`sandbox-repo-invalid`, from the same
137
+ // `sandboxRepoProblem` this calls), so what is left to throw is the forge's settings: no block at all (its token is
138
+ // unset, whatever its URL says), or an address that is not a plain http(s) URL.
139
+ let remote;
140
+ try { remote = sandboxRemoteUrl(plan.kind, plan.repo, config); }
141
+ catch { return refused("sandbox-forge-unconfigured", `this preparation has no usable settings for ${SANDBOX_FORGE_SETTINGS[plan.kind] ?? "the trigger's forge"}, so it cannot clone the repository`); }
126
142
  const id = request.id ?? newSandboxId();
127
143
  const fingerprint = hash(JSON.stringify({ plan, by: request.by ?? null }));
128
- let remote;
129
144
  try {
130
- remote = sandboxRemoteUrl(plan.kind, plan.repo, config);
131
145
  ensureDir(config.sandboxDir, { env });
132
146
  fs.mkdirSync(config.sandboxDir, { recursive: true, mode: 0o700 });
133
147
  ensureDir(config.sandboxDir, { env });
package/src/sandbox.mjs CHANGED
@@ -10,7 +10,7 @@ import { configError } from "./config.mjs";
10
10
  import { assertJobUser, CONTAINER_HOME, SHIPPED_IMAGE_UID } from "./container-spec.mjs";
11
11
  import { insideDir } from "./docker-run.mjs";
12
12
  import { CGROUP_PARENT, cgroupParentFor } from "./cpu-reserve.mjs";
13
- import { DEFAULT_JOB_SIZE, recordedJobSize } from "./job-size.mjs";
13
+ import { DEFAULT_JOB_SIZE, jobSizeEnv, recordedJobSize } from "./job-size.mjs";
14
14
  import { makeImagePreflight } from "./image-preflight.mjs";
15
15
  import { decideJobUser, JOB_USER_FIX, makeDaemonFactsReader, relabelsPrivateMounts, resolveImageUser, socketFacts } from "./job-user.mjs";
16
16
  import { DEFAULT_EGRESS_PROXY, NETWORK_SUFFIX, createJobNetwork, egressArmed, egressEnv, egressProxyName, networkEndpoints, networkExists, networkNameFor, removeJobNetwork, removeNetworkOrSay } from "./egress.mjs";
@@ -43,7 +43,8 @@ export { SANDBOX_LAUNCHERS, SANDBOX_OPEN_GRACE_MS, sandboxLauncher, sandboxSyncR
43
43
  * (env-allowlist.mjs) and throws outright when no provider credential resolves, so a credential-free
44
44
  * container cannot be produced from it. The env here is two variables about the terminal, plus the four
45
45
  * proxy variables when an egress policy is armed, plus `HOME=/home/pi` beside `--user` when the run had a job
46
- * user (issue #341).
46
+ * user (issue #341), plus the size the manifest records (`PI_JOB_SIZE_CPUS`, `PI_JOB_SIZE_MEMORY_MIB`, issue #865):
47
+ * the run's, or for a prepared sandbox the size its trigger's jobs get.
47
48
  * - No `/outbox`, no `/session`, no `/opt/pi-global`. The agent is not running; there is nothing to
48
49
  * chain, no transcript to continue and no overlay to layer.
49
50
  *
@@ -177,9 +178,15 @@ export function buildSandboxRunArgs({ venue = DEFAULT_BACKEND, image, name, work
177
178
  TMOUT: idleSeconds > 0 ? String(idleSeconds) : undefined,
178
179
  // Beside `--user` only (issue #341). Not a credential either.
179
180
  HOME: user !== null ? home : undefined,
181
+ // Issue #865: the size this shell is built at (`size`: the manifest's, the run's own or, for a prepared
182
+ // sandbox, its trigger's), by the job env's own function (`jobSizeEnv`), so a flow command copied in here
183
+ // (`jest --maxWorkers=$PI_JOB_SIZE_CPUS`) sizes its pool as a job of that size would instead of expanding to
184
+ // nothing. A run retained before this release never had the variables; its shell gets them for its recorded
185
+ // size all the same. Two integers, not a credential.
186
+ ...jobSizeEnv(size),
180
187
  // Still NO CREDENTIALS, and that clause is untouched: a proxy URL is not a credential, and
181
188
  // buildContainerEnv is still not reused here. The env is two variables about the terminal, HOME
182
- // beside `--user`, and, when a policy is armed, four about the network.
189
+ // beside `--user`, two about the run's size and, when a policy is armed, four about the network.
183
190
  ...proxyEnv,
184
191
  },
185
192
  // Ahead of the env and the mounts, and well ahead of the image, which the builder keeps as