@edgehero/pi-dispatch 1.10.2 → 2.0.0

package/src/exit-code.mjs

@@ -86,3 +86,18 @@ export function decideWait(exitCode) {
 			return { verdict: "hold", fault: true }; // EXIT_INFRA and every unrecognised code
 	}
 }
+
+/**
+ * A process-level printer for a promise nobody handled (PR #475's review, round 2): Node prints such a rejection's
+ * reason WHOLE, and a Valkey client's error may carry what it sent (connection.mjs scrubs that at the source; this is
+ * the second line of defence for anything else). It prints the message alone and exits 1, which is what Node's own
+ * default does with an unhandled rejection: infra, restarted by the service manager, never a silent carry-on. Installed
+ * by the worker's CLI (every `pi-dispatch` verb, the worker among them) and the receiver's two entry points.
+ */
+export function installRejectionPrinter({ proc = process, write = (line) => process.stderr.write(line) } = {}) {
+	proc.on("unhandledRejection", (reason) => {
+		const message = reason instanceof Error ? reason.message : typeof reason === "string" ? reason : "a non-Error value";
+		write(`error: an unhandled rejection: ${message}\n`);
+		proc.exit(EXIT_INFRA);
+	});
+}

package/src/flow-gate.mjs

@@ -1,3 +1,4 @@
+import { GIT_READ_FLAGS } from "./git-hardening.mjs";
 import { execFile } from "node:child_process";
 import { promisify } from "node:util";
 
@@ -74,9 +75,10 @@ export function aiTriggerAllows(buf) {
 }
 
 async function defaultGit(gitDir, args, { raw = false } = {}) {
-	// hardening flags mirror materialize.mjs defaultGit — keep in sync. No hooks, no fsmonitor, no
-	// pager, so a hostile repo config cannot run code or corrupt output during a read.
-	const hardened = ["-c", "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false", "--no-pager", "-C", gitDir, ...args];
+	// No hooks, no fsmonitor, no pager, so a hostile repo config cannot run code or corrupt output during
+	// a read. This said "mirror materialize.mjs -- keep in sync" until the copy that had NOT stayed in
+	// sync was found (issue #286's sweep); it is an import now, so there is nothing left to remember.
+	const hardened = [...GIT_READ_FLAGS, "-C", gitDir, ...args];
 	const { stdout } = await exec("git", hardened, {
 		encoding: raw ? "buffer" : "utf8",
 		maxBuffer: 16 * 1024 * 1024,

package/src/forgejo-host.mjs

@@ -24,6 +24,7 @@ import { configError } from "./config.mjs";
 import { InfraRetry } from "./processor.mjs";
 import { fetchFailureReason } from "./gitlab-identity.mjs";
 import { matchesBranch } from "./gitlab-host.mjs";
+import { isDeterminateFetchFailure } from "./transient.mjs";
 
 const API_PREFIX = "/api/v1";
 
@@ -37,6 +38,15 @@ export function makeForgejoHost({ apiUrl, fetchFn = fetch } = {}) {
 		try {
 			res = await fetchFn(`${root}${path}`, { headers: { Authorization: `token ${token}` }, redirect: "error" });
 		} catch (err) {
+			// The one condition where the identity modules and this one now agree, because the issue's
+			// acceptance asks for exactly that (#316): a TLS trust failure, a protocol mismatch and a URL
+			// that always redirects are the operator's, and retrying them twice before failing tells
+			// nobody anything. Everything else here stays unconditionally retryable, which is right for a
+			// per-job call behind the queue: a wrong retry costs one more attempt, a wrong refusal costs
+			// the delivery and posts publicly that the deployment is misconfigured.
+			if (isDeterminateFetchFailure(err)) {
+				throw configError(`forgejo-host: GET ${path} failed (${fetchFailureReason(err)})`);
+			}
 			throw new InfraRetry(`forgejo-host: GET ${path} failed (${fetchFailureReason(err)})`);
 		}
 		if (res.status === 404 && notFound !== undefined) return notFound;
@@ -114,6 +124,15 @@ export function makeForgejoHost({ apiUrl, fetchFn = fetch } = {}) {
 				redirect: "error",
 			});
 		} catch (err) {
+			// The one condition where the identity modules and this one now agree, because the issue's
+			// acceptance asks for exactly that (#316): a TLS trust failure, a protocol mismatch and a URL
+			// that always redirects are the operator's, and retrying them twice before failing tells
+			// nobody anything. Everything else here stays unconditionally retryable, which is right for a
+			// per-job call behind the queue: a wrong retry costs one more attempt, a wrong refusal costs
+			// the delivery and posts publicly that the deployment is misconfigured.
+			if (isDeterminateFetchFailure(err)) {
+				throw configError(`forgejo-host: POST ${path} failed (${fetchFailureReason(err)})`);
+			}
 			throw new InfraRetry(`forgejo-host: POST ${path} failed (${fetchFailureReason(err)})`);
 		}
 		if (!res.ok) {

package/src/forgejo-identity.mjs

@@ -22,6 +22,7 @@
  */
 
 import { configError } from "./config.mjs";
+import { isDeterminateFetchFailure, isJsonContentType, isTransientStatus, responseHeaderReader, transientError } from "./transient.mjs";
 import { fetchFailureReason } from "./gitlab-identity.mjs";
 
 const API_PREFIX = "/api/v1";
@@ -47,7 +48,11 @@ export async function resolveForgejoSelfId({ apiUrl, token, botId = null, fetchF
 	try {
 		res = await fetchFn(url, { headers: { Authorization: `token ${token}` }, redirect: "error" });
 	} catch (err) {
-		throw configError(`could not resolve the forgejo bot identity from ${url}: ${fetchFailureReason(err)}`);
+		// Transient, EXCEPT for the trust and redirect faults below, which is where this deliberately
+		// diverges from forgejo-host.mjs: the host retries every rejection unconditionally, and being
+		// unconditional is correct there because a job retry is cheap. A boot refusal is not (issue #316).
+		if (isDeterminateFetchFailure(err)) throw configError(`could not resolve the forgejo bot identity from ${url}: ${fetchFailureReason(err)}`);
+		throw transientError(`could not resolve the forgejo bot identity from ${url}: ${fetchFailureReason(err)}`, err);
 	}
 	if (res.status === 403 || res.status === 401) {
 		// The likely cause, named. A repo-scoped Forgejo token cannot carry `read:user`, so this is the
@@ -58,13 +63,27 @@ export async function resolveForgejoSelfId({ apiUrl, token, botId = null, fetchF
 	}
 	if (!res.ok) {
 		// The status only. A Forgejo error body can echo the request, and the request carried the token.
+		// 401 and 403 were answered above, where they name the scope fix; what is left splits on the
+		// shared rule, so a self-hosted instance answering 502 mid-restart no longer reads as a
+		// misconfiguration the operator has to go and find.
+		if (isTransientStatus(res.status, responseHeaderReader(res))) {
+			throw transientError(`could not resolve the forgejo bot identity: GET /user returned ${res.status}`);
+		}
 		throw configError(`could not resolve the forgejo bot identity: GET /user returned ${res.status}`);
 	}
 	let body;
 	try {
 		body = await res.json();
 	} catch (err) {
-		throw configError(`could not resolve the forgejo bot identity: unparseable JSON from GET /user (${err?.message ?? "unknown"})`);
+		// A body that will not parse is ambiguous in the expensive direction, so the CONTENT TYPE
+		// decides it. A truncated JSON body is transient. An HTML page is not: it is a Cloudflare
+		// Access or OIDC portal answering instead of the forge, and no amount of restarting gets past
+		// one (issue #316).
+		const contentType = responseHeaderReader(res);
+		if (contentType("content-type") !== undefined && !isJsonContentType(contentType)) {
+			throw configError(`could not resolve the forgejo bot identity: GET /user answered ${String(contentType("content-type"))} instead of JSON, so something other than the Forgejo API replied`);
+		}
+		throw transientError(`could not resolve the forgejo bot identity: unparseable JSON from GET /user (${err?.message ?? "unknown"})`, err);
 	}
 	const id = body?.id;
 	if (!Number.isInteger(id)) {

package/src/get-token.mjs

@@ -28,6 +28,7 @@ import { Octokit as RealOctokit } from "@octokit/rest";
 import { configError } from "./config.mjs";
 import { resolveSelfId } from "./identity.mjs";
 import { InfraRetry } from "./processor.mjs";
+import { isDeterminateFetchFailure, isDeterminateFsCode, isTransientStatus, octokitHeaderReader } from "./transient.mjs";
 
 /**
  * Build the auth surface for `cfg = { source, patVar?, appId?, installationId?, privateKeyPath? }`.
@@ -138,14 +139,39 @@ function requireToken(raw, what) {
 	return token;
 }
 
-/** Run `gh auth token` (array args, no shell) and return the trimmed token, or throw configError. */
+/**
+ * Run `gh auth token` (array args, no shell) and return the trimmed token.
+ *
+ * THIS RUNS PER JOB, not only at boot: `makeGitHubAuth`'s gh arm mints once to resolve the identity and
+ * then again on every delivery, so a fault here is a fault on the job path.
+ *
+ * `promisify(execFile)` overloads `error.code`, and issue #316 is what that overloading cost. A SPAWN
+ * failure sets it to a string errno; a non-zero EXIT sets it to the integer status. The old comment
+ * called both deterministic and tagged both, so `EAGAIN` and `EMFILE` under fd pressure -- the host being
+ * momentarily busy -- were reported to the issue author as "is the gh CLI installed and logged in?" and
+ * never retried.
+ *
+ * So: a spawn errno splits on the same allow-list every other absence question uses, and `ENOENT` (no
+ * binary on PATH) stays determinate. A NON-ZERO EXIT stays determinate too, and that is a judgment call
+ * rather than a derivation. `gh auth token` exits 1 for a logged-out host far more often than for a
+ * locked keyring or a token refresh that could not reach the network, and the only thing that separates
+ * them is a table of gh's own stderr strings, which is exactly the hand-maintained table this project
+ * keeps deleting. The residual is named here rather than papered over.
+ */
 async function runGhAuthToken(execFileAsync) {
 	let stdout;
 	try {
 		({ stdout } = await execFileAsync("gh", ["auth", "token"]));
 	} catch (error) {
-		// ENOENT (binary absent) or a non-zero exit (logged out / broken) -- both deterministic.
 		const detail = error?.code ?? error?.message ?? "unknown";
+		// A child killed by a SIGNAL sets `code` to null and `signal` to the name. An OOM-killed `gh` is
+		// the host being under pressure, not a deployment that is wrong.
+		if (error?.signal) {
+			throw new InfraRetry(`\`gh auth token\` was killed by ${error.signal}; the host is under pressure`);
+		}
+		if (typeof error?.code === "string" && !isDeterminateFsCode(error.code)) {
+			throw new InfraRetry(`\`gh auth token\` could not be run (${detail}); the host is momentarily unable to spawn it`);
+		}
 		throw configError(`\`gh auth token\` failed (${detail}); is the gh CLI installed and logged in?`);
 	}
 	// Logged-out gh can exit 0 with empty stdout; an empty token is the money hole, so refuse it.
@@ -177,30 +203,53 @@ function repoNameOf(repo) {
 
 /**
  * Map an octokit/auth-app rejection to the retry-vs-config distinction the queue depends on.
- * Retryable (InfraRetry): 429, 403 + Retry-After (secondary rate limit), 5xx, and status-less
- * network faults (ENOTFOUND/ECONNRESET/ETIMEDOUT). Deterministic (configError): 401 bad
- * credentials, 404 unknown installation, and any other 4xx. Collapsing all 4xx to config would
- * turn a transient rate-limit into a permanent failure, so the classes are kept disjoint.
+ *
+ * Determinate (configError): 401 bad credentials and 404 unknown installation, both of which refuse
+ * identically until an operator changes something. Everything the shared rule calls transient is an
+ * `InfraRetry`; every other 4xx is determinate.
+ *
+ * TWO CORRECTIONS TO WHAT THIS COMMENT USED TO CLAIM (issue #316), because the old version described a
+ * function that did not exist.
+ *
+ * It said 403-with-retry-after was the rate-limit case. That is the SECONDARY limit. GitHub answers the
+ * PRIMARY limit -- the one a busy deployment actually hits, and which clears within the hour -- with 403,
+ * `x-ratelimit-remaining: 0` and no `retry-after`, so it fell through to the catch-all and became a
+ * permanent refusal. 408 and 425 went the same way, and so did an unparseable status, which
+ * `@octokit/request-error` reports as the NUMBER 0.
+ *
+ * It also said status-less network faults reach the bottom branch. They do not:
+ * `@octokit/request`'s fetch wrapper turns every network rejection into `RequestError(message, 500)`, so
+ * that branch is unreachable through octokit, and the 5xx arm has been doing the work all along.
+ *
+ * TWO CONSEQUENCES OF THAT SAME FACT, and they run in opposite directions.
+ *
+ * A TRUST OR REDIRECT FAULT ARRIVES WEARING A 500. The wrapper keeps the original rejection as `cause`,
+ * so the chain is intact, but the status says 500 and 500 is transient. A private CA in front of a GHES
+ * instance, or a TLS-inspecting corporate proxy re-signing `api.github.com`, would otherwise be retried
+ * forever instead of naming `NODE_EXTRA_CA_CERTS`. So the determinate check runs FIRST.
+ *
+ * AND AN ABSENT STATUS MEANS THIS NEVER REACHED THE NETWORK. Octokit always sets one, so a status-less
+ * rejection came from local code: `@octokit/auth-app` signing the JWT with a key that is not PKCS//8, the
+ * commonest App setup mistake there is. That is determinate, and it used to be retried and then reported
+ * as an infrastructure failure with the real reason nowhere an operator would look.
  */
 function classifyAppMintError(error) {
 	const status = typeof error?.status === "number" ? error.status : undefined;
-	const retryAfter = error?.response?.headers?.["retry-after"];
+	const detail = error?.message ?? "unknown";
 
+	if (isDeterminateFetchFailure(error)) return configError(`app token mint refused: ${detail}`);
 	if (status === 401) return configError("app token mint refused: bad app credentials (401)");
 	if (status === 404) return configError("app token mint refused: unknown installation (404)");
-	if (status === 429) return new InfraRetry("app token mint: rate limited (429)");
-	if (status === 403 && retryAfter !== undefined) {
-		return new InfraRetry("app token mint: secondary rate limit (403 + retry-after)");
-	}
-	if (status !== undefined && status >= 500) {
-		return new InfraRetry(`app token mint: upstream error (${status})`);
+	if (status === undefined) {
+		// Local, and therefore the operator's. `code` is worth carrying: a bad PEM surfaces as an OpenSSL
+		// DECODER error whose code says more than its message does.
+		const code = error?.code ? `${error.code}: ` : "";
+		return configError(`app token mint failed before any request was made (${code}${detail})`);
 	}
-	if (status !== undefined) {
-		return configError(`app token mint failed (${status}): ${error?.message ?? "unknown"}`);
+	if (isTransientStatus(status, octokitHeaderReader(error), detail)) {
+		return new InfraRetry(`app token mint: transient upstream refusal (${status})`);
 	}
-	// No HTTP status -> network-level fault. Retry per INT-RUNNER-EXIT-CODE-PROTOCOL.
-	const detail = error?.code ? `${error.code}: ` : "";
-	return new InfraRetry(`app token mint: network fault: ${detail}${error?.message ?? "unknown"}`);
+	return configError(`app token mint failed (${status}): ${detail}`);
 }
 
 /**

package/src/git-dirty.mjs

@@ -1,14 +1,22 @@
 import { execFileSync } from "node:child_process";
+import { GIT_READ_FLAGS } from "./git-hardening.mjs";
 
 /**
  * Report a folder's git working-tree state: `true` = dirty, `false` = clean, `null` = not a usable
  * git repository. Reads the WORKING TREE via `git status --porcelain` — distinct from
  * flow-gate.mjs's object-store read, which reads committed content at a pinned SHA; the two are kept
  * separate on purpose. `exec` is injectable for tests.
+ *
+ * HARDENED, and this is the site where it is not hypothetical. `git status` REFRESHES THE INDEX, so it
+ * invokes `core.fsmonitor` -- an operator-supplied command -- where `rev-parse` does not. The folder this
+ * reads is the same path a local job mounts at /workspace with write access, so the agent can write
+ * `.git/config` and the next `pi-dispatch run` on that folder executes it ON THE HOST, outside any
+ * container. This file was invisible to the sweep that hardened its six siblings, because that census
+ * grepped for the flags that were PRESENT and this one had none.
  */
 export function gitDirty(folder, { exec = execFileSync } = {}) {
 	try {
-		const out = exec("git", ["-C", folder, "status", "--porcelain"], { encoding: "utf8" });
+		const out = exec("git", [...GIT_READ_FLAGS, "-C", folder, "status", "--porcelain"], { encoding: "utf8" });
 		return out.trim().length > 0;
 	} catch {
 		return null;

package/src/git-hardening.mjs

@@ -0,0 +1,33 @@
+/**
+ * The `-c` pairs that stop a hostile repository config executing code on the worker HOST, outside any
+ * container. `materialize.mjs` states the intent in one line and it is the whole of it: no hooks, no
+ * external filters, no pager.
+ *
+ * A LEAF that imports nothing, on `container-spec.mjs`'s precedent and for its reason. `materialize.mjs`
+ * is the file three "keep in sync" comments already named as canonical, but it imports `flow-gate.mjs`
+ * and `flow-gate.mjs` needs these flags too, so canonical-by-comment could not become
+ * canonical-by-import without a cycle. The array moves here and the comments become imports.
+ *
+ * Seven sites carried this by hand and one of them -- `prepare-local.mjs` -- was missing
+ * `core.fsmonitor`. Nothing was exploitable: its only command is `git rev-parse HEAD`, which does not
+ * refresh the index, so the fsmonitor hook is never invoked, and the `.pi/` materialisation beside it
+ * uses materialize's own hardened git. That is exactly the shape of a drift that survives review --
+ * harmless in the copy that has it wrong, load-bearing in the six that have it right -- and it is why
+ * this file exists rather than an eighth careful copy.
+ *
+ * `core.fsmonitor` is the one worth naming: it makes git run an operator-supplied command on any
+ * index-refreshing read, and a local-folder job's agent can write `.git/config` inside `/workspace`, so
+ * the value is attacker-controlled by construction on exactly the path that reads it.
+ */
+export const GIT_SAFE_CONFIG = Object.freeze(["-c", "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false"]);
+
+/**
+ * The whole prefix for a READ that must run no repository-supplied code. `-C <dir>` follows at the call
+ * site, because two callers pass a `-C` and one (doctor's probe) supplies its own directory handling.
+ *
+ * Two constants rather than one: `prepare-github.mjs` INTERLEAVES its clone-specific locks
+ * (`protocol.ext.allow`, an empty `credential.helper`) between the fsmonitor pair and `--no-pager`, so a
+ * single flat prefix would not compose there and would force that file's argv to be reordered for the
+ * convenience of this one.
+ */
+export const GIT_READ_FLAGS = Object.freeze([...GIT_SAFE_CONFIG, "--no-pager"]);

package/src/github-app-setup.mjs

@@ -235,16 +235,28 @@ export async function runGithubAppSetup(argv = [], deps = {}) {
 			fs.writeFileSync(envPath, "");
 			say(`note: no .env existed here — created one for the lines you approved\n`);
 		}
-		updateEnvFile(envPath, "GITHUB_AUTH_SOURCE", "app", { fs, overwrite: true });
-		updateEnvFile(envPath, "GITHUB_APP_ID", String(app.id), { fs, overwrite: true });
-		updateEnvFile(envPath, "GITHUB_APP_PRIVATE_KEY_PATH", pemPath, { fs, overwrite: true });
-		summary.push(["auth source", "GITHUB_AUTH_SOURCE=app (+ app id and key path) written to .env"]);
-		if (offerSecret) {
-			// setEnvKeyIfEmpty, NOT overwrite: an operator's existing webhook secret is what their
-			// already-configured hooks sign with — replacing it would invalidate working deliveries.
-			const { changed } = updateEnvFile(envPath, "WEBHOOK_SECRET", app.webhook_secret, { fs });
-			say(changed ? "✓ WEBHOOK_SECRET set in .env (value not shown)\n" : "✓ WEBHOOK_SECRET already set in .env — kept (your configured hooks keep verifying)\n");
-			summary.push(["WEBHOOK_SECRET", changed ? "set from the App's minted secret (value not shown)" : "already set — kept, so existing deliveries stay valid"]);
+		// WRAPPED, because `updateEnvFile` can now refuse rather than write: a `.env` owned by another
+		// account or another group, and a value no loader of that file can read back. Unwrapped, the
+		// refusal would leave a half-done setup -- the PEM already on disk at 0600, possibly an empty
+		// `.env` just created above -- and a stack trace instead of the summary this command exists to
+		// print. The App itself is already created on GitHub by this point, so saying exactly what landed
+		// is the whole remaining value.
+		try {
+			updateEnvFile(envPath, "GITHUB_AUTH_SOURCE", "app", { fs, overwrite: true });
+			updateEnvFile(envPath, "GITHUB_APP_ID", String(app.id), { fs, overwrite: true });
+			updateEnvFile(envPath, "GITHUB_APP_PRIVATE_KEY_PATH", pemPath, { fs, overwrite: true });
+			summary.push(["auth source", "GITHUB_AUTH_SOURCE=app (+ app id and key path) written to .env"]);
+			if (offerSecret) {
+				// setEnvKeyIfEmpty, NOT overwrite: an operator's existing webhook secret is what their
+				// already-configured hooks sign with — replacing it would invalidate working deliveries.
+				const { changed } = updateEnvFile(envPath, "WEBHOOK_SECRET", app.webhook_secret, { fs });
+				say(changed ? "✓ WEBHOOK_SECRET set in .env (value not shown)\n" : "✓ WEBHOOK_SECRET already set in .env — kept (your configured hooks keep verifying)\n");
+				summary.push(["WEBHOOK_SECRET", changed ? "set from the App's minted secret (value not shown)" : "already set — kept, so existing deliveries stay valid"]);
+			}
+		} catch (err) {
+			say(`error: could not write the App's lines into ${envPath}: ${err?.message}\n`);
+			say(`    → the App exists on GitHub and its key is at ${pemPath}. Add GITHUB_AUTH_SOURCE=app, GITHUB_APP_ID=${app.id} and GITHUB_APP_PRIVATE_KEY_PATH=${pemPath} to that file by hand\n`);
+			summary.push(["auth source", `NOT written: ${err?.message}`]);
 		}
 		say(`✓ credentials written\n`);
 
@@ -320,7 +332,12 @@ export async function runGithubAppSetup(argv = [], deps = {}) {
 		// so this one IS printed in full.
 		say(`\nsetup would write to ${envPath}:\n      GITHUB_APP_INSTALLATION_ID=${chosen.id}${chosen.account?.login ? ` (account: ${chosen.account.login})` : ""}\n`);
 		if (await consent(prompt, "write it? [y/N] ")) {
-			updateEnvFile(envPath, "GITHUB_APP_INSTALLATION_ID", String(chosen.id), { fs, overwrite: true });
+			try {
+				updateEnvFile(envPath, "GITHUB_APP_INSTALLATION_ID", String(chosen.id), { fs, overwrite: true });
+			} catch (err) {
+				say(`error: could not write GITHUB_APP_INSTALLATION_ID into ${envPath}: ${err?.message}\n`);
+				say(`    → add GITHUB_APP_INSTALLATION_ID=${chosen.id} to that file by hand\n`);
+			}
 			say("✓ GITHUB_APP_INSTALLATION_ID written\n");
 			summary.push(["installation id", `GITHUB_APP_INSTALLATION_ID=${chosen.id} written to .env`]);
 		} else {
@@ -346,7 +363,7 @@ function printSummary(say, summary, { noWebhook }) {
 		say("  Polling delivery is a follow-up; until it lands, forge triggers for this App will not fire on their own.\n");
 	} else {
 		say("  start the receiver so deliveries have somewhere to land: `pi-dispatch-receiver`\n");
-		say("  (or the deploy/docker-compose.yml receiver profile), reachable at the --webhook-url you gave GitHub.\n");
+		say("  (or, where the folder holds deploy/docker-compose.yml, its receiver profile), reachable at the --webhook-url you gave GitHub.\n");
 	}
 	say("  `pi-dispatch doctor` re-checks the whole deployment, including these credentials.\n");
 }

package/src/github-prompt.mjs

@@ -3,7 +3,10 @@
  *
  * ISOLATION BOUNDARY (read before touching the delimiter below):
  * The string this returns is written to /job/prompt.md and handed to session.prompt() as the USER
- * prompt — never a system prompt, never appendSystemPrompt (see image/runner/run-job.mjs:21,37,93).
+ * prompt: never a system prompt, never appendSystemPrompt. See image/runner/run-job.mjs:105 (read from
+ * /job/prompt.md) and :367 (`session.prompt(prompt)`), and image/runner/src/loader.mjs:366, whose
+ * appendSystemPromptOverride carries only the guardrails, the outbox protocol and the personas (why it
+ * overrides rather than discovers: :219-224).
  * That placement IS the control: issue/PR text is data because it enters as a user turn, after the
  * persona and the baked HARD_RULES system prompt, which the model treats as authoritative
  * (CONST-ISSUE-TEXT-IS-DATA). The `## Triggering …` heading and the code fence around the payload are

package/src/gitlab-host.mjs

@@ -26,6 +26,7 @@
 import { configError } from "./config.mjs";
 import { InfraRetry } from "./processor.mjs";
 import { fetchFailureReason } from "./gitlab-identity.mjs";
+import { isDeterminateFetchFailure } from "./transient.mjs";
 
 const API_PREFIX = "/api/v4";
 
@@ -39,6 +40,15 @@ export function makeGitLabHost({ apiUrl = "https://gitlab.com", fetchFn = fetch
 		try {
 			res = await fetchFn(`${root}${path}`, { headers: { "PRIVATE-TOKEN": token }, redirect: "error" });
 		} catch (err) {
+			// The one condition where the identity modules and this one now agree, because the issue's
+			// acceptance asks for exactly that (#316): a TLS trust failure, a protocol mismatch and a URL
+			// that always redirects are the operator's, and retrying them twice before failing tells
+			// nobody anything. Everything else here stays unconditionally retryable, which is right for a
+			// per-job call behind the queue: a wrong retry costs one more attempt, a wrong refusal costs
+			// the delivery and posts publicly that the deployment is misconfigured.
+			if (isDeterminateFetchFailure(err)) {
+				throw configError(`gitlab-host: GET ${path} failed (${fetchFailureReason(err)})`);
+			}
 			throw new InfraRetry(`gitlab-host: GET ${path} failed (${fetchFailureReason(err)})`);
 		}
 		if (res.status === 404 && notFound !== undefined) return notFound;
@@ -122,6 +132,15 @@ export function makeGitLabHost({ apiUrl = "https://gitlab.com", fetchFn = fetch
 				redirect: "error",
 			});
 		} catch (err) {
+			// The one condition where the identity modules and this one now agree, because the issue's
+			// acceptance asks for exactly that (#316): a TLS trust failure, a protocol mismatch and a URL
+			// that always redirects are the operator's, and retrying them twice before failing tells
+			// nobody anything. Everything else here stays unconditionally retryable, which is right for a
+			// per-job call behind the queue: a wrong retry costs one more attempt, a wrong refusal costs
+			// the delivery and posts publicly that the deployment is misconfigured.
+			if (isDeterminateFetchFailure(err)) {
+				throw configError(`gitlab-host: POST ${path} failed (${fetchFailureReason(err)})`);
+			}
 			throw new InfraRetry(`gitlab-host: POST ${path} failed (${fetchFailureReason(err)})`);
 		}
 		if (!res.ok) {

package/src/gitlab-identity.mjs

@@ -16,6 +16,7 @@
  */
 
 import { configError } from "./config.mjs";
+import { isDeterminateFetchFailure, isJsonContentType, isTransientStatus, responseHeaderReader, transientError } from "./transient.mjs";
 
 /** Resolve the acting identity's integer user id from `GET /user`. `fetchFn` is injected for tests. */
 export async function resolveGitLabSelfId({ apiUrl, token, fetchFn = fetch }) {
@@ -27,17 +28,33 @@ export async function resolveGitLabSelfId({ apiUrl, token, fetchFn = fetch }) {
 	try {
 		res = await fetchFn(url, { headers: { "PRIVATE-TOKEN": token }, redirect: "error" });
 	} catch (err) {
-		throw configError(`gitlab identity: GET /user failed (${fetchFailureReason(err)})`);
+		// Unreachable is not misconfigured. The adjacent gitlab-host.mjs has classified this as
+		// InfraRetry since it was written; this file disagreed, and the disagreement decided whether
+		// the receiver ever came back from a forge restart (issue #316).
+		if (isDeterminateFetchFailure(err)) throw configError(`gitlab identity: GET /user failed (${fetchFailureReason(err)})`);
+		throw transientError(`gitlab identity: GET /user failed (${fetchFailureReason(err)})`, err);
 	}
 	if (!res.ok) {
 		// The status alone, never the body: an error body can echo the token back.
+		if (isTransientStatus(res.status, responseHeaderReader(res))) {
+			throw transientError(`gitlab identity: GET /user returned ${res.status}`);
+		}
 		throw configError(`gitlab identity: GET /user returned ${res.status}`);
 	}
 	let body;
 	try {
 		body = await res.json();
 	} catch (err) {
-		throw configError(`gitlab identity: GET /user returned unparseable JSON (${err?.message ?? "unknown"})`);
+		// A truncated body or a proxy interstitial, not a deployment an operator can fix.
+		// A body that will not parse is ambiguous in the expensive direction, so the CONTENT TYPE
+		// decides it. A truncated JSON body is transient. An HTML page is not: it is a Cloudflare
+		// Access or OIDC portal answering instead of the forge, and no amount of restarting gets past
+		// one (issue #316).
+		const contentType = responseHeaderReader(res);
+		if (contentType("content-type") !== undefined && !isJsonContentType(contentType)) {
+			throw configError(`gitlab identity: GET /user answered ${String(contentType("content-type"))} instead of JSON, so something other than the GitLab API replied`);
+		}
+		throw transientError(`gitlab identity: GET /user returned unparseable JSON (${err?.message ?? "unknown"})`, err);
 	}
 	if (!Number.isInteger(body?.id)) {
 		throw configError("gitlab identity: GET /user returned no integer id");

package/src/host-registry.mjs

@@ -135,6 +135,17 @@ export function makeHostRegistry({ redis, name, now = () => Date.now(), ttlMs =
 		// supersede lease `wait:key:<dedupId>` -- and only after checking the lease is still ours, which is
 		// the same ownership check this key gets for free by being named after its only writer.
 		await bounded(redis.pexpire(key, ttlMs), timeoutMs);
+		// RE-CHECKED HERE, AND ONLY HERE (issue #302). `close`'s drain is bounded at one `timeoutMs` while
+		// this function can spend one per command, so a write that outlived the drain would SADD the name
+		// back AFTER `close`'s SREM had removed it: the ghost the `inFlight` drain exists to prevent,
+		// arriving by a different door. Measured: `hset@0, pexpire@74, del@104, srem@105, sadd@145`.
+		//
+		// NOT before the PEXPIRE, which is the tempting symmetry and is wrong. A close landing between the
+		// HSET and the PEXPIRE would then leave `host:h:<name>` with NO EXPIRY AT ALL, where today it dies in
+		// ninety seconds -- invisible, because every reader walks `host:live` and the SREM emptied it, so a
+		// keyspace leak rather than a ghost peer, and the reader's own prune does not reach it. The SADD is
+		// the index write `close` undoes; the PEXPIRE is the row's own fuse and is always worth attempting.
+		if (closed) return;
 		await bounded(redis.sadd(HOST_SET, name), timeoutMs);
 	};
 
@@ -149,12 +160,22 @@ export function makeHostRegistry({ redis, name, now = () => Date.now(), ttlMs =
 		try {
 			inFlight = write({ ...facts, name, beatAt: now() });
 			await inFlight;
-			if (!reachable) {
+			// GATED ON `closed`, IN BOTH ARMS, and that is the whole of issue #302. `close` drains with ONE
+			// `bounded(inFlight, timeoutMs)` while `write` spends a fresh `timeoutMs` on each of its commands,
+			// so a write can outlive the drain -- and the continuation here then speaks through the boot's own
+			// `log` closure, stamped with a host that has stopped. Measured against a stalled Valkey:
+			// `host_registry_unreachable` at +76ms and `host_registry_restored` at +108ms after `close()` had
+			// RESOLVED. The restored arm is not the afterthought it looks like: a drain that gave up can see the
+			// write SUCCEED late just as easily as fail, and #302 named only the catch.
+			//
+			// This is `makeWatchCloser`'s `reloadLog` posture reached from the other side: work already in
+			// flight cannot be recalled, so what a close gates is its VOICE.
+			if (!reachable && !closed) {
 				reachable = true;
 				log("host_registry_restored", { host: name });
 			}
 		} catch (err) {
-			if (reachable) {
+			if (reachable && !closed) {
 				reachable = false;
 				log("host_registry_unreachable", { host: name, reason: err?.message });
 			}
@@ -194,9 +215,9 @@ export function makeHostRegistry({ redis, name, now = () => Date.now(), ttlMs =
 
 		/**
 		 * Start beating. ONE `setInterval` -- the first in `worker/src`, every other timer here being a
-		 * `setTimeout` -- and `.unref()`'d so it can never hold the process open, which is the posture the
-		 * three `fs.watch` watchers already take. `stop` is registered as an extraCloser beside the runtime
-		 * queue, so a clean shutdown clears it before `process.exit`.
+		 * `setTimeout` -- and `.unref()`'d so it can never hold the process open. `close` is registered as an
+		 * extraCloser beside the runtime queue, so a clean shutdown clears it before `process.exit`; the three
+		 * `fs.watch` watchers take the same two-part posture since issue #295, unref'd AND closed.
 		 */
 		async start(fields = {}, { intervalMs = HOST_BEAT_MS } = {}) {
 			if (closed || timer) return; // a second start would leak the first interval
@@ -218,6 +239,12 @@ export function makeHostRegistry({ redis, name, now = () => Date.now(), ttlMs =
 			timer = null;
 			// Drain before deleting, bounded like everything else here: an unbounded wait on a beat that is
 			// itself hung would be the shutdown hang this module's timeout exists to prevent.
+			//
+			// So the drain is BEST-EFFORT, and it deliberately stays at one `timeoutMs` (issue #302): `write`
+			// can spend one per command, so this can return while a write is still running, and widening the
+			// wait to match would triple a shutdown bound whose entire point is to be short. What makes that
+			// safe is the `closed` FLAG rather than the wait -- it gates the SADD inside `write` and both log
+			// arms in `beat`, so a write that outlives this drain can neither re-index the row nor say a word.
 			await bounded(inFlight ?? Promise.resolve(), timeoutMs).catch(() => {});
 			try {
 				await bounded(redis.del(hostKey(name)), timeoutMs);

package/src/identity.mjs

@@ -2,8 +2,10 @@
  * Resolve the acting GitHub identity's numeric `id` -- the value that appears in webhook
  * `sender.id`. The receiver's bot-loop guard compares an incoming `sender.id` against this to
  * refuse events the harness itself authored (an unbounded paid recursion otherwise). If the id
- * cannot be resolved the guard cannot arm, so this fails CLOSED: any failure throws a tagged
- * config error and the process must not boot.
+ * cannot be resolved the guard cannot arm, so this fails CLOSED: any failure throws and the
+ * process must not boot. WHICH failure it throws is what decides whether the process comes back
+ * (issue #316): a determinate fault is tagged and stays stopped, a transient one is untagged and
+ * is restarted.
  *
  * The `octokit` client is INJECTED, already authenticated by the caller (user token for pat/gh,
  * app-JWT for app). This module never constructs Octokit -- keeping it a pure, testable leaf, per
@@ -11,6 +13,7 @@
  */
 
 import { configError } from "./config.mjs";
+import { isDeterminateFetchFailure, isTransientStatus, octokitHeaderReader, transientError } from "./transient.mjs";
 
 /**
  * Resolve the acting identity's numeric user id from `auth = { source, octokit }`.
@@ -20,8 +23,24 @@ import { configError } from "./config.mjs";
  *   up in `sender.id` is `slug[bot]`, resolved via `GET /users/{username}`. The App id is a
  *   different number and would never match `sender.id`, so the two-step is required.
  *
- * Returns an integer id. Throws `configError` on unknown/missing source, missing octokit, any
- * octokit rejection, or a non-integer id.
+ * Returns an integer id. Throws `configError` on unknown/missing source, missing octokit, a determinate
+ * octokit refusal, or a non-integer id.
+ *
+ * A TRANSIENT octokit rejection throws UNTAGGED instead (issue #316). The tag is not decoration here: it
+ * is the difference between a service that comes back and one that does not. `receiver/src/cli.mjs` maps
+ * a tagged throw to `EXIT_POLICY` (2), which `RestartPreventExitStatus=2` and nssm's `AppExit 2 Exit`
+ * deliberately leave stopped; and on the worker, whose `cli.mjs` never sees this because `start.mjs`
+ * catches it best-effort, the tag decides whether that forge stays credential-less for the lifetime of
+ * the process, after which every job of that kind is publicly told the deployment is misconfigured. A
+ * `GET /user` that timed out is none of those things.
+ *
+ * TWO THINGS THE STATUS ALONE CANNOT TELL YOU, both of which have to be answered before it is consulted.
+ * `@octokit/request`'s fetch wrapper turns EVERY network rejection into `RequestError(message, 500)` and
+ * keeps the original as `cause`, so a private CA or a redirect arrives wearing a transient status and the
+ * determinate check has to run first. And an ABSENT status means the failure never reached the HTTP layer
+ * at all, because octokit always sets one: it came from local code, which in practice means signing the
+ * App JWT with a key that is not PKCS//8 -- the commonest App setup mistake there is, and one that no
+ * amount of restarting fixes.
  */
 export async function resolveSelfId(auth) {
 	const source = auth?.source;
@@ -47,6 +66,12 @@ export async function resolveSelfId(auth) {
 			id = user.id;
 		}
 	} catch (error) {
+		// Order is load-bearing; see the docblock. Trust and redirect faults are wearing a 500, and a
+		// status-less rejection never reached the network.
+		const status = typeof error?.status === "number" ? error.status : undefined;
+		if (!isDeterminateFetchFailure(error) && status !== undefined && isTransientStatus(status, octokitHeaderReader(error), error?.message)) {
+			throw transientError(`resolveSelfId: could not reach GitHub to resolve self identity: ${error.message}`, error);
+		}
 		throw configError(`resolveSelfId: could not resolve self identity: ${error.message}`);
 	}