@edgehero/pi-dispatch 0.1.0

package/src/import-pi.mjs

@@ -0,0 +1,451 @@
+/**
+ * `pi-dispatch import-pi` — stage a CREDENTIAL-FREE copy of the host's `pi` setup into a global overlay
+ * dir, so every job can reuse it (REQ-GLOBAL-PI-OVERLAY). Point `PI_GLOBAL_PI_DIR` at the result and the
+ * worker mounts it `:ro` into each container, layered UNDER each repo's own `.pi/`.
+ *
+ * The overlay must never carry a secret (CONST-TOKEN-SCOPED-PER-JOB): the provider key stays in the host's
+ * `auth.json` / env and reaches the container through the env allowlist, never a mounted file. So this
+ * command copies only the safe subset and REFUSES a `models.json` that embeds a literal key.
+ *
+ * Copied:   models.json (definitions only, sanitized), skills/, APPEND_SYSTEM.md, and extensions/ (verbatim;
+ *           the admin extension is hard-blocked). Extensions come along BY DEFAULT -- staging is the vetting
+ *           step, and an overlay missing the operator's own extensions is not the setup they asked for --
+ *           with `--no-extensions` as the escape hatch. Every extension staged is PRINTED by name, because
+ *           this is the moment the operator can still see what is about to run inside every job container.
+ * Staged:   packages/ — only under --with-packages — pinned third-party pi packages, installed here on the
+ *           host so a job container can load them from the overlay with NO network access (issue #58).
+ * Never:    auth.json, settings.json, sessions/, themes/, prompts/, tools/.
+ */
+import { existsSync, readFileSync, writeFileSync, mkdirSync, readdirSync, statSync, copyFileSync, renameSync, rmSync } from "node:fs";
+import { execFile } from "node:child_process";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { promisify } from "node:util";
+import { PACKAGES_SUBDIR, STAGE_MANIFEST, parsePackagesFile } from "./packages.mjs";
+
+/** A valid skill/extension entry name: lowercase kebab/underscore, no dots (no "..") and no slashes. */
+export const ENTRY_NAME_RE = /^[a-z0-9](?:[a-z0-9_.-]{0,62}[a-z0-9])?$/i;
+/** The admin extension — never duplicated into a job overlay (it can enqueue paid jobs: a recursion vector). */
+export const ADMIN_RE = /pi-dispatch|dispatch-admin/i;
+
+/** The pi resource kinds a package may contribute by convention dir, when it carries no `pi` manifest. */
+const RESOURCE_DIRS = ["extensions", "skills", "prompts", "themes"];
+
+const execFileAsync = promisify(execFile);
+
+/**
+ * The default package-stager runner. ARRAY argv, never a shell string -- a package name from a config file
+ * must never be able to become shell syntax on the operator's host. See `npmExecOptions` for the one
+ * platform on which `shell: true` is nonetheless unavoidable, and why it is safe there.
+ */
+function defaultExec(file, args, options) {
+	return execFileAsync(file, args, options);
+}
+
+// Resolve the host's pi agent dir the way pi's getAgentDir() does (env override, else ~/.pi/agent).
+// Custom: the worker CLI depends on @earendil-works/pi-ai/compat, not the whole pi-coding-agent SDK;
+// importing the SDK just to read one well-known path is not worth the weight.
+function defaultFrom(env) {
+	return env.PI_CODING_AGENT_DIR || join(homedir(), ".pi", "agent");
+}
+
+/** A config value that defers to the environment/a command rather than embedding a literal secret. */
+function isIndirection(v) {
+	return typeof v === "string" && (v.startsWith("$") || v.startsWith("!"));
+}
+
+/**
+ * Find a literal secret in a parsed models.json. Returns a human-readable location, or null if clean.
+ * A provider `apiKey`, or an auth-ish header, that is a plain string (not `$ENV` / `!cmd`) is a literal.
+ */
+export function findLiteralSecret(models) {
+	const providers = models?.providers;
+	if (!providers || typeof providers !== "object") return null;
+	for (const [name, cfg] of Object.entries(providers)) {
+		if (typeof cfg?.apiKey === "string" && !isIndirection(cfg.apiKey)) return `providers.${name}.apiKey`;
+		const headers = cfg?.headers;
+		if (headers && typeof headers === "object") {
+			for (const [h, val] of Object.entries(headers)) {
+				if (/auth|api[-_]?key|token|secret|bearer/i.test(h) && typeof val === "string" && !isIndirection(val)) {
+					return `providers.${name}.headers.${h}`;
+				}
+			}
+		}
+	}
+	return null;
+}
+
+export async function runImportPi(argv = [], deps = {}) {
+	const {
+		env = process.env,
+		cwd = process.cwd(),
+		out = (s) => process.stdout.write(s),
+		fs = { existsSync, readFileSync, writeFileSync, mkdirSync, readdirSync, statSync, copyFileSync, renameSync, rmSync },
+		exec = defaultExec,
+		// Injected like `fs`/`exec`/`out` so the win32 npm branch below is reachable from a test on any host;
+		// it is the branch that was dead on arrival precisely because nothing could exercise it here.
+		platform = process.platform,
+	} = deps;
+
+	// Extensions are copied unless the operator says otherwise. `--with-extensions` is still accepted and is
+	// now a no-op, so an existing setup script keeps working and keeps meaning what it always meant.
+	const withExtensions = !argv.includes("--no-extensions");
+	const withPackages = argv.includes("--with-packages");
+	const from = flagValue(argv, "--from") ?? defaultFrom(env);
+	const to = flagValue(argv, "--to") ?? join(cwd, "pi-global");
+	const packagesFile = flagValue(argv, "--packages-file") ?? env.PI_PACKAGES_FILE ?? join(cwd, "pi-packages.json");
+
+	if (!fs.existsSync(from)) {
+		out(`error: no pi setup found at ${from}\n  Is pi installed and configured? Set PI_CODING_AGENT_DIR or pass --from <dir>.\n`);
+		return 1;
+	}
+
+	// Pre-flight the one hard security gate BEFORE writing anything: a literal key in models.json aborts the
+	// whole import so no half-overlay is produced and no secret is ever written to the overlay.
+	const modelsSrc = join(from, "models.json");
+	let modelsText;
+	if (fs.existsSync(modelsSrc)) {
+		modelsText = fs.readFileSync(modelsSrc, "utf8");
+		let parsed;
+		try {
+			parsed = JSON.parse(modelsText);
+		} catch {
+			modelsText = null; // malformed: skip it with a warning rather than abort the whole import
+		}
+		if (parsed) {
+			const leak = findLiteralSecret(parsed);
+			if (leak) {
+				out(
+					`error: ${modelsSrc} embeds a literal secret at ${leak}.\n` +
+						`  The overlay is mounted into an adversarial-input container, so it must be credential-free.\n` +
+						`  Move the key to auth.json (\`pi\` login) or reference the environment (e.g. "$MY_KEY"), then re-run.\n`,
+				);
+				return 1;
+			}
+		}
+	}
+
+	const results = [];
+	fs.mkdirSync(to, { recursive: true });
+
+	// models.json — definitions only, already proven literal-secret-free above.
+	if (modelsText) {
+		fs.writeFileSync(join(to, "models.json"), modelsText);
+		results.push(["models.json", "custom model/provider definitions (credential-free)"]);
+	} else if (fs.existsSync(modelsSrc)) {
+		results.push(["models.json", "SKIPPED — not valid JSON"]);
+	}
+
+	// skills/ — copy each named skill dir (SKILL.md + its files), skipping symlinks and odd names.
+	const skillsCount = copyNamedDirs(fs, join(from, "skills"), join(to, "skills"), out);
+	if (skillsCount > 0) results.push(["skills/", `${skillsCount} skill${skillsCount === 1 ? "" : "s"}`]);
+
+	// APPEND_SYSTEM.md — the operator's global persona.
+	if (fs.existsSync(join(from, "APPEND_SYSTEM.md"))) {
+		fs.copyFileSync(join(from, "APPEND_SYSTEM.md"), join(to, "APPEND_SYSTEM.md"));
+		results.push(["APPEND_SYSTEM.md", "global persona (layers under each repo's persona)"]);
+	}
+
+	// extensions/ -- the sharp edge, and copied BY DEFAULT: the operator staged this overlay to be their own
+	// setup, and one they have to arm twice more is not that. The admin extension stays hard-blocked (it can
+	// enqueue paid jobs from inside a container: a recursion vector, not part of this relaxation).
+	//
+	// Every copied name is listed, not just counted. What lands here runs in every job container from the
+	// next job onward, so the operator gets ONE moment to read the actual names -- a bare "3 extensions" row
+	// would leave them re-deriving the list from a directory they cannot see from the worker host.
+	const extSrc = join(from, "extensions");
+	if (withExtensions && fs.existsSync(extSrc)) {
+		const { copied, blocked } = copyExtensions(fs, extSrc, join(to, "extensions"), out);
+		if (copied.length > 0) {
+			results.push(["extensions/", `${copied.length} extension${copied.length === 1 ? "" : "s"} -- these LOAD in every job; VET THESE`]);
+			// Note-less rows: the printer emits them bare, so the names read as a list under the count above.
+			for (const name of copied) results.push([`  - ${name}`, ""]);
+		}
+		for (const name of blocked) out(`  blocked extension "${name}" — the admin extension must never run inside a job.\n`);
+		out(
+			"\n⚠ Extensions run code against adversarial input with open network egress and are NOT scanned for\n" +
+				"  secrets. They load in every job as staged -- review every one listed below, and set\n" +
+				"  PI_GLOBAL_ALLOW_EXTENSIONS=0 in .env if you need them off.\n",
+		);
+	} else if (fs.existsSync(extSrc)) {
+		results.push(["extensions/", "skipped -- --no-extensions was passed (nothing from extensions/ reaches a job)"]);
+	}
+
+	// packages/ — pinned third-party pi packages, staged from npm on THIS host so the job container never
+	// needs the network (issue #58). All-or-nothing: a failure leaves no half-staged set to load.
+	if (withPackages) {
+		const staged = await stagePackages({ fs, exec, out, packagesFile, to, platform });
+		if (staged.error) {
+			out(`error: ${staged.error}\n`);
+			return 1;
+		}
+		const n = staged.packages.length;
+		results.push(["packages/", `${n} package${n === 1 ? "" : "s"} -- third-party code, VET THESE`]);
+		for (const warn of staged.warnings) results.push([`packages/${warn.dir}`, `WARN: ${warn.reason}`]);
+	} else if (fs.existsSync(join(to, PACKAGES_SUBDIR))) {
+		results.push(["packages/", "kept -- re-run with --with-packages to refresh"]);
+	}
+
+	out(`Imported the credential-free subset of ${from} → ${to}\n\n`);
+	// A note-less row is a list item under the row above it (the extension names), so it prints bare rather
+	// than padded out to a column that has nothing to hold.
+	for (const [name, note] of results) out(note ? `  ${name.padEnd(18)} ${note}\n` : `  ${name}\n`);
+	out(`\n  (auth.json, settings.json, sessions/ are never copied — your credential stays in env/auth.json.)\n`);
+	out(nextSteps(to, withExtensions, withPackages));
+	return 0;
+}
+
+/** Copy `<src>/<name>/**` for each valid, non-symlink child dir. Returns the count copied. */
+function copyNamedDirs(fs, src, dst, out) {
+	if (!fs.existsSync(src)) return 0;
+	let n = 0;
+	for (const name of fs.readdirSync(src)) {
+		if (!ENTRY_NAME_RE.test(name)) {
+			out(`  skipped "${name}" — unexpected name\n`);
+			continue;
+		}
+		const childSrc = join(src, name);
+		if (fs.statSync(childSrc).isSymbolicLink?.() || !fs.statSync(childSrc).isDirectory()) continue;
+		copyTree(fs, childSrc, join(dst, name));
+		n++;
+	}
+	return n;
+}
+
+/**
+ * Like copyNamedDirs but reports the admin extension it refuses to copy. Returns the NAMES copied, not a
+ * count: the caller prints them, so the operator sees exactly what is now going into job containers.
+ */
+function copyExtensions(fs, src, dst, out) {
+	const copied = [];
+	const blocked = [];
+	for (const name of fs.readdirSync(src)) {
+		if (ADMIN_RE.test(name)) {
+			blocked.push(name);
+			continue;
+		}
+		if (!ENTRY_NAME_RE.test(name)) {
+			out(`  skipped "${name}" — unexpected name\n`);
+			continue;
+		}
+		const childSrc = join(src, name);
+		const st = fs.statSync(childSrc);
+		if (st.isSymbolicLink?.()) continue;
+		if (st.isDirectory()) copyTree(fs, childSrc, join(dst, name));
+		else fs.copyFileSync(childSrc, join(dst, name));
+		copied.push(name);
+	}
+	return { copied, blocked };
+}
+
+/** Recursively copy a directory tree, skipping symlinks (a symlink could point outside the source). */
+function copyTree(fs, src, dst) {
+	fs.mkdirSync(dst, { recursive: true });
+	for (const entry of fs.readdirSync(src)) {
+		const s = join(src, entry);
+		const st = fs.statSync(s);
+		if (st.isSymbolicLink?.()) continue;
+		if (st.isDirectory()) copyTree(fs, s, join(dst, entry));
+		else fs.copyFileSync(s, join(dst, entry));
+	}
+}
+
+/**
+ * Stage every package pinned in `packagesFile` into `<to>/packages/<dir>` and write the stage manifest.
+ * Returns `{ packages, warnings }`, or `{ error }` -- ALL-OR-NOTHING, because a half-staged set is worse
+ * than none: pi would load the packages that made it and silently skip the rest (issue #58).
+ *
+ * Each package is installed into a private `.staging-<i>` dir, asserted there, and only renamed into place
+ * once EVERY package has passed. A staged dir must be SELF-CONTAINED (`package.json` + its own
+ * `node_modules/`) because at job time it is resolved from a read-only mount with no network and no
+ * install step -- so every assertion below is about that property.
+ */
+async function stagePackages({ fs, exec, out, packagesFile, to, platform = process.platform }) {
+	if (!fs.existsSync(packagesFile)) {
+		return { error: `--with-packages needs a packages file, none at ${packagesFile}\n  Run \`pi-dispatch init\` to scaffold one, or pass --packages-file <path>.` };
+	}
+
+	let entries;
+	try {
+		entries = parsePackagesFile(fs.readFileSync(packagesFile, "utf8"), packagesFile);
+	} catch (error) {
+		// Refused before a single directory is created, so a bad file stages nothing at all.
+		return { error: error.message };
+	}
+
+	const packagesRoot = join(to, PACKAGES_SUBDIR);
+	const rootExisted = fs.existsSync(packagesRoot);
+	fs.mkdirSync(packagesRoot, { recursive: true });
+
+	const npmBin = platform === "win32" ? "npm.cmd" : "npm";
+	const stagingDirs = [];
+	const prepared = [];
+	const renamed = [];
+	const warnings = [];
+
+	try {
+		for (const [index, entry] of entries.entries()) {
+			const staging = join(packagesRoot, `.staging-${index}`);
+			fs.rmSync(staging, { recursive: true, force: true }); // a crashed earlier run may have left one
+			stagingDirs.push(staging);
+			fs.mkdirSync(staging, { recursive: true });
+			// A private root package.json pins npm's idea of "the project" to the staging dir, so it cannot
+			// walk up and install into (or read config from) the operator's own checkout.
+			fs.writeFileSync(join(staging, "package.json"), `${JSON.stringify({ name: "pi-dispatch-staging", private: true }, null, 2)}\n`);
+
+			// ARRAY argv, never a shell string: the name and version come from a config file and must never be
+			// able to become shell syntax on the operator's host. The install target is the exec's `cwd`, NOT a
+			// `--prefix <staging>` pair -- npm installs into the cwd's node_modules by default, and dropping the
+			// flag removes the only filesystem PATH from argv. What is left is nothing but literal flags and one
+			// `name@version` token already validated against NPM_NAME_RE + EXACT_VERSION_RE; that is the property
+			// npmExecOptions relies on below.
+			//
+			// --ignore-scripts is load-bearing: without it the lifecycle scripts of this package AND of every
+			// transitive dependency would run as the operator, on the operator's host, at stage time.
+			// --omit=peer because pi aliases its own packages for extensions at load time, so a staged peer
+			// copy is ignored dead weight -- and a floating pi version at that (CONST-PI-VERSION-PINNED).
+			// --install-strategy=nested asks npm to keep every dependency inside the package dir; step 4 below
+			// ASSERTS the result rather than trusting the flag, whose name and default have moved across npm
+			// versions.
+			const args = [
+				"install",
+				`${entry.name}@${entry.version}`,
+				"--omit=dev",
+				"--omit=peer",
+				"--omit=optional",
+				"--ignore-scripts",
+				"--install-strategy=nested",
+				"--no-audit",
+				"--no-fund",
+				"--loglevel=error",
+			];
+			out(`  staging ${entry.name}@${entry.version} -> packages/${entry.dir}\n`);
+			try {
+				await exec(npmBin, args, npmExecOptions(platform, staging));
+			} catch (error) {
+				const detail = String(error?.stderr ?? error?.message ?? "").trim();
+				throw new Error(`npm install failed for ${entry.name}@${entry.version}: ${detail}`);
+			}
+
+			const source = join(staging, "node_modules", entry.name);
+			let pkg;
+			try {
+				pkg = JSON.parse(fs.readFileSync(join(source, "package.json"), "utf8"));
+			} catch {
+				throw new Error(`${entry.name}@${entry.version}: npm reported success but there is no readable package.json at ${join(source, "package.json")}`);
+			}
+			if (pkg.version !== entry.version) {
+				throw new Error(`${entry.name}: npm staged version ${JSON.stringify(pkg.version)}, not the pinned ${JSON.stringify(entry.version)} (CONST-PI-VERSION-PINNED)`);
+			}
+
+			// Dependency completeness -- catches hoisting whatever npm's flag defaults do this month. A
+			// hoisted dependency would only surface as an import failure inside a job, hours later.
+			for (const dep of Object.keys(pkg.dependencies ?? {})) {
+				if (!fs.existsSync(join(source, "node_modules", dep))) {
+					throw new Error(`${entry.name}: dependency "${dep}" is not inside the package dir -- npm hoisted it out, so the staged copy could not import it at run time (no network, no install)`);
+				}
+			}
+
+			// A package that contributes no pi resources loads as a silent no-op; staging exists to turn that
+			// run-time nothing into a stage-time error the operator can act on.
+			const manifest = pkg.pi !== null && typeof pkg.pi === "object" ? pkg.pi : null;
+			const hasResourceDir = RESOURCE_DIRS.some((name) => fs.existsSync(join(source, name)));
+			if (!manifest && !hasResourceDir) {
+				throw new Error(`${entry.name} is not a pi package -- no "pi" manifest in package.json and none of ${RESOURCE_DIRS.join("/")}; it would load as a silent no-op`);
+			}
+
+			// Containment: manifest entries are resolved relative to the package dir at job time, so one that
+			// climbs out of it would reach the rest of the read-only overlay.
+			const escaping = manifest && findEscapingEntry(manifest);
+			if (escaping) {
+				throw new Error(`${entry.name}: pi manifest entry ${JSON.stringify(escaping)} leaves the package dir (no ".." segment, no leading "/")`);
+			}
+
+			// Warn, do not refuse: --ignore-scripts means a build/postinstall step did NOT run and an optional
+			// dependency was NOT fetched, so such a package is staged INCOMPLETE and may fail at run time.
+			const scriptKeys = ["install", "preinstall", "postinstall"].filter((key) => typeof pkg.scripts?.[key] === "string");
+			const hasOptional = Object.keys(pkg.optionalDependencies ?? {}).length > 0;
+			if (scriptKeys.length > 0 || hasOptional) {
+				const declares = [...scriptKeys.map((key) => `scripts.${key}`), ...(hasOptional ? ["optionalDependencies"] : [])].join(", ");
+				warnings.push({ dir: entry.dir, reason: `${entry.name} declares ${declares} -- staged with --ignore-scripts, so it is INCOMPLETE and may fail at run time` });
+			}
+
+			prepared.push({ entry, source });
+		}
+
+		// Renames happen only after EVERY package has passed, so a failure on the last one cannot leave the
+		// earlier ones swapped in beside a stale manifest.
+		for (const { entry, source } of prepared) {
+			const dest = join(packagesRoot, entry.dir);
+			fs.rmSync(dest, { recursive: true, force: true }); // replace a previous stage of the same package
+			// renameSync, never copyTree: copyTree's symlink guard uses statSync, which FOLLOWS links, so it
+			// would copy the target of every node_modules/.bin symlink instead of skipping it.
+			fs.renameSync(source, dest);
+			renamed.push(dest);
+		}
+	} catch (error) {
+		for (const dest of renamed) fs.rmSync(dest, { recursive: true, force: true });
+		for (const staging of stagingDirs) fs.rmSync(staging, { recursive: true, force: true });
+		if (!rootExisted) fs.rmSync(packagesRoot, { recursive: true, force: true });
+		return { error: `${error.message}\n  Nothing was staged -- fix ${packagesFile} (or the package) and re-run with --with-packages.` };
+	}
+
+	for (const staging of stagingDirs) fs.rmSync(staging, { recursive: true, force: true });
+
+	const stageManifest = { stagedAt: new Date().toISOString(), packages: entries.map(({ name, version, dir }) => ({ name, version, dir })) };
+	fs.writeFileSync(join(packagesRoot, STAGE_MANIFEST), `${JSON.stringify(stageManifest, null, 2)}\n`);
+	return { packages: entries, warnings };
+}
+
+/**
+ * The exec options for one `npm install`: always `cwd: <staging>` (that IS the install target now that
+ * `--prefix` is gone), plus `shell: true` on win32 and nowhere else.
+ *
+ * WHY shell:true is REQUIRED on win32: npm ships there as `npm.cmd`, and since Node 18.20.2 / 20.12.2
+ * (CVE-2024-27980) spawning a `.cmd`/`.bat` WITHOUT a shell throws EINVAL outright. This package floors at
+ * Node >=22.19, so every Node it can run on has that behaviour -- without this, `--with-packages` fails on
+ * every Windows host with a misleading "spawn npm.cmd EINVAL" and the branch above is dead on arrival.
+ *
+ * WHY shell:true is SAFE HERE SPECIFICALLY, which it would NOT be in general: with `--prefix` replaced by
+ * `cwd`, argv holds no filesystem path at all -- only literal flags this file spells out, plus the single
+ * `name@version` token, and BOTH halves of that token were validated before anything was created (packages.mjs
+ * rejects any name failing NPM_NAME_RE and any version failing EXACT_VERSION_RE, neither of which admits a
+ * space, quote, or cmd metacharacter). So no operator-supplied string that could survive as shell syntax ever
+ * reaches the command line. Re-introducing a path -- or loosening either regex -- breaks that argument, so
+ * this option must be revisited together with them.
+ */
+function npmExecOptions(platform, staging) {
+	return platform === "win32" ? { cwd: staging, shell: true } : { cwd: staging };
+}
+
+/** The first string anywhere in a `pi` manifest that leaves the package dir, or null when all are contained. */
+function findEscapingEntry(value) {
+	if (typeof value === "string") {
+		return /^[\\/]/.test(value) || value.split(/[\\/]/).includes("..") ? value : null;
+	}
+	if (Array.isArray(value) || (value !== null && typeof value === "object")) {
+		for (const child of Object.values(value)) {
+			const hit = findEscapingEntry(child);
+			if (hit) return hit;
+		}
+	}
+	return null;
+}
+
+function flagValue(argv, flag) {
+	const i = argv.indexOf(flag);
+	return i >= 0 && i + 1 < argv.length ? argv[i + 1] : undefined;
+}
+
+function nextSteps(to, withExtensions, withPackages) {
+	const steps = [`Set PI_GLOBAL_PI_DIR=${to} in .env`, "pi-dispatch doctor        # verifies the overlay is credential-free"];
+	// The vetting step is no longer a switch to flip -- it already happened by staging. What is left is the
+	// off switch, named here so an operator who does not want the extensions is not left hunting for it.
+	if (withExtensions) steps.push("Vet the extensions listed above -- they load in every job; set PI_GLOBAL_ALLOW_EXTENSIONS=0 in .env to disable them");
+	// Same inversion for packages: staging is what loads them, so the step worth naming is how to withhold
+	// them from a trigger that should not run third-party code.
+	if (withPackages) steps.push('Staged packages load in every job -- set `run.packages: false` on any trigger in triggers.json that must not load them');
+	return `
+Next:
+${steps.map((step, i) => `  ${i + 1}. ${step}\n`).join("")}`;
+}

package/src/index.mjs

@@ -0,0 +1,177 @@
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { DelayedError, UnrecoverableError, Worker } from "bullmq";
+import { InfraRetry, runJob } from "./processor.mjs";
+
+const exec = promisify(execFile);
+
+export const QUEUE = "pi-jobs";
+export const JOB_TIMEOUT_MS = 30 * 60 * 1000; // REQ-JOB-TIMEOUT-30M
+
+/**
+ * Build the BullMQ processor.
+ *
+ * It MUST declare exactly three parameters (job, token, signal). BullMQ only allocates an
+ * AbortController when `processor.length >= 3` (it inspects the function's arity at construction),
+ * so dropping the unused `token` would silently disable BOTH the 30-minute timeout and the shutdown
+ * abort -- with no error. A test asserts the arity precisely because the failure is silent.
+ *
+ * Dependencies are injected so this is testable without a live queue: `cancelJob` (fired by the
+ * timeout), `stopContainer` (fired by the abort), and the orchestration deps.
+ *
+ * Once per job, before runJob, it resolves the runtime-settings overlay via `getSettings`
+ * (INT-CONFIG-OVERLAY-CONTRACT). A present-but-invalid overlay resolves to a POLICY refusal RETURNED
+ * (never thrown), so BullMQ marks the job completed without a retry (CONST-RETRY-INFRA-ONLY). A valid
+ * overlay fills the effective `provider`/`model`/`maxTurns`/`dailyCap`/`weeklyCap`/`monthlyCap`/`softHoldPct`
+ * under `job.data > overlay > env` precedence and re-binds the worker slot count via `applyConcurrency`.
+ * The overlay changes which values the spend caps take, never when they are checked -- reserveBudget still
+ * runs inside runJob against the freshly passed caps (CONST-BUDGET-BEFORE-TOKENS).
+ */
+export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, applyConcurrency = () => {}, pauseUntil = () => null, deps, recordRun = () => {}, timeoutMs = JOB_TIMEOUT_MS, now = () => Date.now() }) {
+	return async function processor(job, token, signal) {
+		// Scoped pause windows (REQ-SCOPED-PAUSE-WINDOWS): if this job's folder/repo is inside an active pause
+		// window, DEFER it to the window end via BullMQ's delayed set -- the job keeps its identity/dedup and
+		// auto-resumes when re-picked. This is FIRST, before the kill timer, the settings read, and the budget
+		// reservation, so a deferred job arms no timer, reserves no slot, and spends nothing
+		// (CONST-BUDGET-BEFORE-TOKENS). `moveToDelayed` needs the worker's `token`; the `> now + 1s` guard keeps
+		// a boundary tick from busy-deferring. A thrown `DelayedError` is how BullMQ learns the job was deferred
+		// (worker.js recognises it) rather than completed or failed.
+		// One clock snapshot for both the window lookup and the guard, so they cannot disagree across the
+		// call (and a test can inject a fixed clock). `now` defaults to the real wall clock in production.
+		const nowMs = now();
+		const until = pauseUntil(job.data, nowMs);
+		if (until && until > nowMs + 1000) {
+			await job.moveToDelayed(until, token);
+			throw new DelayedError();
+		}
+
+		const startedAt = new Date().toISOString();
+		const name = `pi-job-${job.id}`;
+		const timer = setTimeout(() => {
+			// BullMQ has no per-job kill timer; this is ours. cancelJob raises the AbortSignal.
+			Promise.resolve(cancelJob(job.id, "job-timeout-30m")).catch(() => {});
+		}, timeoutMs);
+
+		// Abort (timeout OR shutdown) => stop the container. docker stop sends SIGTERM then SIGKILL
+		// after the grace period; the runner exits and runContainer returns/throws.
+		const onAbort = () => {
+			Promise.resolve(stopContainer(name)).catch(() => {});
+		};
+		signal.addEventListener("abort", onAbort, { once: true });
+
+		try {
+			const settings = await getSettings();
+			if (settings.invalid) {
+				// A present-but-invalid overlay is a POLICY refusal, RETURNED (never thrown) so BullMQ marks the
+				// job completed and does not retry a file that can never parse (CONST-RETRY-INFRA-ONLY). Resolved
+				// before runJob, so no budget slot is reserved and no container starts (CONST-BUDGET-BEFORE-TOKENS).
+				// recordRun leaves the durable settings-overlay-invalid trace for the admin extension.
+				// No provider/model here, alone among the terminal results: the overlay is the thing that failed to parse, so no honest effective value exists -- buildRecord defaults both null.
+				const result = { outcome: "policy", reason: "settings-overlay-invalid", exitCode: null, turns: null, tokens: null, budgetReserved: false };
+				recordRun({ job, result, startedAt, endedAt: new Date().toISOString() });
+				return result;
+			}
+
+			// Re-bind the worker slot count to the effective concurrency before the run (no-op default when
+			// unwired, e.g. a bare makeProcessor under test).
+			applyConcurrency(settings.concurrency);
+
+			// Fill the effective job settings under `job.data > overlay > env` precedence: an explicit per-job
+			// field wins; an omitted one takes the overlay value, else env, resolved at this job's start
+			// (INT-CONFIG-OVERLAY-CONTRACT). Receiver GitHub jobs carry no provider/model/maxTurns, so this fill
+			// supplies the provider the container env allowlist requires -- absent it, the allowlist refuses a job
+			// only after its budget slot is reserved. The `caps`/`softHoldPct` passed to runJob change which
+			// values reserveBudget checks, never when it runs.
+			const effectiveJob = {
+				...job.data,
+				provider: job.data.provider ?? settings.provider,
+				model: job.data.model ?? settings.model,
+				maxTurns: job.data.maxTurns ?? settings.maxTurns,
+				maxTokens: job.data.maxTokens ?? settings.maxTokens, // optional per-job token budget (issue #25); null => runner meter only
+			};
+
+			const result = await runJob(effectiveJob, {
+				redis,
+				// The three spend windows (week/month null when disabled) and the soft-hold band, resolved this
+				// job-start under overlay > env. reserveBudget checks them before the container.
+				caps: { day: settings.dailyCap, week: settings.weeklyCap, month: settings.monthlyCap },
+				softHoldPct: settings.softHoldPct,
+				// The daily TOKEN cap (issue #25), same overlay > env resolution. Check-AFTER, so it gates the
+				// NEXT job on prior recorded spend; null => the daily token counter is disabled.
+				tokenCap: settings.dailyTokenCap,
+				...deps,
+				runContainer: (ctx) => deps.runContainer({ ...ctx, name, signal }),
+				// collectChain (INT-OUTBOX-CONTRACT) reads the completed parent's REAL BullMQ job: its `.id`
+				// (the parent id children carry) and `.data` (kind/chainDepth). runJob's own `job` is the
+				// effectiveJob -- a spread of job.data with no `.id`/`.data` -- so inject the real wrapper here,
+				// mirroring the name/signal injection above. Omitted when unwired so a bare processor falls back
+				// to runJob's no-op default (a chain fault can never flip a completed outcome either way).
+				...(deps.collectChain ? { collectChain: (ctx) => deps.collectChain({ ...ctx, job }) } : {}),
+				// prepareWorkspace needs the REAL BullMQ job's `.id` to derive a cron job's scheduled-for
+				// instant from the deterministic repeat:<id>:<millis> jobId (DES-CRON-VIA-BULLMQ-SCHEDULER)
+				// for the local /job/event.json. runJob's own `job` is the effectiveJob -- a spread of
+				// job.data with no `.id` -- and the real wrapper is only in scope here, so inject it as
+				// `queueJobId`, mirroring the collectChain injection above. Omitted when unwired so a bare
+				// processor keeps runJob's plain (job, token) call.
+				...(deps.prepareWorkspace ? { prepareWorkspace: (j, t) => deps.prepareWorkspace(j, t, { queueJobId: job.id }) } : {}),
+			});
+			recordRun({ job, result, startedAt, endedAt: new Date().toISOString() });
+			return result;
+		} catch (error) {
+			recordRun({ job, error, startedAt, endedAt: new Date().toISOString() });
+			if (error instanceof InfraRetry) throw error; // retryable: BullMQ retries per attempts
+			// A non-retryable, non-infra error (our bug) must not retry forever. UnrecoverableError
+			// records it as failed-and-distinct in the queue's failed set without a retry.
+			throw new UnrecoverableError(error.message);
+		} finally {
+			clearTimeout(timer);
+			signal.removeEventListener("abort", onAbort);
+		}
+	};
+}
+
+export function createWorker({ connection, concurrency, getSettings, redis, deps, recordRun, limiter, pauseUntil, extraClosers = [] }) {
+	let worker; // referenced by cancelJob/applyConcurrency before assignment; only called later, so the TDZ is fine
+	const processor = makeProcessor({
+		cancelJob: (id, reason) => worker.cancelJob(id, reason),
+		stopContainer: (name) => exec("docker", ["stop", "-t", "5", name]),
+		redis,
+		getSettings,
+		// Late-bound over `worker`: an overlay concurrency change re-binds the live slot count at the next
+		// job start. Guarded so only an integer that actually differs touches the property.
+		applyConcurrency: (n) => {
+			if (Number.isInteger(n) && worker.concurrency !== n) worker.concurrency = n;
+		},
+		pauseUntil,
+		deps,
+		recordRun,
+	});
+
+	worker = new Worker(QUEUE, processor, {
+		// maxRetriesPerRequest: null is REQUIRED for BullMQ's blocking connections, or it throws.
+		connection: { ...connection, maxRetriesPerRequest: null },
+		concurrency,
+		maxStalledCount: 0, // a stalled paid job FAILS, never silently re-runs (verified live)
+		...(limiter ? { limiter } : {}),
+	});
+
+	const shutdown = async () => {
+		// Abort active jobs (=> docker stop via onAbort), then close. Without the cancel,
+		// worker.close() would wait up to 30 minutes for the container.
+		await Promise.resolve(worker.cancelAllJobs?.("shutdown")).catch(() => {});
+		await worker.close();
+		// Close auxiliary resources (e.g. a cron scheduler) after the worker drains. Per-item catch
+		// so one failing or absent closer never strands the others or blocks exit -- matches the
+		// swallow posture on cancelAllJobs above.
+		await Promise.all(extraClosers.map((c) => Promise.resolve(c.close?.()).catch(() => {})));
+		process.exit(0);
+	};
+	process.once("SIGTERM", shutdown);
+	process.once("SIGINT", shutdown);
+	// Windows never delivers an external SIGTERM. nssm's console-stop delivers Ctrl-C => SIGINT
+	// (handled above); SIGBREAK covers console-close. Route it to the same shutdown so a stopped
+	// worker still aborts in-flight jobs and docker-stops their containers rather than orphaning them.
+	if (process.platform === "win32") process.once("SIGBREAK", shutdown);
+
+	return worker;
+}