@edgehero/pi-dispatch 4.3.1 → 5.0.0

package/src/scheduler-stall-guard.mjs

@@ -13,6 +13,8 @@
  * Custom: per-scheduler stall accounting; BullMQ's maxStalledCount does not cover scheduler jobs -- CONST-RETRY-INFRA-ONLY carve-out ("BullMQ will never do this for us")
  */
 
+import { cronRunOf, isSchedulerJobId } from "./bullmq-internals.mjs";
+
 // The prefix every stall counter lives under, so `KEYS pi-dispatch:sched-stalls*` still shows an operator
 // the whole feature -- the affordance `wait:`, `slot:` and `budget:` all assume. Exported alongside the
 // builder so the admin panel and the integration teardown compose keys through one definition and cannot
@@ -36,8 +38,8 @@ export const stallKey = (schedulerId) => `${STALL_KEY}:${schedulerId}`;
 // stalls, opposite outcome, decided by an unrelated trigger (issue #267).
 //
 // Per-field TTLs would have fixed it in place and are not available: `HEXPIRE` does not exist on the pinned
-// `valkey/valkey:8` (verified, `ERR unknown command`, recorded under DES-HOST-REGISTRY). A key per entity is
-// the only shape that gets per-entity expiry.
+// `valkey/valkey:8.1` (`VALKEY_IMAGE`; verified on 8.1.11, `ERR unknown command`, recorded under DES-HOST-REGISTRY).
+// A key per entity is the only shape that gets per-entity expiry.
 //
 // The EXPIRE still ROLLS on every stall, deliberately, and that is not `budget.mjs`'s set-once rule being
 // broken. A budget window is a CALENDAR window and must not be pushed forward by traffic or a busy day
@@ -59,16 +61,20 @@ const STALL_WINDOW_SECONDS = 24 * 60 * 60;
 export function makeStallGuard({ redis, threshold, removeJobScheduler, log }) {
 	return async function onStalled(jobId) {
 		try {
-			// Ordinary jobs are bounded by `maxStalledCount: 0`; only scheduler jobs reach this accounting.
-			if (!jobId.startsWith("repeat:")) return;
+			// Ordinary jobs are bounded by `maxStalledCount: 0`; only scheduler jobs reach this accounting. A hand-fired
+			// `manual:` run is an ordinary job, so it is not counted here either.
+			if (!isSchedulerJobId(jobId)) return;
 
-			const schedulerId = jobId.slice("repeat:".length, jobId.lastIndexOf(":"));
-			if (schedulerId === "") {
-				// Degenerate `repeat:<n>` / `repeat::<n>` with no scheduler segment: an empty hash field would
-				// pool every such id into one counter, so log and skip rather than hincrby an empty key.
+			// Any id length: a scheduler an earlier release made for a longer cron id still stalls, and this guard is the
+			// one that stops it re-paying.
+			const run = cronRunOf(jobId);
+			if (run === null) {
+				// A `repeat:` id with no scheduler segment (`repeat:<n>`, `repeat::<n>`) or none BullMQ mints: an empty or
+				// guessed counter would pool unrelated ids into one, so log and skip rather than count a key.
 				log("scheduler_stall_unparsed", { jobId });
 				return;
 			}
+			const schedulerId = run.id;
 
 			const key = stallKey(schedulerId);
 			const count = Number(await redis.incr(key));

package/src/schedules.mjs

@@ -17,6 +17,8 @@ import { isAbsolute } from "node:path";
 import { configError } from "./config.mjs";
 import { triggerCapAboveDeployment } from "./money.mjs";
 import { parseTriggers } from "./triggers.mjs";
+import { withJobDataVersion } from "./job-data-version.mjs";
+import { LOCAL_JOB_KEPT_COMPLETED_S, LOCAL_JOB_KEPT_FAILED_S } from "./retention.mjs";
 
 /**
  * Parse, validate, and select the cron schedules from the triggers file named by `config.triggersFile`.
@@ -24,7 +26,7 @@ import { parseTriggers } from "./triggers.mjs";
  * valid deployment. `readFileSync`/`existsSync` are injectable so tests exercise the full path with no
  * real filesystem.
  */
-export function loadSchedules(config, { readFileSync = fsReadFileSync, existsSync = fsExistsSync, fleet = false } = {}) {
+export function loadSchedules(config, { readFileSync = fsReadFileSync, existsSync = fsExistsSync, fleet = false, dataVersions = {} } = {}) {
 	const path = config.triggersFile;
 	if (path === null || path === undefined) return []; // cron disabled
 
@@ -44,7 +46,7 @@ export function loadSchedules(config, { readFileSync = fsReadFileSync, existsSyn
 		throw configError(`trigger at index ${above}: run.maxCostUsd is above this deployment's PI_MAX_COST_USD. A trigger can only narrow the per-job cost cap, so lower it or remove it (${path})`);
 	}
 
-	return triggers.filter((t) => t.on.type === "cron").map((t) => normalizeCronSchedule(t, path, existsSync, fleet));
+	return triggers.filter((t) => t.on.type === "cron").map((t) => normalizeCronSchedule(t, path, existsSync, fleet, dataVersions));
 }
 
 /**
@@ -120,7 +122,7 @@ export function cronPlacement(run, { existsSync = fsExistsSync, fleet = false }
 	return fleet ? "elsewhere" : "refused";
 }
 
-function normalizeCronSchedule({ on, run }, path, existsSync, fleet) {
+function normalizeCronSchedule({ on, run }, path, existsSync, fleet, dataVersions = {}) {
 	// The pure validator already guaranteed a non-empty, `:`-free, charset-valid, unique id and a
 	// well-formed pattern; folder existence is the one fs-dependent check it deferred to here.
 	//
@@ -182,10 +184,21 @@ function normalizeCronSchedule({ on, run }, path, existsSync, fleet) {
 	// `portfolio` (issue #505) is conditional like `command`, so an unflagged schedule grows no key, and it rides at JOB
 	// level, never inside `trigger`: `trigger` is copied verbatim into `/job/event.json`, and whether a job may send a
 	// budget plan is the worker's decision to re-check against the live file, not a fact for the agent to read.
-	const data = { kind: "local", folder: run.folder, flow: run.flow, task: run.task, ...(run.command !== undefined && { command: run.command }), provider: run.provider, model: run.model, maxTurns: run.maxTurns, github: run.github, packages: run.packages, image: run.image, ...(run.backend !== undefined && { backend: run.backend }), ...(run.excludeTools !== undefined && { excludeTools: run.excludeTools }), ...(run.mcp !== undefined && { mcp: run.mcp }), ...(run.models !== undefined && { models: run.models }), ...(run.maxCostUsd !== undefined && { maxCostUsd: run.maxCostUsd }), ...(run.skillsDir !== undefined && { skillsDir: run.skillsDir }), ...(run.secrets !== undefined && { secrets: run.secrets }), ...(run.secretsProfile !== undefined && { secretsProfile: run.secretsProfile }), resume: run.resume, ...(run.portfolio !== undefined && { portfolio: run.portfolio }), trigger: { id: on.id, pattern: on.pattern } };
+	//
+	// `progress` (issue #644) only when armed, as on a forge job, so an unflagged schedule grows no key. A cron job never
+	// carries `statusComment`: the loader refuses `true` there, with no issue or pull request to post on. Nor `recover`
+	// (issue #645), for the same posture: the loader refuses `true` on cron, so there is no armed value to carry.
+	// `steer` (issue #646) DOES ride a cron job, progress's rule: only when armed, at JOB level (never inside `trigger`,
+	// which becomes /job/event.json), and never part of the repeat:<id>:<millis> id. An operator can steer a folder job.
+	//
+	// Stamped like every builder's (issue #761, `withJobDataVersion`): the template is what every tick of this schedule
+	// stores, so a field that needs a version above 1 says so on every job, and nothing is added while none does.
+	// `dataVersions` is the version rule, injectable for tests.
+	const data = withJobDataVersion({ kind: "local", folder: run.folder, flow: run.flow, task: run.task, ...(run.command !== undefined && { command: run.command }), provider: run.provider, model: run.model, maxTurns: run.maxTurns, github: run.github, packages: run.packages, image: run.image, ...(run.backend !== undefined && { backend: run.backend }), ...(run.excludeTools !== undefined && { excludeTools: run.excludeTools }), ...(run.excludeExtensions !== undefined && { excludeExtensions: run.excludeExtensions }), ...(run.excludeExtensionTools !== undefined && { excludeExtensionTools: run.excludeExtensionTools }), ...(run.mcp !== undefined && { mcp: run.mcp }), ...(run.models !== undefined && { models: run.models }), ...(run.fallbackModels !== undefined && { fallbackModels: run.fallbackModels }), ...(run.maxCostUsd !== undefined && { maxCostUsd: run.maxCostUsd }), ...(run.skillsDir !== undefined && { skillsDir: run.skillsDir }), ...(run.secrets !== undefined && { secrets: run.secrets }), ...(run.secretsProfile !== undefined && { secretsProfile: run.secretsProfile }), resume: run.resume, ...(run.portfolio !== undefined && { portfolio: run.portfolio }), ...(run.progress === true && { progress: true }), ...(run.steer === true && { steer: true }), trigger: { id: on.id, pattern: on.pattern } }, dataVersions);
 	// Retention only; the deterministic repeat:<id>:<millis> jobId supplies dedup, so no jobId here, and
-	// scheduler jobs are not retried (DES-CRON-VIA-BULLMQ-SCHEDULER) so no attempts/backoff.
-	const opts = { removeOnComplete: { age: 24 * 3600 }, removeOnFail: { age: 7 * 24 * 3600 } };
+	// scheduler jobs are not retried (DES-CRON-VIA-BULLMQ-SCHEDULER) so no attempts/backoff. A local job's retention, in
+	// SECONDS (BullMQ's `age`), from the one place the console's re-run reads it too (retention.mjs).
+	const opts = { removeOnComplete: { age: LOCAL_JOB_KEPT_COMPLETED_S }, removeOnFail: { age: LOCAL_JOB_KEPT_FAILED_S } };
 
 	return { schedulerId: on.id, name: "local", pattern: on.pattern, data, opts };
 }

package/src/secret-bundles.mjs

@@ -0,0 +1,245 @@
+/**
+ * Project secret bundles (issue #704, REQ-TRIGGER-SECRETS, DES-PROJECT-SECRET-BUNDLES): a project's whole
+ * environment, read HOST-SIDE by ONE resolver call per job, before the container starts.
+ *
+ * `run.secrets` grants per trigger and per field, and its resolver runs once per reference. A bundle grants per
+ * project and runs once per JOB: the triggers file's `secretBundles` binds a project to a bundle profile and a
+ * reference, and the resolver prints every variable of that environment on stdout. A variable the operator adds
+ * or removes in their manager (or file) reaches the next job with no edit to the reviewed file. That is the
+ * asked-for trade, recorded where `DES-PER-TRIGGER-SECRET-PROFILE` once rejected it: the reviewed file no longer
+ * lists the fields, so the review of WHAT a project holds moves to the manager.
+ *
+ * THE SEAM IS STILL A COMMAND. `secrets.mjs`' rule holds verbatim: the reference grammar belongs to the
+ * operator's script, nothing here recognises a manager's notation, and the resolver's exit code is the runner's
+ * (`INT-RUNNER-EXIT-CODE-PROTOCOL`): 0 carries the environment, 2 says the reference is wrong (refused, never
+ * retried), anything else says the manager could not be reached (retried). `runResolver` is imported, not
+ * copied, so the spawn, the stdio, the timeout, the abort and the kill escalation are the ones `run.secrets` has.
+ *
+ * WHAT IS DIFFERENT, and each for a reason:
+ *   - the byte cap is the whole environment's, `MAX_BUNDLE_BYTES`, and every value still holds to
+ *     `MAX_SECRET_BYTES`, so a bundle cannot carry a value a single reference could not;
+ *   - the output is PARSED, by a declared format, strictly (`parseBundleOutput`);
+ *   - a reserved name is a REFUSAL naming the variable, never a drop. A variable an operator added in their
+ *     manager that silently never arrives is this project's worst outcome, and dropping is that;
+ *   - errors and log lines name VARIABLES (and dotenv line numbers), never a value, a reference or a byte the
+ *     resolver printed. A variable name is the operator's own choice from their own manager, the class
+ *     `run.secrets` names freely in its logs; a line that failed to parse is not named even by its first word,
+ *     because a malformed line can be the second half of a value.
+ */
+
+import { spawn } from "node:child_process";
+import { providerKeyCandidates } from "./env-allowlist.mjs";
+import { findDuplicateKey } from "./json-duplicates.mjs";
+import { mergeSecretProfiles, withinRoots } from "./secret-profiles.mjs";
+import { MAX_SECRET_BYTES, realExecutable, runResolver } from "./secrets.mjs";
+import { ENV_NAME, RESERVED_ENV_NAMES, SECRET_BUNDLE_VARS_MAX } from "./triggers.mjs";
+
+/**
+ * The largest output a bundle resolver may print: 256 KiB. Four `MAX_SECRET_BYTES`, so a bundle holds a few PEM
+ * chains and a service-account JSON beside its plain keys, and the whole of it sits in one worker's memory once
+ * per job start (never per reference). On overflow the child is killed and the job refuses, NEVER truncated, the
+ * rule `MAX_SECRET_BYTES` states: a cut environment is a variable silently missing.
+ */
+export const MAX_BUNDLE_BYTES = 256 * 1024;
+
+/**
+ * Names a BUNDLE may not carry, beyond the ones `run.secrets` may not bind (issue #704 review). A bundle's contents are
+ * not reviewed: they are whatever the operator's manager holds at the job's start, written by whoever can write that
+ * manager, so a variable that changes what CODE runs in the job, or how it trusts the network, is refused by name
+ * rather than injected. Each one steers a loader or a tool the job image runs, whatever the agent does:
+ *   - the dynamic loaders (`LD_*`, `DYLD_*`) and `PATH`, which choose the binaries and libraries every process loads;
+ *   - Node's `NODE_OPTIONS` (the runner gives its children a metering preload through it, `image/runner/src/child-watch.mjs`,
+ *     so a bundle value would also blind the child meter) and `NODE_PATH`, and TLS trust (`NODE_EXTRA_CA_CERTS`,
+ *     `NODE_TLS_REJECT_UNAUTHORIZED`, `SSL_CERT_FILE`, `SSL_CERT_DIR`), which `provider-steering.mjs` deliberately leaves
+ *     bindable by a REVIEWED trigger and which an unreviewed bundle must not set;
+ *   - the shells' start-up hooks (`BASH_ENV`, `ENV`, `PROMPT_COMMAND`), run by every non-interactive shell the agent opens;
+ *   - git's command and config hooks (`GIT_SSH`, `GIT_SSH_COMMAND`, `GIT_ASKPASS`, `GIT_EXEC_PATH`, `GIT_PROXY_COMMAND`,
+ *     `SSH_ASKPASS`, and every `GIT_CONFIG*`, which can set `core.sshCommand` or a credential helper), which run on the
+ *     push of the job's own branch with its forge token in hand;
+ *   - the interpreters' start-up hooks (`PYTHONPATH`, `PYTHONHOME`, `PYTHONSTARTUP`, `PERL5OPT`, `PERL5LIB`, `RUBYOPT`,
+ *     `RUBYLIB`, `JAVA_TOOL_OPTIONS`), git's template directory (`GIT_TEMPLATE_DIR`, hooks copied into every clone);
+ *   - the other tools' TLS trust (`GIT_SSL_NO_VERIFY`, `GIT_SSL_CAINFO`, `GIT_SSL_CAPATH`, `REQUESTS_CA_BUNDLE`,
+ *     `CURL_CA_BUNDLE`) and npm's whole configuration (`npm_config_*`, matched CASE-INSENSITIVELY because npm reads it
+ *     so: a registry, a script shell, an `ignore-scripts` turned off).
+ * The table is held to `docs/secrets.md`'s list between its markers by a test, both directions.
+ * A REFUSAL naming the variable (`secret-name-reserved`), never a drop, the header's rule. `run.secrets` keeps its own
+ * narrower set: a trigger is reviewed, and a deploy flow may legitimately bind, say, `PYTHONPATH`.
+ */
+export const BUNDLE_LOADER_NAMES = Object.freeze(["PATH", "NODE_OPTIONS", "NODE_PATH", "NODE_EXTRA_CA_CERTS", "NODE_TLS_REJECT_UNAUTHORIZED", "SSL_CERT_FILE", "SSL_CERT_DIR", "BASH_ENV", "ENV", "PROMPT_COMMAND", "GIT_SSH", "GIT_SSH_COMMAND", "GIT_ASKPASS", "GIT_EXEC_PATH", "GIT_PROXY_COMMAND", "GIT_TEMPLATE_DIR", "GIT_SSL_NO_VERIFY", "GIT_SSL_CAINFO", "GIT_SSL_CAPATH", "SSH_ASKPASS", "REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE", "PYTHONPATH", "PYTHONHOME", "PYTHONSTARTUP", "PERL5OPT", "PERL5LIB", "RUBYOPT", "RUBYLIB", "JAVA_TOOL_OPTIONS"]);
+/** Prefixes, exact case; and `npm_config_`, compared case-insensitively. */
+export const BUNDLE_LOADER_PREFIXES = Object.freeze(["LD_", "DYLD_", "GIT_CONFIG"]);
+export const BUNDLE_LOADER_PREFIXES_ANY_CASE = Object.freeze(["npm_config_"]);
+const LOADER_NAME_SET = new Set(BUNDLE_LOADER_NAMES);
+
+/** Whether a bundle may not carry `name` for the loader reason above. Exported for doctor and the docs' test. */
+export function bundleLoaderName(name) {
+	return LOADER_NAME_SET.has(name) || BUNDLE_LOADER_PREFIXES.some((prefix) => name.startsWith(prefix)) || BUNDLE_LOADER_PREFIXES_ANY_CASE.some((prefix) => name.toLowerCase().startsWith(prefix));
+}
+
+/**
+ * Parse a bundle resolver's stdout by its DECLARED format. `{ vars }` (an object of own string properties, in the
+ * resolver's order), or `{ failure, name?, line? }` with `failure` one of OUR words:
+ *
+ *   - `malformed`   a dotenv line that is not `NAME=value`, a blank or a `#` comment; JSON that is not one object;
+ *   - `quoted`      a dotenv value that starts with a quote: no quoting is the grammar, and a manager that quotes
+ *                   (or an operator who did) would otherwise hand the container the quotes as part of the value;
+ *   - `not-string`  a JSON value that is not a string (a number, a nested object), named by its variable;
+ *   - `bad-name`    a name that is not an environment variable name (ENV_NAME), or `__proto__`; not named;
+ *   - `duplicate`   one name twice: which value wins would be the parser's choice, never the operator's;
+ *   - `empty-value` an empty value: a set-but-empty variable is how a broken manager reads as configured;
+ *   - `nul`         a value with a NUL, which `-e NAME=VALUE` would silently cut;
+ *   - `value-too-large` a value over `MAX_SECRET_BYTES`;
+ *   - `too-many`    over `SECRET_BUNDLE_VARS_MAX` variables;
+ *   - `empty`       no variable at all: exit 0 that grants nothing is the silent-no-op class `runResolver` refuses
+ *                   for a single value, and an operator who emptied an environment by accident must hear it.
+ *
+ * DOTENV IS STRICT: `NAME=value` per line, the value verbatim to the end of the line, no quoting, no `export `, no
+ * spaces around `=`, no expansion, no continuation. Every dotenv dialect differs on exactly those, and a parser
+ * that accepts one dialect's quotes silently mangles another's values. One trailing `\r` per line is removed, so a
+ * resolver on Windows (CRLF) reads as it was written; a value that needs a `\r`, a newline or a leading quote goes
+ * through `json`, which is what that format is for (PEMs and other multi-line values).
+ */
+export function parseBundleOutput(text, format) {
+	const vars = {};
+	const seen = new Set();
+	const add = (name, value, line = null) => {
+		if (typeof name !== "string" || !ENV_NAME.test(name) || name === "__proto__") return { failure: "bad-name", line };
+		if (seen.has(name)) return { failure: "duplicate", name };
+		if (typeof value !== "string") return { failure: "not-string", name };
+		if (value === "") return { failure: "empty-value", name };
+		if (value.includes("\u0000")) return { failure: "nul", name };
+		if (Buffer.byteLength(value, "utf8") > MAX_SECRET_BYTES) return { failure: "value-too-large", name };
+		seen.add(name);
+		if (seen.size > SECRET_BUNDLE_VARS_MAX) return { failure: "too-many" };
+		// An own data property whatever the name (the charset is checked above, `__proto__` refused): never `vars[name] =`.
+		Object.defineProperty(vars, name, { value, enumerable: true, writable: true, configurable: true });
+		return null;
+	};
+	if (format === "json") {
+		let parsed;
+		try {
+			parsed = JSON.parse(text);
+		} catch {
+			return { failure: "malformed" };
+		}
+		if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) return { failure: "malformed" };
+		// A non-string value first, so a duplicate key INSIDE a nested object is reported as what it is (a value that is
+		// not a string), never as a duplicate variable named after a key no variable has.
+		const nested = Object.keys(parsed).find((name) => typeof parsed[name] !== "string");
+		if (nested !== undefined) return ENV_NAME.test(nested) && nested !== "__proto__" ? { failure: "not-string", name: nested } : { failure: "bad-name" };
+		// JSON keeps the LAST of two equal keys, so a duplicate would be a value silently won: the triggers file's rule.
+		const duplicate = findDuplicateKey(text);
+		if (duplicate) return ENV_NAME.test(duplicate.key) ? { failure: "duplicate", name: duplicate.key } : { failure: "bad-name" };
+		for (const name of Object.keys(parsed)) {
+			const bad = add(name, parsed[name]);
+			if (bad) return bad.failure === "bad-name" ? { failure: "bad-name" } : bad;
+		}
+	} else if (format === "dotenv") {
+		const lines = text.split("\n");
+		for (let i = 0; i < lines.length; i++) {
+			const raw = lines[i].endsWith("\r") ? lines[i].slice(0, -1) : lines[i];
+			if (/^[ \t]*$/.test(raw) || /^[ \t]*#/.test(raw)) continue;
+			const eq = raw.indexOf("=");
+			if (eq <= 0) return { failure: "malformed", line: i + 1 };
+			const name = raw.slice(0, eq);
+			const value = raw.slice(eq + 1);
+			if (!ENV_NAME.test(name)) return { failure: name.trim() !== name || name.startsWith("export ") ? "malformed" : "bad-name", line: i + 1 };
+			if (value.startsWith('"') || value.startsWith("'")) return { failure: "quoted", name };
+			// A carriage return left after the one a CRLF line ends with is part of no dotenv value anyone means to write;
+			// a value that needs one goes through json.
+			if (value.includes("\r")) return { failure: "malformed", line: i + 1 };
+			const bad = add(name, value, i + 1);
+			if (bad) return bad;
+		}
+	} else {
+		// Unreachable through the declaration parsers, which refuse any other format at boot or overlay read.
+		return { failure: "malformed" };
+	}
+	if (seen.size === 0) return { failure: "empty" };
+	return { vars };
+}
+
+/**
+ * Build the bundle seam the processor injects: `{ lookup, resolve }`.
+ *
+ *   - `lookup(profile, { overlayBundles })` answers, with no spawn, whether this host can serve the profile:
+ *     `{ ok }`, `{ profileUnknown }` or `{ ambiguous }`. FREE, so the processor asks it before the mint, with the
+ *     other free refusals (CONST-BUDGET-BEFORE-TOKENS); its rule is `makeSecretsResolver`'s lookup verbatim (own
+ *     names only, the realpath probe, an overlay path bounded by PI_SECRET_RESOLVER_ROOTS, a name declared in both
+ *     sources ambiguous).
+ *   - `resolve(job, binding, { signal, overlayBundles })` runs the resolver ONCE for `binding.ref` and returns
+ *     `{ ok, vars }`, `{ profileUnknown }`, `{ ambiguous }`, `{ reserved: name }`, `{ unresolved, failure, ... }`
+ *     or `{ unreachable, failure, ... }`. `unresolved` and `unreachable` are `true`, never a name: one call holds
+ *     every variable, so there is no single variable to blame for an exit code.
+ *
+ * `envBundles` is parsed at BOOT (config.mjs `secretBundleProfiles`), `overlayBundles` per job, `secretProfiles`'
+ * split and its reason. NOTHING IS ZEROED, `makeSecretsResolver`'s honest half, and the same property holds: the
+ * returned map is a local of `runJob`, never attached to the job, never logged, never in a result or a record
+ * (the record carries names, rebuilt in run-history.mjs).
+ */
+export function makeBundleResolver({ envBundles = {}, roots = [], forwardEnv = [], timeoutMs = 10000, spawnFn = spawn, hostEnv = process.env, realExecutablePath = realExecutable, log = () => {} } = {}) {
+	function lookup(profile, { overlayBundles = {} } = {}) {
+		const merged = mergeSecretProfiles(envBundles, overlayBundles);
+		if (merged.ambiguous) return { ambiguous: merged.ambiguous };
+		const declared = Object.hasOwn(merged.profiles, profile) ? merged.profiles[profile] : undefined;
+		if (declared === null || typeof declared !== "object" || typeof declared.path !== "string") return { profileUnknown: profile };
+		let path;
+		try {
+			path = realExecutablePath(declared.path);
+		} catch {
+			path = null;
+		}
+		if (!path) return { profileUnknown: profile };
+		// The bound on the REALPATH, in the worker, for env-declared profiles exempt: secrets.mjs gives the reasons.
+		if (!Object.hasOwn(envBundles, profile) && !withinRoots(path, roots)) return { profileUnknown: profile };
+		return { ok: true, path, format: declared.format };
+	}
+
+	async function resolve(job, binding, { signal, overlayBundles = {} } = {}) {
+		const found = lookup(binding?.profile, { overlayBundles });
+		if (!found.ok) return found;
+		const outcome = await runResolver(spawnFn, found.path, binding.ref, { timeoutMs, hostEnv, signal, maxBytes: MAX_BUNDLE_BYTES });
+		if (outcome.failure) {
+			// The bundle ID, our own failure vocabulary, the script's small integer exit and a byte COUNT; never the
+			// reference, the path, or the resolver's words.
+			log("secret_bundle_resolve_failed", { bundle: binding.id ?? null, failure: outcome.failure, detail: outcome.detail, code: outcome.code ?? null, stderrBytes: outcome.stderrBytes });
+			const shape = { failure: outcome.detail, code: outcome.code ?? null, stderrBytes: outcome.stderrBytes };
+			return outcome.failure === "infra" ? { unreachable: true, ...shape } : { unresolved: true, ...shape };
+		}
+		const parsed = parseBundleOutput(outcome.value, found.format);
+		if (parsed.failure) {
+			log("secret_bundle_resolve_failed", { bundle: binding.id ?? null, failure: "policy", detail: parsed.failure, name: parsed.name ?? null, line: parsed.line ?? null });
+			return { unresolved: true, failure: parsed.failure, name: parsed.name ?? null, line: parsed.line ?? null, code: 0, stderrBytes: outcome.stderrBytes };
+		}
+		// The reserved names, all three sets `run.secrets` is held to, asked of the WHOLE bundle: the statically
+		// knowable set the triggers file refuses at load (it cannot see a bundle's names, which exist only once read),
+		// this job's provider credential variables, and PI_FORWARD_ENV. A REFUSAL naming the first one in the
+		// resolver's order, never a drop (see the header), and never resolved by ordering: buildContainerEnv writes
+		// the provider credential BEFORE the secrets loop, so a bundle carrying ANTHROPIC_API_KEY would silently
+		// replace the key every job of the project spends (secrets.mjs records the class).
+		const reserved = new Set([...RESERVED_ENV_NAMES, ...providerKeyCandidates(job?.provider), ...forwardEnv]);
+		const clash = Object.keys(parsed.vars).find((name) => reserved.has(name) || bundleLoaderName(name));
+		if (clash !== undefined) return { reserved: clash };
+		return { ok: true, vars: parsed.vars };
+	}
+
+	return { lookup, resolve };
+}
+
+/**
+ * Merge a bundle's variables with the trigger's own `run.secrets` values (issue #704). `run.secrets` WINS a name
+ * clash: it is the narrower grant, written for this one trigger in the reviewed file, and the bundle is the
+ * project's default. The shadowed names are returned, sorted, for the record, so an override is visible rather
+ * than inferred. Own properties throughout, so no name can reach a prototype.
+ */
+export function mergeBundleSecrets(bundleVars = {}, triggerSecrets = {}) {
+	const merged = {};
+	const define = (name, value) => Object.defineProperty(merged, name, { value, enumerable: true, writable: true, configurable: true });
+	const shadowed = [];
+	for (const [name, value] of Object.entries(bundleVars)) {
+		if (Object.hasOwn(triggerSecrets, name)) shadowed.push(name);
+		else define(name, value);
+	}
+	for (const [name, value] of Object.entries(triggerSecrets)) define(name, value);
+	return { secrets: merged, shadowed: shadowed.sort() };
+}

package/src/secret-profiles.mjs

@@ -60,17 +60,27 @@ export function parseSecretProfiles(raw) {
 		if (!PROFILE_NAME.test(name)) {
 			throw configError(`PI_SECRET_PROFILES profile name ${JSON.stringify(name)} may use letters, digits, dot, dash and underscore only`);
 		}
-		if (name in profiles) {
+		if (Object.hasOwn(profiles, name)) {
 			throw configError(`PI_SECRET_PROFILES declares ${JSON.stringify(name)} twice -- one of the two is not the resolver you think is running`);
 		}
 		if (path === "" || !isAbsolutePath(path)) {
 			throw configError(`PI_SECRET_PROFILES profile ${JSON.stringify(name)} needs an ABSOLUTE path to its resolver -- a service manager's working directory is not your shell's, so a relative path is a different file on every host`);
 		}
-		profiles[name] = path;
+		defineProfile(profiles, name, path);
 	}
 	return profiles;
 }
 
+/**
+ * OWN properties only, here and in every lookup over a profile table (issue #704 review). The charset admits
+ * `toString`, `constructor` and `__proto__`, so `name in table` answers "declared" for a name nobody wrote,
+ * and `table[name] = path` with `__proto__` sets the object's prototype instead of declaring a profile. A
+ * defined property is an own, enumerable data property whatever the name.
+ */
+function defineProfile(table, name, path) {
+	Object.defineProperty(table, name, { value: path, enumerable: true, writable: true, configurable: true });
+}
+
 /** Absolute on this platform, accepting a Windows drive letter or UNC root as `service.mjs` does. */
 function isAbsolutePath(path) {
 	return isAbsolute(path) || /^([A-Za-z]:[\\/]|\\\\)/.test(path);
@@ -92,7 +102,7 @@ function isAbsolutePath(path) {
  */
 export function mergeSecretProfiles(envProfiles = {}, overlayProfiles = {}) {
 	for (const name of Object.keys(overlayProfiles)) {
-		if (name in envProfiles) return { ambiguous: name };
+		if (Object.hasOwn(envProfiles, name)) return { ambiguous: name };
 	}
 	return { profiles: { ...envProfiles, ...overlayProfiles } };
 }
@@ -118,3 +128,93 @@ export function withinRoots(candidate, roots = []) {
 		return target === base || target.startsWith(base + sep);
 	});
 }
+
+/**
+ * The profiles a host would actually resolve, for its capability row and for doctor (issue #704): every
+ * env-declared profile, plus each overlay-declared one whose resolver is an executable file whose REALPATH
+ * sits inside `roots`. That is the lookup `secrets.mjs -> makeSecretsResolver` makes per job, so a host
+ * never advertises an overlay profile it would answer `secret-profile-unknown` for, and never hides one it
+ * would serve.
+ *
+ * `realExecutablePath` is REQUIRED and injected, which keeps this module fs-free: the worker passes
+ * `secrets.mjs -> realExecutable`, the probe its resolver uses, and that module is not imported here
+ * because it loads pi through `env-allowlist.mjs`.
+ *
+ * Env profiles are taken as declared, with no probe, which is what the row always published: doctor is
+ * where a missing env resolver is reported, and probing it here would change an existing host's row.
+ *
+ * `{ ambiguous }` when a name is declared in both sources (`mergeSecretProfiles`). The resolver then
+ * refuses EVERY secrets job on this host, whatever profile it names, so the caller advertises no secret
+ * profile at all rather than one it would refuse.
+ */
+export function servableSecretProfiles({ envProfiles = {}, overlayProfiles = {}, roots = [], realExecutablePath }) {
+	const merged = mergeSecretProfiles(envProfiles, overlayProfiles);
+	if (merged.ambiguous) return { ambiguous: merged.ambiguous };
+	const profiles = { ...envProfiles };
+	for (const [name, declared] of Object.entries(overlayProfiles)) {
+		if (typeof declared !== "string") continue;
+		let path;
+		try {
+			path = realExecutablePath(declared);
+		} catch {
+			path = null;
+		}
+		if (path && withinRoots(path, roots)) defineProfile(profiles, name, declared);
+	}
+	return { profiles };
+}
+
+/**
+ * The output formats a bundle resolver may declare (issue #704). DECLARED, never sniffed: `{"A":"1"}` is one
+ * malformed dotenv line and `A=1` is malformed JSON, so a sniffer would have to pick, and the pick would change
+ * the day a manager prints a JSON-looking value. The operator says which one their script prints.
+ */
+export const BUNDLE_FORMATS = Object.freeze(["dotenv", "json"]);
+
+/**
+ * Parse `PI_SECRET_BUNDLES`: `name:/abs/path:format,other:/abs/other:format` (issue #704). Returns
+ * `{ [name]: { path, format } }`, empty when unset. Throws (config-tagged, so the worker refuses to boot) on
+ * anything malformed, `parseSecretProfiles`' doctrine and for its reason.
+ *
+ * ITS OWN NAMESPACE, never a flag on a `PI_SECRET_PROFILES` entry. The two scripts answer different questions
+ * (one prints one value for one reference, the other prints a whole environment) and are called with different
+ * caps and parsers: a profile reachable as a bundle would have its single value parsed as dotenv and refused
+ * as `malformed` on every job, and a bundle reachable as a profile would put its whole environment into ONE
+ * variable. Two variables keep each script reachable only as what it was written to be, the reason
+ * `PI_WAIT_PROFILES` is separate from both.
+ *
+ * The NAME splits on the FIRST colon and the FORMAT on the LAST one, so `shop:C:\pi\shop.cmd:json` keeps its
+ * drive letter (the hazard `parseSecretProfiles` states). The format is REQUIRED: a missing one is a boot
+ * refusal naming the two choices, never a default, because a default would be the sniffing above with a
+ * slower fuse (the first operator whose script prints JSON gets every job refused as `malformed`).
+ */
+export function parseSecretBundleProfiles(raw) {
+	if (raw === undefined || raw === null || String(raw).trim() === "") return {};
+	const bundles = {};
+	for (const entry of String(raw).split(",")) {
+		const text = entry.trim();
+		if (text === "") continue;
+		const cut = text.indexOf(":");
+		const last = text.lastIndexOf(":");
+		if (cut <= 0 || last === cut) {
+			throw configError(`PI_SECRET_BUNDLES entries must be name:/absolute/path:format (format ${BUNDLE_FORMATS.join(" or ")}), got ${JSON.stringify(text)}`);
+		}
+		const name = text.slice(0, cut).trim();
+		const path = text.slice(cut + 1, last).trim();
+		const format = text.slice(last + 1).trim();
+		if (!PROFILE_NAME.test(name)) {
+			throw configError(`PI_SECRET_BUNDLES profile name ${JSON.stringify(name)} may use letters, digits, dot, dash and underscore only`);
+		}
+		if (!BUNDLE_FORMATS.includes(format)) {
+			throw configError(`PI_SECRET_BUNDLES profile ${JSON.stringify(name)} must end in :dotenv or :json, the format its resolver prints -- it is never guessed from the output`);
+		}
+		if (Object.hasOwn(bundles, name)) {
+			throw configError(`PI_SECRET_BUNDLES declares ${JSON.stringify(name)} twice -- one of the two is not the resolver you think is running`);
+		}
+		if (path === "" || !isAbsolutePath(path)) {
+			throw configError(`PI_SECRET_BUNDLES profile ${JSON.stringify(name)} needs an ABSOLUTE path to its resolver -- a service manager's working directory is not your shell's, so a relative path is a different file on every host`);
+		}
+		Object.defineProperty(bundles, name, { value: Object.freeze({ path, format }), enumerable: true, writable: true, configurable: true });
+	}
+	return bundles;
+}

package/src/secrets.mjs

@@ -84,8 +84,12 @@ export function secretsArmed(job) {
  *
  * An unrecognised exit code is INFRA on `decideRetry`'s own reasoning: a runner that exits with something
  * we do not recognise is one we cannot reason about, and retrying-then-alerting beats accepting it as done.
+ *
+ * EXPORTED for the bundle resolver (issue #704, `secret-bundles.mjs`), which is this same seam with a larger
+ * `maxBytes` (a whole environment, not one value): one spawn, one exit mapping, one timeout and one abort, so a
+ * fix to any of them reaches both rather than one of two copies.
  */
-function runResolver(spawnFn, path, reference, { timeoutMs, hostEnv, signal }) {
+export function runResolver(spawnFn, path, reference, { timeoutMs, hostEnv, signal, maxBytes = MAX_SECRET_BYTES }) {
 	return new Promise((resolve) => {
 		let child;
 		try {
@@ -149,7 +153,7 @@ function runResolver(spawnFn, path, reference, { timeoutMs, hostEnv, signal }) {
 
 		child.stdout?.on("data", (d) => {
 			bytes += d.length;
-			if (bytes > MAX_SECRET_BYTES) {
+			if (bytes > maxBytes) {
 				overflowed = true;
 				stop("overflow");
 				return;
@@ -196,6 +200,15 @@ function stripOneNewline(text) {
 	return text;
 }
 
+/** The resolver's executable probe: the realpath when it is an executable file, else null. Throws when
+ * the path does not resolve, which every caller folds into "not usable". Exported so the capability row
+ * (`servableSecretProfiles`, start.mjs) and doctor ask the question this resolver asks, not a copy of it. */
+export function realExecutable(p) {
+	const real = realpathSync(p);
+	const st = statSync(real);
+	return st.isFile() && (st.mode & 0o111) !== 0 ? real : null;
+}
+
 /**
  * Build the pre-spend resolver the processor injects.
  *
@@ -235,11 +248,7 @@ export function makeSecretsResolver({
 	// follows symlinks -- so `lstat` would refuse an ordinary /usr/local/bin/op -> ../Cellar/... The mode is
 	// deliberately NOT checked here: a group- or world-writable resolver is a doctor WARNING, on the
 	// env-setup precedent, because refusing a job over a mode bit at job time is a new class of refusal.
-	realExecutablePath = (p) => {
-		const real = realpathSync(p);
-		const st = statSync(real);
-		return st.isFile() && (st.mode & 0o111) !== 0 ? real : null;
-	},
+	realExecutablePath = realExecutable,
 	log = () => {},
 } = {}) {
 	return async function resolveSecrets(job, { signal, overlayProfiles = {} } = {}) {
@@ -247,7 +256,8 @@ export function makeSecretsResolver({
 		if (merged.ambiguous) return { ambiguous: merged.ambiguous };
 
 		const wanted = typeof job?.secretsProfile === "string" && job.secretsProfile !== "" ? job.secretsProfile : DEFAULT_SECRETS_PROFILE;
-		const declared = merged.profiles[wanted];
+		// Own properties only: the charset admits `toString` and `constructor`, which every object inherits.
+		const declared = Object.hasOwn(merged.profiles, wanted) ? merged.profiles[wanted] : undefined;
 		// A LOOKUP, never a fallback. There is deliberately no "if only one profile is declared, use it"
 		// rule: env-allowlist.mjs records what the last `else` of that shape cost, and a default whose
 		// meaning changes the day a second profile appears is that mistake with a slower fuse.
@@ -267,7 +277,7 @@ export function makeSecretsResolver({
 		// executable on the host". An env-declared profile is exempt: PI_SECRET_PROFILES already lives
 		// beside the forge tokens and the App key, so requiring roots for it would bound nothing and would
 		// break every deployment that never wanted panel authoring.
-		const fromOverlay = !(wanted in envProfiles);
+		const fromOverlay = !Object.hasOwn(envProfiles, wanted);
 		if (fromOverlay && !withinRoots(path, roots)) return { profileUnknown: wanted };
 
 		const references = job?.secrets ?? {};

package/src/service-env.mjs

@@ -21,7 +21,7 @@
  *
  * Nothing here loads a `.env` into a process (`docs/secrets.md`): the result is an object the caller judges from.
  */
-import { constants as fsConstants } from "node:fs";
+import { constants as fsConstants, readFileSync } from "node:fs";
 import { userInfo } from "node:os";
 import { dirname, join } from "node:path";
 import { SYSTEMD_HAZARD_SHAPES, decodeEnvFile, envFileHazard, readEnvAssignments } from "./env-file.mjs";
@@ -36,6 +36,8 @@ export function serviceEnvLoader(platform) {
  * The deployment `.env` as the service is judged by (issue #464, gate round 3): decoded from its BYTES with the
  * hardened reader, and `hazard` set to the first thing the service's loader reads differently from this reader, or
  * refuses to load (a NUL, invalid UTF-8, an environment too big to exec, a lone CR), as `{ line, what, fix }`, or null.
+ * `raw` is the content as given, for a reader that judges the bytes itself (`readValkeyKeys`), since decoding erases
+ * invalid UTF-8.
  */
 export function serviceEnvFileOf(raw, path, loader) {
 	const { text, loadHazard } = decodeEnvFile(raw, { loader });
@@ -43,9 +45,9 @@ export function serviceEnvFileOf(raw, path, loader) {
 	let hazard = null;
 	if (h !== null) {
 		const shape = h.shape !== undefined ? SYSTEMD_HAZARD_SHAPES[h.shape] : null;
-		hazard = shape ? { line: h.line, what: `has ${shape.what}${h.detail ? ` (${h.detail})` : ""}`, fix: shape.fix } : { line: h.line, what: "is one this command cannot read the way the service's loader will (an open quote, a continuation, or a line that runs)", fix: "fix that line, then re-run doctor" };
+		hazard = shape ? { line: h.line, shape: h.shape, what: `has ${shape.what}${h.detail ? ` (${h.detail})` : ""}`, fix: shape.fix } : { line: h.line, what: "is one this command cannot read the way the service's loader will (an open quote, a continuation, or a line that runs)", fix: "fix that line, then re-run doctor" };
 	}
-	return { text, path, loader, hazard };
+	return { text, raw, path, loader, hazard };
 }
 
 /**
@@ -236,12 +238,43 @@ export function readDeploymentEnv({ dir, fs, uid = process.geteuid?.(), names =
  * The `.env` of the deployment in `dir`, read the way doctor reads it (a regular file only, as bytes, one read) and
  * resolved for `keys` (`resolveServiceEnv`). For a caller that has no file of its own in hand (the admin panel).
  * Returns the resolution plus `path`, `unreadable` (a reason, when something is at the path and could not be read) and
- * `untrusted` (`envFileTrust`'s reason); in either case nothing was taken from it.
+ * `untrusted` (`envFileTrust`'s reason); in either case nothing was taken from it. `shellRunsLine` is the line a sourcing shell does
+ * not read as an assignment, whatever keys were asked for: it can set any of them without spelling one (`. ./more.env`, `eval`), so
+ * `hazardSkipped`, which needs the key spelled, says nothing of it. Only for that loader: systemd ignores such a line and the cmd wrapper
+ * splits it on its `=`. A CR on a comment line is left out, since the shell reads past it and only a writer of this file minds.
  */
 export function deploymentServiceEnv({ env, dir, keys, platform = process.platform, fs, uid = process.geteuid?.() }) {
 	// Issue #471 (gate rounds 1 and 2): one descriptor, and nothing taken from a file another account can change.
 	const { path, bytes, unreadable, untrusted } = readDeploymentEnv({ dir, fs, uid });
 	const raw = untrusted ? null : bytes;
 	const file = raw === null ? null : serviceEnvFileOf(raw, path, serviceEnvLoader(platform));
-	return { ...resolveServiceEnv({ env, file, keys }), path, unreadable, untrusted, hazard: file?.hazard ?? null };
+	const shellRunsLine = file?.loader === "shell" && file.hazard !== null && file.hazard.shape !== "shell-crlf-comment" ? file.hazard.line : null;
+	return { ...resolveServiceEnv({ env, file, keys }), path, unreadable, untrusted, hazard: file?.hazard ?? null, shellRunsLine };
+}
+
+/**
+ * Deployment keys for a CLI verb that acts for the deployment (review of PR #575): `keys` resolved by issue #471's rule
+ * (`resolveServiceEnv`, above), the one doctor and `up` read the service's keys by. This shell's value where
+ * it sets the key, else the `.env` in `cwd` from a line the service's loader reads as written. Where the two disagree,
+ * or a line naming the key cannot be read as the loader reads it, or the file cannot be read and this shell sets none,
+ * the answer is unknown and the verb refuses (`problem`), as `up` refuses on a PI_JOB_IMAGE disagreement: a job queued
+ * on a queue no worker drains, or from a triggers file the worker does not read, is a silent no-op. Returns
+ * `{ env }` (this shell's with the file's values filled in) or `{ problem }`.
+ */
+export function cliDeploymentEnv(env, keys, { cwd = process.cwd(), platform = process.platform, readFile = (p) => readFileSync(p) } = {}) {
+	const envPath = join(cwd, ".env");
+	let file;
+	try {
+		file = serviceEnvFileOf(readFile(envPath), envPath, serviceEnvLoader(platform));
+	} catch (error) {
+		const unset = keys.filter((k) => typeof env[k] !== "string");
+		if (error?.code !== "ENOENT" && unset.length > 0) return { problem: `${envPath} could not be read (${error?.code ?? "error"}), so ${unset.join(" and ")} cannot be told` };
+		return { env };
+	}
+	const read = resolveServiceEnv({ env, file, keys });
+	const [d] = read.disagreements;
+	if (d) return { problem: `${d.key} is ${JSON.stringify(d.shell)} in this shell and ${JSON.stringify(d.file)} in ${envPath}: make them agree (the service runs the file's)` };
+	const unread = [...read.unread.map((u) => u.key), ...read.hazardSkipped];
+	if (unread.length > 0) return { problem: `${envPath} has a line for ${unread.join(" and ")} that the service's loader may read differently, so it cannot be told (pi-dispatch doctor names the line)` };
+	return { env: read.env };
 }