@edgehero/pi-dispatch 4.3.1 → 5.0.0

package/src/community-auth-patches.mjs

@@ -0,0 +1,168 @@
+/** Audited adaptations of MIT community packages. No extension executes while staging or verifying. */
+import * as nodeFs from "node:fs";
+import { createHash } from "node:crypto";
+import { join, relative, dirname, isAbsolute } from "node:path";
+import { COMMUNITY_SOURCE_PINS } from "./community-auth-pins.mjs";
+import { COMMUNITY_ADAPTED_SOURCE_PINS } from "./community-auth-output-pins.mjs";
+
+import { COMMUNITY_AUTH_ADAPTERS } from "./community-auth-metadata.mjs";
+export { COMMUNITY_AUTH_ADAPTERS } from "./community-auth-metadata.mjs";
+const sha = (bytes) => createHash("sha256").update(bytes).digest("hex");
+const fail = (code) => { const error = new Error(code); error.code = code; throw error; };
+function replaceOnce(source, before, after) {
+	if (!source.includes(before) || source.indexOf(before) !== source.lastIndexOf(before)) fail("auth-adapter-source-mismatch");
+	return source.replace(before, after);
+}
+
+export function adaptCommunitySource(adapter, path, input) {
+	let source = input;
+	if (adapter === "pi-anthropic-oauth@0.3.6") {
+		if (path === "src/index.ts") {
+			source = replaceOnce(source, 'name: "Claude Pro/Max",', 'name: "Claude Pro/Max",\n      isSubscription: true,');
+			source = replaceOnce(source, '  pi.registerProvider("anthropic", {', '  Object.defineProperty(streamAnthropicOAuth, Symbol.for("pi-dispatch.subscription-adapter"), { value: "pi-anthropic-oauth@0.3.6" });\n  pi.registerProvider("anthropic", {');
+		}
+		if (path === "src/auth.ts") {
+			source = replaceOnce(source, "callbacks: OAuthLoginCallbacks,\n):", "callbacks: OAuthLoginCallbacks,\n  now: () => number = Date.now,\n):");
+			source = replaceOnce(source, "credentials: OAuthCredentials,\n):", "credentials: OAuthCredentials,\n  signal?: AbortSignal,\n  now: () => number = Date.now,\n):");
+			source = replaceOnce(source, 'headers: makeTokenHeaders(),\n        body: JSON.stringify({\n          grant_type: "refresh_token",', 'headers: makeTokenHeaders(),\n        signal,\n        body: JSON.stringify({\n          grant_type: "refresh_token",');
+			source = replaceOnce(source, '    if (isTransientTokenError(error) && credentials.expires > Date.now()) {\n      return { ...credentials, expires: Date.now() + 30_000 };\n    }\n', "");
+			source = source.replaceAll("expires: Date.now() + data.expires_in", "expires: now() + data.expires_in");
+			source = replaceOnce(source, "    const response = await fetch(url, init);", "    init.signal?.throwIfAborted();\n    const response = await dispatchFetch(url, { ...init, signal: AbortSignal.any([...(init.signal ? [init.signal] : []), AbortSignal.timeout(15_000)]) });");
+			source = replaceOnce(source, "      await new Promise((resolve) => setTimeout(resolve, delayMs));", "      await new Promise<void>((resolve, reject) => {\n        const done = () => { init.signal?.removeEventListener(\"abort\", abort); resolve(); };\n        const timer = setTimeout(done, delayMs);\n        const abort = () => { clearTimeout(timer); reject(new Error(\"auth-network-unavailable\")); };\n        init.signal?.addEventListener(\"abort\", abort, { once: true });\n        if (init.signal?.aborted) abort();\n      });");
+			source = replaceOnce(source, '      `${label} failed: ${response.status} ${bodyText}`,', '      `auth-token-http-${response.status}`,');
+			source = replaceOnce(source, '    throw new Error(`Token refresh failed: ${detail}`);', '    throw error;');
+			source = replaceOnce(source, "    localAuthorization = await createLocalAuthorization(state);", "    localAuthorization = await createLocalAuthorization(state);\n    callbacks.signal?.addEventListener(\"abort\", () => localAuthorization?.cancel(), { once: true });\n    if (callbacks.signal?.aborted) localAuthorization.cancel();");
+			source = 'let dispatchFetch: typeof fetch = globalThis.fetch;\nexport function configureDispatchAuth(transport: typeof fetch) { dispatchFetch = transport; }\n' + source;
+		}
+		if (path === "src/prompt.ts") {
+			const start = source.indexOf("  const paragraphs = text.split(");
+			const end = source.indexOf("\n}\n\nfunction resolvePiRewritePattern", start);
+			if (start < 0 || end < start) fail("auth-adapter-source-mismatch");
+			source = source.slice(0, start) + "  return text;" + source.slice(end);
+		}
+		if (path === "src/convert.ts") {
+			source = replaceOnce(source, "export function convertPiToolsToAnthropic(tools: Tool[], isOAuth: boolean): ToolUnion[] {", "export function convertPiToolsToAnthropic(tools: Tool[], isOAuth: boolean): ToolUnion[] {\n  const names = tools.map((tool) => isOAuth ? toClaudeCodeToolName(tool.name) : tool.name);\n  if (new Set(names).size !== names.length) throw new Error(\"auth-tool-name-collision\");");
+		}
+		if (path === "src/stream.ts") {
+			source = replaceOnce(source, "        dangerouslyAllowBrowser: true,", "        dangerouslyAllowBrowser: true,\n        fetch: options?.fetch,");
+			source = replaceOnce(source, "      // Raw stream instead of the MessageStream helper:", "      const checkedParams = (await options?.onPayload?.(params, model)) ?? params;\n\n      // Raw stream instead of the MessageStream helper:");
+			source = replaceOnce(source, ".create(params, {", ".create(checkedParams as MessageCreateParamsStreaming, {");
+			source = replaceOnce(source, "        } catch {\n          // Response hooks are best-effort and should not break streaming.\n        }", "        } catch (error) { throw error; }");
+			source = replaceOnce(source, "  cache_creation_input_tokens?: number | null;", "  cache_creation_input_tokens?: number | null;\n  cache_creation?: { ephemeral_1h_input_tokens?: number | null };");
+			source = replaceOnce(source, "    const thinkingTokens = usage.output_tokens_details?.thinking_tokens;", "    if (usage.cache_creation?.ephemeral_1h_input_tokens != null) output.usage.cacheWrite1h = usage.cache_creation.ephemeral_1h_input_tokens;\n    const thinkingTokens = usage.output_tokens_details?.thinking_tokens;");
+			source = replaceOnce(source, "            output.usage.totalTokens =", "            output.usage.cacheWrite1h = (event.message.usage as MessageDeltaUsage).cache_creation?.ephemeral_1h_input_tokens ?? 0;\n            output.usage.totalTokens =");
+		}
+	}
+	if (adapter === "@gotgenes/pi-anthropic-auth@3.4.3" && path === "src/oauth-transport.ts") {
+		// Shape first, then let the dispatch model/payload/cost guards inspect what will actually be sent.
+		source = replaceOnce(source, "      const upstream = await composeCallerOnPayload(payload, payloadModel);", "      const upstream = payload;");
+		source = replaceOnce(source, "        return upstream;", "        return composeCallerOnPayload(upstream, payloadModel);");
+		source = replaceOnce(source, "      versionSync?.recordRequest(upstream);\n      return shapeAnthropicOAuthPayload(upstream);", "      const checked = await composeCallerOnPayload(shapeAnthropicOAuthPayload(upstream), payloadModel);\n      versionSync?.recordRequest(checked);\n      return checked;");
+		// Keep the caller's measured fetch. No body rewrite or second request may follow the final payload check.
+		source = replaceOnce(source, "      ...(versionSync ? { fetch: versionSync.fetch } : {}),", "");
+	}
+	if (adapter === "@gotgenes/pi-anthropic-auth@3.4.3" && path === "src/index.ts") source = replaceOnce(source, '  pi.unregisterProvider("anthropic");', '  Object.defineProperty(streamSimple, Symbol.for("pi-dispatch.subscription-adapter"), { value: "@gotgenes/pi-anthropic-auth@3.4.3" });\n  pi.unregisterProvider("anthropic");');
+	if (adapter === "@gotgenes/pi-anthropic-auth@3.4.3" && path === "src/system-prompt-shaping.ts") {
+		for (const [signature, next, returned] of [
+			["export function shapeAnthropicOAuthSystemPrompt(systemPrompt: string): string {", "\n}\n\nfunction isPiOwnedSection", "systemPrompt"],
+			["export function shapeSystemUpdateText(text: string): string | undefined {", "\n}\n\n/**\n * Apply the rule table", "text"],
+		]) {
+			const start = source.indexOf(signature);
+			const end = source.indexOf(next, start);
+			if (start < 0 || end < start) fail("auth-adapter-source-mismatch");
+			source = source.slice(0, start + signature.length) + `\n  return ${returned};` + source.slice(end);
+		}
+		const signature = "export function shapeSystemBlocks(blocks: TextBlock[]): TextBlock[] {";
+		const start = source.indexOf(signature);
+		if (start < 0 || !source.trimEnd().endsWith("}")) fail("auth-adapter-source-mismatch");
+		source = source.slice(0, start + signature.length) + "\n  return blocks;\n}\n";
+	}
+	return source;
+}
+
+// A changed adaptation must be explicitly restaged even when the upstream package version stays the same.
+export const COMMUNITY_ADAPTER_REVISION = sha(JSON.stringify(COMMUNITY_SOURCE_PINS) + adaptCommunitySource.toString());
+
+function regularFile(fs, path) { const stat = fs.lstatSync(path); if (!stat.isFile() || stat.isSymbolicLink()) fail("auth-adapter-unsafe-path"); return stat; }
+export function communityPackageDigest(root, { fs = nodeFs } = {}) {
+	let files = 0;
+	let bytes = 0;
+	const entries = [];
+	function walk(path) {
+		const stat = fs.lstatSync(path);
+		if (stat.isSymbolicLink()) fail("auth-adapter-unsafe-path");
+		if (stat.isDirectory()) { for (const name of fs.readdirSync(path).sort()) if (join(path, name) !== join(root, ".dispatch-auth-adapter.json")) walk(join(path, name)); return; }
+		if (!stat.isFile() || ++files > 20_000 || (bytes += stat.size) > 150 * 1024 * 1024) fail("auth-adapter-tree-unbounded");
+		entries.push(`${relative(root, path).split("\\").join("/")}\0${sha(fs.readFileSync(path))}\0`);
+	}
+	walk(root);
+	return sha(entries.sort().join(""));
+}
+
+function packageCheck(adapter, root, fs) {
+	if (typeof root !== "string" || !isAbsolute(root)) fail("auth-adapter-unsafe-path");
+	const pin = COMMUNITY_AUTH_ADAPTERS[adapter];
+	if (!pin?.community) fail("auth-adapter-unsupported");
+	regularFile(fs, join(root, "package.json"));
+	const pkg = JSON.parse(fs.readFileSync(join(root, "package.json"), "utf8"));
+	if (pkg.name !== pin.package || pkg.version !== pin.version) fail("auth-adapter-version-mismatch");
+	if (adapter === "pi-anthropic-oauth@0.3.6") {
+		const path = join(root, "node_modules/@anthropic-ai/sdk/package.json");
+		regularFile(fs, path);
+		const sdk = JSON.parse(fs.readFileSync(path, "utf8"));
+		if (sdk.name !== "@anthropic-ai/sdk" || sdk.version !== "0.124.0") fail("auth-adapter-version-mismatch");
+	}
+	const seen = new Set();
+	function dependencies(directory) {
+		if (seen.has(directory)) return;
+		seen.add(directory);
+		const manifestPath = join(directory, "package.json");
+		regularFile(fs, manifestPath);
+		const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
+		for (const name of Object.keys(manifest.dependencies ?? {})) {
+			if (!/^(?:@[A-Za-z0-9._-]+\/)?[A-Za-z0-9._-]+$/.test(name)) fail("auth-adapter-dependencies-unresolved");
+			let search = directory;
+			let resolved;
+			while (search === root || search.startsWith(`${root}/`)) {
+				const candidate = join(search, "node_modules", name);
+				if (fs.existsSync(join(candidate, "package.json"))) { resolved = candidate; break; }
+				search = dirname(search);
+			}
+			if (!resolved) fail("auth-adapter-dependencies-unresolved");
+			dependencies(resolved);
+		}
+	}
+	dependencies(root);
+	return pin;
+}
+
+export function prepareCommunityAuthAdapter({ adapter, root, fs = nodeFs }) {
+	const pin = packageCheck(adapter, root, fs);
+	const outputs = {};
+	for (const [path, expected] of Object.entries(COMMUNITY_SOURCE_PINS[adapter])) {
+		regularFile(fs, join(root, path));
+		const original = fs.readFileSync(join(root, path), "utf8");
+		if (sha(original) !== expected) fail("auth-adapter-source-mismatch");
+		outputs[path] = adaptCommunitySource(adapter, path, original);
+	}
+	// Verify every source before changing one; partial staging never receives an approval receipt.
+	for (const [path, source] of Object.entries(outputs)) fs.writeFileSync(join(root, path), source);
+	const digest = communityPackageDigest(root, { fs });
+	const receipt = { version: 1, adapter, adapterRevision: COMMUNITY_ADAPTER_REVISION, package: pin.package, packageVersion: pin.version, digest, sources: Object.fromEntries(Object.entries(outputs).map(([path, source]) => [path, sha(source)])) };
+	fs.writeFileSync(join(root, ".dispatch-auth-adapter.json"), JSON.stringify(receipt));
+	return { adapter, package: pin.package, version: pin.version, digest };
+}
+
+export function verifyCommunityAuthAdapter({ adapter, root, digest, fs = nodeFs }) {
+	if (typeof digest !== "string" || !/^[a-f0-9]{64}$/.test(digest)) fail("auth-adapter-unpinned");
+	const pin = packageCheck(adapter, root, fs);
+	regularFile(fs, join(root, ".dispatch-auth-adapter.json"));
+	const receipt = JSON.parse(fs.readFileSync(join(root, ".dispatch-auth-adapter.json"), "utf8"));
+	if (receipt.version !== 1 || receipt.adapter !== adapter || receipt.adapterRevision !== COMMUNITY_ADAPTER_REVISION || receipt.digest !== digest || communityPackageDigest(root, { fs }) !== digest) fail("auth-adapter-unpinned");
+	for (const path of Object.keys(COMMUNITY_SOURCE_PINS[adapter])) {
+		regularFile(fs, join(root, path));
+		const actual = sha(fs.readFileSync(join(root, path)));
+		if (actual !== receipt.sources?.[path] || actual !== COMMUNITY_ADAPTED_SOURCE_PINS[adapter]?.[path]) fail("auth-adapter-source-mismatch");
+	}
+	return { adapter, package: pin.package, version: pin.version, digest, compatibility: "verified" };
+}

package/src/community-auth-pins.mjs

@@ -0,0 +1,38 @@
+// Published npm artifacts, MIT licenses retained in staged packages and test fixtures.
+export const COMMUNITY_SOURCE_PINS = {
+	"pi-anthropic-oauth@0.3.6": {
+		"LICENSE": "e46cbd8e0af9d9642434b1f2c80e5de17253a6766f0e334e676b1486f470f6f3",
+		"README.md": "2cb469bb8c3e2f5561de322e6758ed3f81b4fe0fd11bf27864b81f5df511fca7",
+		"package.json": "53eb5f1772b1bdf571a8d5395c48be2b5c92fb2aec69743e8b85ae9248167068",
+		"src/auth.ts": "31d1c7df052c5353b1c198fa52a2f0b1b416bc83f6496e3c502b738b3ca4b461",
+		"src/convert.ts": "5263f2f308b674cdde710c087c4d7d8ff8dd19cae965a06a02e4bfcb0c1940d0",
+		"src/index.ts": "32c73197d08cc87bd6e69caea7591cbe116a1630a4777135e330f8d4992078d4",
+		"src/prompt.ts": "e3c7c7d595a96be754cea87e9866ae58756e78e7a8112933c9dc85284fd74a1a",
+		"src/stream.ts": "7b26bd0758d45e80ce8703d1f5a4dd7713cba25d64c76d17f0271424e38aa2b9",
+		"src/usage.ts": "a8c9054514d93fae5e689ddc647fb41d4c91f6a64ccab1c79e59efb2d027d0cb",
+		"src/version.ts": "0de73b8019b0cba6fc315908deac2a406a2b447f991b28c5de3bf2bdf3c56950"
+	},
+	"@gotgenes/pi-anthropic-auth@3.4.3": {
+		"CHANGELOG.md": "89c464b1114b2e317492e8ef0ad0d87c49f6f1378af1d1e81592873ba551edd0",
+		"LICENSE": "2770a907d8f37ee6fcf820b7f9fc2f51ec4de2aeacf2eeaca0d71b5f18938c61",
+		"README.md": "5e111b6319c5824e1df60f886eede5256784b2f41e3545b889c6d23cb0c551aa",
+		"package.json": "a2ccd4a410071897465739bf63fbe91f851e1bf66ae2025926f5c4c29cb6a4bb",
+		"src/account-profile.ts": "1596f0c4d6e416c72057f91380f7cff6ed032738c9419e5a5881958f8a191cc1",
+		"src/anthropic-message.ts": "9e9cd18f9a5126a3cce20baa042751b635cc16280e4e69050026c975950b5699",
+		"src/billing-header.ts": "69c88b5f1eda91a1e070bdc91f07c10c9a607d0ddfabbea4f007f38bb011de55",
+		"src/billing-version-sync.ts": "f7c20f42eb244bac1300fec8fb56133fd10c621c09c5db93fc28ad77f5083f47",
+		"src/claude-code-version.ts": "ab44ae9493f9cc404586961ab2f0a7c1b51123d01cafc8f94e72ff1b8db6d961",
+		"src/constants.ts": "03ac3d5c0d54a3e78ba37fe517da58e3e4632950e3d1d331a80830dd6d50df65",
+		"src/debug.ts": "a06644b3d52c8040b61338820881ab83acda3a0238b177261b9f5d6c4464991b",
+		"src/diagnostics.ts": "6ef582e93c3c8eba564202c2f4c079304561783a4a1dcb08007b206f5fc19880",
+		"src/extension-config.ts": "886446388fb75c0094c78caa4196a58025fe2a72ddab762416792d292723751d",
+		"src/extra-provider-shaping.ts": "ec95df3f3dd34a2a50cc8d790e35f8d774b506634fdb34893db7f654051add87",
+		"src/host-transport.ts": "c5a130e95da77636437695f4bce6987a34790464d66608352def7a6de967adef",
+		"src/index.ts": "b9e46cb59ca1f1d30f85979a65c8d26031672a479b01b47b6682f989f1a71dcb",
+		"src/oauth-transport.ts": "bc3d3c78143093089dfebc9ebdde35ccf9bf3bc1a8db4cb14de25a4b16740deb",
+		"src/request-shaping.ts": "0583250b5f18cd01ac9f030635ecfb0f2b0284fa2c85c51747153bd08efc7722",
+		"src/system-prompt-sections.ts": "9d922f8d9feb7a4b6ef45e12b9b47db04052cdf8a096d4f7dac87d2f30aa9036",
+		"src/system-prompt-shaping.ts": "524d6e877eec2d58b9d0ba9539b89053f7a65bf34f55d33b11611909e517a138",
+		"src/version-rejection.ts": "aac221a3877251fe90d0afba6d9cba65eedd52d360d44e15a2bbe449b544eef1"
+	}
+};

package/src/config.mjs

@@ -7,18 +7,21 @@
 
 import { chmodSync, existsSync, lstatSync, mkdirSync, realpathSync, statSync } from "node:fs";
 import { homedir, hostname, tmpdir } from "node:os";
-import { delimiter, isAbsolute, posix } from "node:path";
-import { DEFAULT_BACKEND, backendRefusals, parseBackendFloor, parseBackendList } from "./backends.mjs";
+import { delimiter, isAbsolute, posix, relative, resolve, sep } from "node:path";
+import { SESSION_KEY_PATTERN } from "./session-key.mjs";
+import { DEFAULT_BACKEND, OPENSHELL_BACKENDS, OPENSHELL_BIND_BACKEND, backendRefusals, parseBackendFloor, parseBackendList } from "./backends.mjs";
 import { egressArmed, egressProxyName } from "./egress.mjs";
 import { MINTED_TOKEN_VARS, RECEIVER_ONLY_VARS } from "./forges.mjs";
 import { SWEEP_INTERVAL_HOURS, SWEEP_INTERVAL_MAX_HOURS } from "./retention-sweep.mjs";
-import { parseSecretProfiles } from "./secret-profiles.mjs";
+import { RUN_MIRROR_MAX_RUNS_DEFAULT, RUN_MIRROR_MAX_RUNS_MAX, RUN_MIRROR_MAX_RUNS_MIN } from "./retention.mjs";
+import { DOCTOR_INTERVAL_HOURS, DOCTOR_INTERVAL_MAX_HOURS } from "./doctor-spawner.mjs";
+import { parseSecretBundleProfiles, parseSecretProfiles } from "./secret-profiles.mjs";
 import { WAIT_AFTER_MAX_DEFAULT_MS, WAIT_INTERVAL_FLOOR_MS, parseWaitProfiles } from "./wait-for.mjs";
 import { imageRefProblem } from "./image-ref.mjs";
 import { jobSizeDefaults } from "./job-size.mjs";
 import { hostBudgetSettings } from "./host-budget.mjs";
 import { CONTAINER_ENV_NAMES, KEYLESS_ENV_NAME, RUNNER_ENV_NAMES } from "./reserved-env.mjs";
-import { modelListProblem } from "./model-ref.mjs";
+import { modelIdProblem, modelListProblem, providerIdProblem } from "./model-ref.mjs";
 import { WORKER_NAME_RE } from "./worker-name.mjs";
 
 export { WORKER_NAME_RE };
@@ -170,7 +173,7 @@ const WORKER_ONLY_WHY = Object.freeze({
  */
 export const EGRESS_ENV_VARS = new Set(["HTTPS_PROXY", "HTTP_PROXY", "NO_PROXY", "NODE_USE_ENV_PROXY"]);
 
-function forwardEnvList(raw, egressArmed = false) {
+export function forwardEnvList(raw, egressArmed = false) {
 	const names = commaList(raw);
 	const minted = names.filter((n) => MINTED_TOKEN_VARS.has(n));
 	if (minted.length > 0) {
@@ -223,6 +226,33 @@ function forwardEnvList(raw, egressArmed = false) {
 	return names;
 }
 
+/**
+ * `PI_MODEL` (issue #673): the deployment's default model, held to the one model-id rule every other source of a
+ * job's model is (`modelIdProblem`: a trigger's `run.model`, the settings overlay, `pi-dispatch run --model`). The
+ * run record writes whichever source won, and the console's runs filter accepts every id that rule does, so a run
+ * on the env default is never one the console cannot find. Refused at boot, like every other knob here; doctor
+ * fails the same row first. SET BUT EMPTY IS REFUSED, not read as unset: before #673 an empty value survived
+ * `??` and every job ran on a model named "", and an `.env` line `PI_MODEL=` is more likely a template left
+ * half-filled than a wish for the default. The value is never echoed. The original case is kept: pi resolves
+ * ids exactly, and only the check lowercases.
+ */
+export function modelFrom(env) {
+	const raw = env.PI_MODEL;
+	if (raw === undefined) return DEFAULT_MODEL;
+	const problem = modelIdProblem(raw);
+	if (problem !== null) throw configError(`invalid PI_MODEL: it ${problem} (leave it unset for the default)`);
+	return raw;
+}
+
+/** `PI_PROVIDER` (issue #673), by `modelFrom`'s rule against the provider shape. */
+export function providerFrom(env) {
+	const raw = env.PI_PROVIDER;
+	if (raw === undefined) return DEFAULT_PROVIDER;
+	const problem = providerIdProblem(raw);
+	if (problem !== null) throw configError(`invalid PI_PROVIDER: it ${problem} (leave it unset for the default)`);
+	return raw;
+}
+
 /**
  * PI_ALLOWED_MODELS (issue #502): the deployment's allowed-model list, comma-separated `provider/model`, the
  * grammar a trigger's `run.models` has (`modelListProblem`, model-ref.mjs), so a list means the same thing in
@@ -289,6 +319,27 @@ function backendFloorOf(env) {
 	}
 }
 
+/**
+ * The openshell venue's two settings (issue #684): `PI_OPENSHELL_ENGINE`, the gateway's compute driver whose CLI reads the
+ * job image (`docker`, the default, or `podman`, the two v1 admits), and `PI_OPENSHELL_GATEWAY`, a registered gateway's
+ * name (unset: the CLI's active one). Judged only while the venue is blessed, as doctor judges them, so a value left in
+ * the env of a deployment that never chose the venue refuses nothing; while blessed a bad one stops the boot here rather
+ * than every job at its start.
+ *
+ * Issue #686 part 3: both OpenShell venues share them (one gateway, one engine), so either name in PI_BACKENDS judges
+ * them. Bind mode is refused on the Podman driver: it was measured on the Docker driver only, and a rootless Podman maps
+ * the sandbox's uid away from the worker's, so a bind would be neither readable nor removable by the uid that owns it.
+ */
+export function openshellFrom(env, backends) {
+	const engine = env.PI_OPENSHELL_ENGINE === undefined || env.PI_OPENSHELL_ENGINE === "" ? "docker" : env.PI_OPENSHELL_ENGINE;
+	const gateway = env.PI_OPENSHELL_GATEWAY || null;
+	if (!OPENSHELL_BACKENDS.some((name) => backends.includes(name))) return { engine, gateway };
+	if (engine !== "docker" && engine !== "podman") throw configError(`PI_OPENSHELL_ENGINE must be docker or podman, the compute drivers the openshell venue admits (got ${JSON.stringify(engine)})`);
+	if (gateway !== null && !/^[A-Za-z0-9][A-Za-z0-9_.-]{0,62}$/.test(gateway)) throw configError("PI_OPENSHELL_GATEWAY must be a registered gateway's name (`openshell gateway list`), or unset for the CLI's active gateway");
+	if (engine === "podman" && backends.includes(OPENSHELL_BIND_BACKEND)) throw configError(`PI_BACKENDS names ${OPENSHELL_BIND_BACKEND} with PI_OPENSHELL_ENGINE=podman: bind mode was measured on OpenShell's Docker driver only, and a rootless Podman maps the sandbox's uid away from the worker's, so its binds would not be the worker's to read or remove. Use the docker driver, or the copy venue (openshell)`);
+	return { engine, gateway };
+}
+
 /**
  * THE BOOT REFUSAL. A deployment whose configuration needs something its backends cannot provide is refused
  * here, named backend by named property, rather than discovering it per job.
@@ -389,6 +440,37 @@ export function globalExtensionsEnabled(env) {
 	);
 }
 
+/**
+ * Does the worker publish each running job's transcript to Valkey (issue #643, `DES-LIVE-TRANSCRIPT-STREAM`)?
+ *
+ * ON by default: watching a run is what the console and the panel are for, and what leaves the host is redacted,
+ * capped at 2,000 entries and gone 24 hours after the run. `PI_LIVE_TRANSCRIPTS=0` is the opt-out, for a deployment
+ * that must keep transcript text (PII: issue and tool text) off its Valkey. Strict, for `globalExtensionsEnabled`'s
+ * reason: `PI_LIVE_TRANSCRIPTS=false` read as ON would be an operator believing PII stays on the host while it does
+ * not, so any value but unset, empty, "1" or "0" refuses to boot.
+ */
+export function liveTranscriptsEnabled(env) {
+	const raw = env.PI_LIVE_TRANSCRIPTS;
+	if (raw === undefined || raw === "" || raw === "1") return true;
+	if (raw === "0") return false;
+	throw configError(`invalid PI_LIVE_TRANSCRIPTS: ${JSON.stringify(raw)} (want "0" to keep transcripts off Valkey, or leave it unset to publish them)`);
+}
+
+/**
+ * Does the worker read each running job's progress from its transcript (issue #644, `DES-RUN-PROGRESS`)? Turns, steps,
+ * the tool running and a declared phase, to the `run:progress:` key, the run record's `progress` and, where a trigger
+ * opts in, the forge status comment. ON by default, like the live transcript. `PI_RUN_PROGRESS=0` turns all three off.
+ * Strict, for `liveTranscriptsEnabled`'s reason: any value but unset, empty, "1" or "0" refuses to boot. Independent of
+ * `PI_LIVE_TRANSCRIPTS`: with transcripts off and progress on, the file is still tailed and no transcript text is
+ * published (the key carries a redacted phase name and note only, from a job that opted into `run.progress`).
+ */
+export function runProgressEnabled(env) {
+	const raw = env.PI_RUN_PROGRESS;
+	if (raw === undefined || raw === "" || raw === "1") return true;
+	if (raw === "0") return false;
+	throw configError(`invalid PI_RUN_PROGRESS: ${JSON.stringify(raw)} (want "0" to turn run progress off, or leave it unset to read it)`);
+}
+
 /**
  * Parse the worker's config from `env` (default process.env). Every MONEY default is conservative:
  * spend controls (`PI_DAILY_CAP`, `PI_MAX_TURNS`) exist to bound money, so they default low, and a
@@ -402,12 +484,14 @@ export function globalExtensionsEnabled(env) {
  * spend caps above, the per-job token scoping, or the admin-extension recursion block.
  */
 export function loadConfig(env = process.env, { fileExists = existsSync } = {}) {
-	const model = env.PI_MODEL ?? DEFAULT_MODEL; // an exact model version, not a moving alias; deterministic per CONST-PI-VERSION-PINNED
+	const model = modelFrom(env); // an exact model version, not a moving alias; deterministic per CONST-PI-VERSION-PINNED. Shape-checked (#673)
 	// #227. Hoisted above the object because `defaultBackend` INDEXES `backends`, and a property cannot read
 	// a sibling of the literal it is in. (Calling the parser twice would be harmless -- `egressEnabled` is
 	// called twice a few properties down for the same reason -- so this is about the index, not the throw.)
 	const backends = backendSet(env);
 	const backendFloor = backendFloorOf(env);
+	// Issue #761: hoisted for the same reason, `consoleAuditRetentionDays` inherits it when its own key is unset.
+	const logRetentionDays = nonNegativeInt(env, "PI_LOG_RETENTION_DAYS", 30); // 0 = keep forever
 	const config = {
 		valkeyUrl: env.VALKEY_URL ?? DEFAULT_VALKEY_URL,
 		// Issue #57. What this machine calls itself: the key of its registry row, the `host` on every log
@@ -423,7 +507,7 @@ export function loadConfig(env = process.env, { fileExists = existsSync } = {})
 		weeklyCap: optionalBoundedInt(env, "PI_WEEKLY_CAP", 1), // REQ-SPEND-CAPS-MULTI-WINDOW; null = weekly window disabled
 		monthlyCap: optionalBoundedInt(env, "PI_MONTHLY_CAP", 1), // null = monthly window disabled
 		softHoldPct: optionalBoundedInt(env, "PI_SOFT_HOLD_PCT", 1, 99), // null = soft-hold band disabled
-		provider: env.PI_PROVIDER ?? DEFAULT_PROVIDER,
+		provider: providerFrom(env),
 		model,
 		maxTurns: positiveInt(env, "PI_MAX_TURNS", 30), // pi has no turn limit; we impose one
 		allowedModels: allowedModelsFrom(env), // issue #502: the deployment's allowed-model list; null = unrestricted. ENV ONLY, never the settings overlay
@@ -464,6 +548,8 @@ export function loadConfig(env = process.env, { fileExists = existsSync } = {})
 		egressProxy: egressProxyName(env), // one derivation, shared with the panel's sandbox (#277)
 		forwardEnv: forwardEnvList(env.PI_FORWARD_ENV, egressEnabled(env)), // extra host var NAMES to forward (e.g. a custom provider's key); explicit allowlist, GitHub token names refused
 		authFromPi: env.PI_AUTH_FROM_PI !== "0", // ON by default: use the key in ~/.pi/agent/auth.json when the env has none (api-key only). PI_AUTH_FROM_PI=0 forces env-only.
+		authConnectionsFile: env.PI_AUTH_CONNECTIONS_FILE ? resolve(env.PI_AUTH_CONNECTIONS_FILE) : null,
+		authStoreDir: env.PI_AUTH_STORE_DIR ? resolve(env.PI_AUTH_STORE_DIR) : null,
 		jobsDir: jobsDirPath(env),
 		// REQ-RESURRECTABLE-SANDBOX. `||` (not `??`) so an empty string falls back, matching logsDir.
 		sandboxDir: env.PI_SANDBOX_DIR || defaultSandboxDir(env),
@@ -485,13 +571,31 @@ export function loadConfig(env = process.env, { fileExists = existsSync } = {})
 		logsDir: logsDirPath(env), // || (not ??) inside logsDirPath, so an empty string falls back to the default
 		settingsFile: settingsFilePath(env), // || (not ??) inside settingsFilePath, so an empty string falls back; INT-CONFIG-OVERLAY-CONTRACT
 		captureJobLogs: env.PI_CAPTURE_JOB_LOGS === "1", // no-pii-in-logs: raw job-log capture is opt-in; anything but "1" is off
-		logRetentionDays: nonNegativeInt(env, "PI_LOG_RETENTION_DAYS", 30), // 0 = keep forever
+		liveTranscripts: liveTranscriptsEnabled(env), // issue #643: the live transcript stream, ON unless PI_LIVE_TRANSCRIPTS=0
+		runProgress: runProgressEnabled(env), // issue #644: run progress (key, record field, status comment), ON unless PI_RUN_PROGRESS=0
+		logRetentionDays,
+		// Issue #761: the console audit trail's own retention (`console-audit/` month files), so that shortening the run
+		// history for privacy does not shorten the security trail with it. Unset or empty INHERITS `logRetentionDays`,
+		// 0 included (keep forever), which is exactly what every release before this one did; "0" keeps forever; any other
+		// non-negative integer is days; anything else refuses to boot naming the key. The worker reads it because the worker
+		// runs the sweep (start.mjs `makeConsoleAuditReaper`); the console never reaps. Not a `PI_CONSOLE_*` key the console
+		// reads (doctor's `CONSOLE_ENV_KEYS`): setting only this one must not make doctor think a console exists here.
+		consoleAuditRetentionDays: nonNegativeInt(env, "PI_CONSOLE_AUDIT_RETENTION_DAYS", logRetentionDays),
+		// Issue #761: how many runs the shared run index (`runs:index`) keeps. Every writer trims the shared index to its
+		// OWN cap, so in a fleet the lowest setting wins, and each trim raises the fleet's horizon (`runs:horizon`) so a
+		// reader sees the cut rather than idle time. Bounded both ways (retention.mjs says why): a value out of bounds, or
+		// not a plain integer, refuses to boot naming the key; unset or empty is the default.
+		runMirrorMaxRuns: boundedInt(env, "PI_RUN_MIRROR_MAX_RUNS", RUN_MIRROR_MAX_RUNS_DEFAULT, RUN_MIRROR_MAX_RUNS_MIN, `an integer ${RUN_MIRROR_MAX_RUNS_MIN}-${RUN_MIRROR_MAX_RUNS_MAX}`, RUN_MIRROR_MAX_RUNS_MAX),
 		// issue #292, OQ-007: how often the three host-side retention sweeps re-run while the worker is up.
 		// 0 = BOOT-ONLY, which is byte-identical to every version before this one -- start.mjs does not even
 		// CONSTRUCT the sweep at 0, so no timer, no closer and no second invocation of any reaper is reachable.
 		// The ceiling is enforced because setInterval clamps a delay past 2^31-1 ms to 1ms (see
 		// SWEEP_INTERVAL_MAX_HOURS); a hot loop over the filesystem is a worse failure than a slow sweep.
 		sweepIntervalHours: nonNegativeInt(env, "PI_SWEEP_INTERVAL_HOURS", SWEEP_INTERVAL_HOURS, SWEEP_INTERVAL_MAX_HOURS),
+		// Issue #712: how often the worker runs `pi-dispatch doctor --publish` as a child (the first run shortly after boot).
+		// 0 = never, and start.mjs then constructs nothing; capped for the sweep's reason (setInterval's clamp).
+		updatesLook: env.PI_UPDATES_LOOK === "1",
+		doctorIntervalHours: nonNegativeInt(env, "PI_DOCTOR_INTERVAL_HOURS", DOCTOR_INTERVAL_HOURS, DOCTOR_INTERVAL_MAX_HOURS),
 		// REQ-RESUMABLE-SESSION. NO DEFAULT AT ALL, deliberately unlike every sibling above: unset means the
 		// feature is unavailable, and a trigger that armed run.resume then refuses PRE-SPEND rather than
 		// running silently without persistence. A transcript is the most PII-bearing artifact this system
@@ -500,6 +604,9 @@ export function loadConfig(env = process.env, { fileExists = existsSync } = {})
 		// not rest on where the other stores default: issue #290 moved logs and settings to ~/.pi-dispatch
 		// and this one still has no default, for the same reason it never did.
 		sessionsDir: env.PI_SESSIONS_DIR || null,
+		// Issue #645: where workspace checkpoints live. Resolved, defaulting INSIDE the session store and null when neither
+		// is set; `checkpointsDirPath` says why. Read here so doctor and the (later) checkpointer cannot derive it twice.
+		checkpointsDir: checkpointsStoreRefusingCollision(env),
 		sessionsTtlDays: nonNegativeInt(env, "PI_SESSIONS_TTL_DAYS", 14), // 0 = keep forever
 		// A bound on how large a transcript may be before it stops being resumed. Not disk hygiene: an
 		// oversized transcript is a prefill an operator never sized PI_MAX_TOKENS for.
@@ -529,6 +636,12 @@ export function loadConfig(env = process.env, { fileExists = existsSync } = {})
 		// hazard `delimitedList` above exists for, arriving from the other side. Unset = the feature is off and
 		// any trigger naming secrets refuses pre-spend, which is why there is no default profile to fall into.
 		secretProfiles: parseSecretProfiles(env.PI_SECRET_PROFILES),
+		// Issue #704. The operator's declared BUNDLE resolvers, `name:absolute-path:format` (dotenv or json, required).
+		// Its own variable, never a flag on PI_SECRET_PROFILES (secret-profiles.mjs says why). Unset = no bundle can
+		// resolve on this host, so a project bound to one refuses pre-spend as secret-bundle-profile-unknown. Bounded by
+		// the same PI_SECRET_RESOLVER_ROOTS below for its overlay half, and timed by PI_SECRET_RESOLVE_TIMEOUT_MS: one
+		// call per job, so one timeout, never a multiple of it.
+		secretBundleProfiles: parseSecretBundleProfiles(env.PI_SECRET_BUNDLES),
 		// The directories a resolver may live in. Default [] FAILS CLOSED exactly as dispatchRunRoots does, and
 		// for a sharper version of its reason: this bounds paths that can arrive from the settings overlay,
 		// which is not the reviewed artifact `triggers.json` is. Unset means the panel can declare no profile
@@ -587,6 +700,7 @@ export function loadConfig(env = process.env, { fileExists = existsSync } = {})
 		gitlab: loadGitLabAuth(env),
 		forgejo: loadForgejoAuth(env),
 		azure: loadAzureAuth(env),
+		openshell: openshellFrom(env, backends), // issue #684: the openshell venue's engine and gateway
 	};
 
 	// #227, and it runs AFTER the object is built rather than inside it: the refusal reads `egress` as well
@@ -710,11 +824,10 @@ export function loadGitHubAuth(env, fileExists) {
 	return { source, patVar, appId, installationId, privateKeyPath, privateKey };
 }
 
-// env-internal TMPDIR, TEMP: the OS temp dir, read to place the default job, sandbox and graph paths
+// env-internal TMPDIR, TEMP: the OS temp dir, read to place the default job and sandbox paths
 // below -- and, only on a host with no home directory to name, as the last-resort fallback for the
 // durable state root. Not a variable of this project's and not a deployment knob: PI_JOBS_DIR,
-// PI_LOGS_DIR, PI_GRAPH_DIR and PI_SETTINGS_FILE are how an operator moves any of them, and
-// .env.example says so.
+// PI_LOGS_DIR and PI_SETTINGS_FILE are how an operator moves any of them, and .env.example says so.
 
 /** This process's effective uid, the one its files are created as, or null where the platform has none (Windows). */
 function currentUid() {
@@ -722,7 +835,7 @@ function currentUid() {
 }
 
 /**
- * The per-account root under the OS temp dir that the default jobs, sandbox and graph paths live in (issue #464):
+ * The per-account root under the OS temp dir that the default jobs and sandbox paths live in (issue #464):
  * `<tmp>/pi-dispatch-<uid>`, or `<tmp>/pi-dispatch` where the platform has no uid (Windows, whose TEMP is per user).
  *
  * Per ACCOUNT because the OS temp dir is shared by every account on a host. The old `<tmp>/pi-dispatch` was created by
@@ -805,7 +918,7 @@ export function sandboxDirOwnerFix(dir) {
 
 /**
  * Secure the per-account root (`ensureAccountTempRoot`) when `path` lies in it, else do nothing (issue #464). The one
- * rule for every default that can live there: the jobs, sandbox and graph dirs, and, on an account with no home, the
+ * rule for every default that can live there: the jobs and sandbox dirs, and, on an account with no home, the
  * run history and the settings overlay (`defaultStateDir`). Returns whether it did.
  */
 export function ensureUnderAccountRoot(path, { env = process.env, uid = currentUid(), fs = { mkdirSync, lstatSync, statSync, chmodSync } } = {}) {
@@ -825,8 +938,8 @@ export function jobsDirOwnerFix(dir, inRoot) {
 
 /**
  * Create `root` mode 0700 when it is absent, and refuse it unless it is a real directory (never a symlink, lstat) owned
- * by `uid`; one this account owns with group or other bits is tightened to 0700. Exported for the admin's graph dir,
- * which lives under the same root. Throws a `configError` naming the path and the remedy.
+ * by `uid`; one this account owns with group or other bits is tightened to 0700. Exported for its tests and for
+ * doctor's reading of its refusals. Throws a `configError` naming the path and the remedy.
  *
  * lstat FIRST, never a recursive mkdir first (gate round 1, measured on Fedora 44): with fs.protected_symlinks on (the
  * default on Fedora and Ubuntu), following another account's symlink under the sticky /tmp fails EACCES, and the
@@ -1037,6 +1150,67 @@ export function logsDirPath(env = process.env, home) {
 	return env.PI_LOGS_DIR || defaultLogsDir(env, home);
 }
 
+/** The checkpoint store's directory name under PI_SESSIONS_DIR when PI_CHECKPOINTS_DIR does not name one (issue #645). */
+export const CHECKPOINTS_SUBDIR = ".checkpoints";
+
+/**
+ * The checkpoint store (issue #645): `PI_CHECKPOINTS_DIR`, else `<PI_SESSIONS_DIR>/.checkpoints`, else `null`, resolved
+ * against the working directory so doctor and the worker name the same absolute path.
+ *
+ * DEFAULTED INSIDE THE SESSION STORE, and only there, on `sessionsDir`'s own reasoning: a checkpoint holds a job's
+ * workspace, which is as PII-bearing as a transcript, so it gets no default of its own on a path nobody chose (a temp
+ * dir above all). Where the operator already chose a private host-only directory for transcripts, a sibling inside it
+ * inherits that decision. The DOT name keeps it out of the session reaper, which judges only key-shaped names and skips
+ * dot names silently (`reapSessions`), so the session TTL never removes a checkpoint. `null` means the feature has
+ * nowhere to write, and whatever uses it says so rather than writing somewhere.
+ */
+export function checkpointsDirPath(env = process.env) {
+	if (env.PI_CHECKPOINTS_DIR) return resolve(env.PI_CHECKPOINTS_DIR);
+	if (env.PI_SESSIONS_DIR) return resolve(env.PI_SESSIONS_DIR, CHECKPOINTS_SUBDIR);
+	return null;
+}
+
+/**
+ * Why the checkpoint store and the session store cannot share this layout, or `null` when they can (issue #645). Both
+ * stores name their entries by 32 lower-case hex characters and each sweeps its own, so a layout where one store's
+ * entries are the other's makes each sweep delete the other's folders: the session reaper removes a key-shaped
+ * directory once it ages, whatever is inside it. Three layouts collide:
+ *
+ *   - the same directory (`PI_CHECKPOINTS_DIR` equal to `PI_SESSIONS_DIR`);
+ *   - the session store INSIDE the checkpoint store, where the checkpoint sweep judges the session store's folders;
+ *   - the checkpoint store inside the session store BELOW a key-shaped name (`<sessions>/<32 hex>[/...]`), which the
+ *     session reaper takes for a session key and removes whole.
+ *
+ * The default (`<sessions>/.checkpoints`) and any other non-key child are fine: the reaper judges only key-shaped names
+ * and skips dot names silently. Compared as resolved paths, the way both stores open them; a link between the two is
+ * not followed, which a later realpath-based check could add if a layout ever needs it.
+ */
+export function checkpointStoreCollision(sessionsDir, checkpointsDir) {
+	if (!sessionsDir || !checkpointsDir) return null;
+	const sessions = resolve(sessionsDir);
+	const checkpoints = resolve(checkpointsDir);
+	if (sessions === checkpoints) return "they are the same directory";
+	const inside = (child, parent) => {
+		const rel = relative(parent, child);
+		// `..` as a whole first segment only: a child literally named `..notes` is inside.
+		return rel !== "" && rel !== ".." && !rel.startsWith(`..${sep}`) && !isAbsolute(rel) ? rel : null;
+	};
+	if (inside(sessions, checkpoints) !== null) return "the session store is inside the checkpoint store";
+	const below = inside(checkpoints, sessions);
+	if (below !== null && SESSION_KEY_PATTERN.test(below.split(sep)[0])) return "the checkpoint store is inside a session key's directory, which the session reaper removes whole";
+	return null;
+}
+
+/** `checkpointsDirPath`, refused at load where it collides with the session store (`checkpointStoreCollision`). */
+function checkpointsStoreRefusingCollision(env) {
+	const dir = checkpointsDirPath(env);
+	const why = checkpointStoreCollision(env.PI_SESSIONS_DIR || null, dir);
+	if (why) {
+		throw configError(`PI_CHECKPOINTS_DIR (${dir}) and PI_SESSIONS_DIR (${resolve(env.PI_SESSIONS_DIR)}) collide: ${why}, so the session reaper and the checkpoint sweep would delete each other's folders. Unset PI_CHECKPOINTS_DIR (the default is <PI_SESSIONS_DIR>/.checkpoints) or point it at a directory outside PI_SESSIONS_DIR`);
+	}
+	return dir;
+}
+
 /**
  * The resolved settings-overlay path. Lives HERE, beside `logsDirPath`, so the pair of durable stores has
  * one derivation each and `loadConfig` does not open-code either. `runtime-settings.mjs` re-exports it, so
@@ -1200,7 +1374,7 @@ export function defaultWorkerName() {
  * typed is refused loudly and never quietly altered. Defaulting is a convenience, so it must not be able
  * to fail; declaring is a statement, so a typo in it must not become a different machine's name.
  */
-function workerName(env) {
+export function workerName(env) {
 	const declared = env.PI_WORKER_NAME;
 	if (declared === undefined || declared === "") return defaultWorkerName();
 	if (!WORKER_NAME_RE.test(declared)) {
@@ -1212,19 +1386,6 @@ function workerName(env) {
 	return declared;
 }
 
-export function defaultGraphDir(env = process.env, uid = currentUid()) {
-	// Under the OS temp dir by default, beside logs/ and jobs/ -- the admin's graph HTML artifact
-	// (issue #54) is host-side display output on the defaultLogsDir doctrine, and deliberately NOT
-	// inside logsDir: INT-RUN-HISTORY-FILE-CONTRACT names that directory's filename shape, and a
-	// stray .html beside the sidecars would widen a contract for a file that is not a record.
-	// Overridable with PI_GRAPH_DIR; exported so the admin resolves the same default without
-	// loadConfig, like defaultSandboxDir above.
-	// Issue #464: under the per-account root, for the jobs dir's reason: another account's `<tmp>/pi-dispatch` made
-	// this one uncreatable.
-	return `${accountTempRoot(env, uid)}/graph`;
-}
-
-
 /**
  * The worker's Azure DevOps auth config, or `null` when none is configured -- same presence rule as the
  * other two optional forges.