@edgehero/pi-dispatch 4.3.0 → 5.0.0

package/src/bullmq-internals.mjs

@@ -0,0 +1,208 @@
+/**
+ * BullMQ's internals, in one place (issue #761): every fact this project reads or writes past BullMQ's public API. The
+ * key names (`bull:<queue>:<type>`), the job hash's field names, the meta hash's flags, the delayed set's score, the
+ * `repeat:<scheduler>:<slot>` id a job scheduler mints (and this project's hand-fire id, `manual:<id>:<ms>`, which is
+ * built to the same shape), and the two rules a custom job id must pass. Every other file imports them from here, so a
+ * BullMQ bump that moves one of them is a change to one file, and `worker/test/bullmq-internals.test.mjs` refuses a
+ * second spelling anywhere in the source trees.
+ *
+ * Each fact was read from the shipped files of bullmq `BULLMQ_VERIFIED_VERSION`, cited beside it, and is pinned three
+ * ways: the unit test holds the text of each cited file, the integration test holds the behaviour against a real
+ * Valkey, and a pins test holds every workspace's bullmq, ioredis and cron-parser pin to the lockfile and to that
+ * version. A bump re-reads the cited files; the tests say which one moved.
+ *
+ * A LEAF: it imports nothing, so the receiver and the admin console import it (`@edgehero/pi-dispatch/bullmq-internals`)
+ * without the worker's dependencies. It never names the library in a string either: valkey-clients.test.mjs allows that
+ * only in the files that construct the library's classes.
+ *
+ * What stays where it is: BullMQ's public API (`Queue`, `Worker`, `getJobs`, the state names), which semver governs and
+ * a bump's changelog announces, and this project's own names (`QUEUE`, `hostQueueName`, `sanitizeJobId`).
+ */
+
+/** The bullmq release every fact below was read from. The installed package must be this version (the pins test). */
+export const BULLMQ_VERIFIED_VERSION = "5.80.4";
+
+/** The key prefix every queue's keys share (queue-keys.js: `constructor(prefix = 'bull')`). */
+export const BULL_PREFIX = "bull";
+
+/** The per-queue keys this project reads, each `bull:<queue>:<type>` (queue-keys.js `getKeys`). */
+export const QUEUE_KEY = Object.freeze({ meta: "meta", wait: "wait", paused: "paused", delayed: "delayed", prioritized: "prioritized", repeat: "repeat" });
+
+/** A queue's key for `type` (queue-keys.js `toKey`: `${prefix}:${name}:${type}`). A job's own hash is `type` = its id. */
+export const queueKey = (queueName, type, prefix = BULL_PREFIX) => `${prefix}:${queueName}:${type}`;
+
+/** A queue's key for `type`: the queue's own `toKey` where it has one (it knows its prefix), else `queueKey`. */
+export function queueKeyOf(q, type) {
+	return typeof q?.toKey === "function" ? q.toKey(type) : queueKey(q?.name, type);
+}
+
+/** A job's own hash in queue `q`: BullMQ keys it by the bare id under the queue's prefix. */
+export const jobKeyOf = (q, jobId) => queueKeyOf(q, jobId);
+
+/**
+ * The fields of a job's own hash this project reads, as job.js `fromJSON` reads them: its kind (`name`), its data, its
+ * enqueue instant, its dedup id (`deid`), its scheduler (`rjk`), its attempts (`atm`, and the older `attemptsMade`,
+ * which `fromJSON` reads first) and its stall count (`stc`).
+ */
+export const JOB_FIELD = Object.freeze({ name: "name", data: "data", timestamp: "timestamp", dedupId: "deid", schedulerKey: "rjk", attempts: "atm", attemptsOld: "attemptsMade", stalls: "stc" });
+
+/** The meta hash's fields: the pause flag (pause-7.lua writes `1`, a resume deletes it) and the library's version. */
+export const META_FIELD = Object.freeze({ paused: "paused", version: "version" });
+
+/**
+ * Whether a queue's meta `paused` field says paused, as BullMQ reads it: the field's PRESENCE (`HEXISTS` in
+ * isQueuePaused.lua, and getTargetQueueList.lua, where any string is true), whatever it holds, so `"0"` and `""` are
+ * paused too. `cmd` is the command the reader sent: `HEXISTS` answers 1 for a field that is there, `HGET` the value,
+ * null for none.
+ */
+export function isPausedFlag(cmd, value) {
+	if (cmd === "hexists") return value === 1 || value === "1";
+	return value !== null && value !== undefined;
+}
+
+/**
+ * The version in a meta hash's `version` field (`libName:version`, queue.js `metaValues`), or null. NOT a fleet fact:
+ * every process that constructs a Queue writes it, so in a fleet of mixed versions it holds whichever wrote last. Only
+ * the integration test reads it, to prove the library it ran against is the verified one.
+ */
+export function metaVersionOf(raw) {
+	const m = typeof raw === "string" ? /^bullmq:(\d+\.\d+\.\d+)$/.exec(raw) : null;
+	return m ? m[1] : null;
+}
+
+/** A SCAN pattern for the meta key of every queue whose name starts with `namePrefix`. */
+export const metaKeyPattern = (namePrefix) => queueKey(`${namePrefix}*`, QUEUE_KEY.meta);
+
+/**
+ * The queue a meta key belongs to (`bull:<name>:meta`), or null. A queue's name holds no `:` (queue-base.js refuses
+ * one), so a key whose middle does is some other key a pattern's `*` matched: a job's own hash, whose custom id may
+ * hold two colons (`bull:pi-jobs:a:b:meta`).
+ */
+export function queueNameOfMetaKey(key) {
+	if (typeof key !== "string") return null;
+	const head = `${BULL_PREFIX}:`;
+	const tail = `:${QUEUE_KEY.meta}`;
+	if (!key.startsWith(head) || !key.endsWith(tail) || key.length <= head.length + tail.length) return null;
+	const name = key.slice(head.length, -tail.length);
+	return name.includes(":") ? null : name;
+}
+
+/** A delayed job's score is `due * 4096 + a counter` (getDelayedScore.lua: `delayedTimestamp * 0x1000`). */
+export const DELAYED_SCORE_SHIFT = 4096;
+
+/**
+ * The latest due time `dueOfScore` passes on: a copy of `DEFER_TIME_MAX_MS` (defer-reasons.mjs, which says why), pinned
+ * equal to it by a test, because this module imports nothing.
+ */
+export const SCORE_DUE_MAX_MS = 253402300799999;
+
+/**
+ * A delayed zset score's due time in ms, or null for anything that is not a non-negative number whose quotient is an
+ * instant a console can draw (at most `SCORE_DUE_MAX_MS`): a score anything with the URL wrote can be any double, and a
+ * due time past that range has no date to show. Only a number or a non-empty string is a score: `Number(null)` and
+ * `Number("")` are 0, and a ZSCORE of a missing member (null) must read as no due time, never as one at the epoch.
+ */
+export const dueOfScore = (score) => {
+	if (typeof score !== "number" && !(typeof score === "string" && score.trim() !== "")) return null;
+	const n = Number(score);
+	if (!Number.isFinite(n) || n < 0) return null;
+	const due = Math.floor(n / DELAYED_SCORE_SHIFT);
+	return Number.isSafeInteger(due) && due <= SCORE_DUE_MAX_MS ? due : null;
+};
+
+/** The prefix of every job a job scheduler mints (job-scheduler.js `getSchedulerNextJobId`, addJobScheduler-11.lua). */
+export const SCHEDULER_JOB_PREFIX = "repeat";
+
+/**
+ * The prefix of a cron trigger's run fired by hand (`manualTriggerJobId`, job-id.mjs). This project's own, but built
+ * to BullMQ's shape: a custom id with a `:` must split into exactly three parts (`customJobIdAllowed`), as a
+ * scheduler's does, so every reader of a cron run reads both with one grammar.
+ */
+export const MANUAL_JOB_PREFIX = "manual";
+
+/** The most digits a slot holds: a JavaScript Date's last instant, 8.64e15 ms, is 16 digits. */
+export const SLOT_DIGITS_MAX = 16;
+
+/** What a scheduler's run id adds to its scheduler's id: the prefix, two colons and a slot. */
+export const SCHEDULER_JOB_ID_OVERHEAD = `${SCHEDULER_JOB_PREFIX}::`.length + SLOT_DIGITS_MAX;
+
+/**
+ * The longest scheduler id a reader of Valkey or a surface that prints one takes, and the charset (the triggers
+ * loader's `ID_CHARSET`, which refuses a `:`). Bounded where it is read, because anything holding the Valkey's URL can
+ * write a scheduler key or a job id, and a surface prints it in running text. 200, not the loader's `CRON_ID_MAX`: a
+ * scheduler that an earlier release upserted for a longer id is still in Valkey after an upgrade (every release before
+ * issue #761 took a cron id of any length), and a surface must still name it so an operator can find and remove it. A
+ * reader of run RECORDS takes any length (`cronRunOf` without `idMax`): a record is this deployment's own history.
+ */
+export const SCHEDULER_ID_MAX = 200;
+export const SCHEDULER_ID_RE = /^[A-Za-z0-9._-]{1,200}$/;
+
+/** The job id a scheduler mints for its run at `ms`: `repeat:<id>:<ms>` (job-scheduler.js, addJobScheduler-11.lua). */
+export const schedulerJobId = (schedulerId, ms) => `${SCHEDULER_JOB_PREFIX}:${schedulerId}:${ms}`;
+
+/** A cron run's job id: the prefix, the trigger's id in the loader's charset, and a slot in decimal with no leading 0. */
+const CRON_RUN_RE = /^(repeat|manual):([A-Za-z0-9._-]+):(0|[1-9][0-9]{0,15})$/;
+
+/**
+ * A cron trigger's run, from its job id: `{ kind, id, ms }`, `kind` `"repeat"` (its scheduler minted it) or `"manual"`
+ * (fired by hand), `id` the trigger's (the scheduler's) id and `ms` the slot. Null for any other id. Strict: both
+ * writers mint the slot from a number, so it is plain digits, and the id is in the loader's charset, which has held
+ * since the first release. `ms` past `Number.MAX_SAFE_INTEGER` (no Date reaches it) is not exact, so a caller that
+ * makes an instant of it checks `Number.isSafeInteger`.
+ *
+ * `idMax` bounds the id for a reader of Valkey or a surface (`SCHEDULER_ID_MAX`); a reader of run records passes none,
+ * since every release before issue #761 took a cron id of any length and its records must still be read.
+ */
+export function cronRunOf(jobId, { idMax = Infinity } = {}) {
+	const m = typeof jobId === "string" ? CRON_RUN_RE.exec(jobId) : null;
+	if (m === null || m[2].length > idMax) return null;
+	return { kind: m[1], id: m[2], ms: Number(m[3]) };
+}
+
+/** The two job id prefixes of a cron trigger's runs, `repeat:<id>:` and `manual:<id>:`, for a scan by prefix. */
+export const cronRunPrefixes = (id) => [`${SCHEDULER_JOB_PREFIX}:${id}:`, `${MANUAL_JOB_PREFIX}:${id}:`];
+
+/**
+ * Whether a job id is in a scheduler's namespace (`repeat:`), whether or not the rest parses: the stall guard counts
+ * such a job and logs one it cannot parse. A hand-fired `manual:` run is not a scheduler's job.
+ */
+export const isSchedulerJobId = (jobId) => typeof jobId === "string" && jobId.startsWith(`${SCHEDULER_JOB_PREFIX}:`);
+
+/**
+ * The scheduled-for instant of a scheduler's run, read from its id (DES-CRON-VIA-BULLMQ-SCHEDULER), or null. A
+ * `manual:` run has no scheduled instant (`manualTriggerJobId` floors its time to the minute), so it reads as null.
+ */
+export function scheduledForMillis(jobId) {
+	const run = cronRunOf(jobId);
+	return run !== null && run.kind === SCHEDULER_JOB_PREFIX ? run.ms : null;
+}
+
+/**
+ * Whether a `repeat:` job is its scheduler's CURRENT next run: BullMQ names that run `repeat:<key>:<next>`, where
+ * `<next>` is the scheduler's score in the queue's repeat zset (isJobSchedulerJob.lua). Only that run is "scheduled,
+ * not in line": a `repeat:` job a gate deferred after the scheduler minted its successor is a real run that waits.
+ */
+export function isSchedulerNext(jobId, rjk, nextRaw) {
+	if (typeof jobId !== "string" || typeof rjk !== "string" || rjk === "") return false;
+	let next;
+	if (typeof nextRaw === "number") {
+		if (!Number.isFinite(nextRaw)) return false;
+		next = String(nextRaw);
+	} else if (typeof nextRaw === "string" && nextRaw !== "") {
+		next = nextRaw;
+	} else {
+		return false;
+	}
+	return jobId === schedulerJobId(rjk, next);
+}
+
+/**
+ * Whether `queue.add` takes `id` as a custom job id (job.js `validateOptions`): not an integer's own spelling ("Custom
+ * Id cannot be integers", `${parseInt(id, 10)}` === id, so `"NaN"` too), and a `:` only when the id splits into exactly
+ * three parts ("Custom Id cannot contain :"), the shape a scheduler's own ids have. An empty id is no custom id at all.
+ */
+export function customJobIdAllowed(id) {
+	if (typeof id !== "string" || id === "") return false;
+	if (`${parseInt(id, 10)}` === id) return false;
+	return !id.includes(":") || id.split(":").length === 3;
+}

package/src/canary-cli.mjs

@@ -0,0 +1,28 @@
+import { parseArgs } from "node:util";
+import { canaryOptions, canaryText, executeCanary } from "./canary.mjs";
+
+export const CANARY_ENV_KEYS = Object.freeze(["PI_WORKER_NAME", "PI_LOGS_DIR", "PI_JOBS_DIR", "PI_SANDBOX_DIR", "PI_SANDBOX_RETENTION_HOURS", "PI_SETTINGS_FILE", "PI_TRIGGERS_FILE", "PI_BACKENDS", "PI_JOB_IMAGE", "PI_MAX_COST_USD", "PI_DAILY_COST_USD", "PI_WEEKLY_COST_USD", "PI_MONTHLY_COST_USD", "PI_DAILY_CAP", "PI_WEEKLY_CAP", "PI_MONTHLY_CAP", "PI_SOFT_HOLD_PCT", "PI_DAILY_TOKEN_CAP", "PI_MAX_TOKENS", "PI_MAX_TURNS", "PI_MODEL", "PI_PROVIDER", "PI_SCOPED_LIMITS_FILE", "PI_PAUSE_WINDOWS_FILE", "PI_ENVELOPE_FILE", "PI_HOST_MEMORY_BUDGET", "PI_HOST_CPU_BUDGET", "PI_HOST_RESERVE_MEMORY", "PI_HOST_RESERVE_CPUS", "PI_MODEL_ENDPOINTS_FILE", "PI_AUTH_FROM_PI", "PI_AUTH_CONNECTIONS_FILE"]);
+const PRIVATE_ENV_KEYS = Object.freeze(["VALKEY_URL", "ANTHROPIC_API_KEY", "ANTHROPIC_OAUTH_TOKEN", "ANTHROPIC_AUTH_TOKEN", "OPENAI_API_KEY"]);
+
+export async function runCanary(args, { env = process.env, write = (s) => process.stdout.write(s), errWrite = (s) => process.stderr.write(s), deploymentEnv = (e) => ({ env: e }), runtimeFactory, now = Date.now } = {}) {
+	let runtime;
+	try {
+		const { values } = parseArgs({ args, allowPositionals: false, options: { role: { type: "string" }, model: { type: "string" }, flow: { type: "string" }, count: { type: "string", default: "3" }, "cap-usd": { type: "string" }, json: { type: "boolean", default: false } } });
+		const options = canaryOptions({ ...values, capUsd: values["cap-usd"] });
+		const deployment = deploymentEnv(env, CANARY_ENV_KEYS);
+		if (deployment.problem) throw new Error(deployment.problem);
+		const credentials = deploymentEnv(deployment.env, PRIVATE_ENV_KEYS);
+		if (credentials.problem) throw new Error("canary provider credentials or Valkey configuration cannot be reconciled between this shell and .env; make those sources agree");
+		const create = runtimeFactory ?? (await import("./canary-runtime.mjs")).createCanaryRuntime;
+		runtime = await create({ env: credentials.env, options, now });
+		const report = await executeCanary(options, { ...runtime, now });
+		write(values.json ? `${JSON.stringify(report)}\n` : canaryText(report));
+		return report.rows.every((r) => r.candidate.outcome === "completed") ? 0 : 2;
+	} catch (error) {
+		// Runtime errors are deliberately fixed messages: git, Docker and provider errors may carry text from a job.
+		errWrite(`error: ${error.message}\n`);
+		return 1;
+	} finally {
+		await runtime?.close?.();
+	}
+}

package/src/canary-lock.mjs

@@ -0,0 +1,41 @@
+import { randomUUID } from "node:crypto";
+import { metaKeyPattern, queueNameOfMetaKey } from "./bullmq-internals.mjs";
+
+export const CANARY_LOCK_KEY = "dispatch:canary:exclusive";
+export const CANARY_LEASE_MS = 60_000;
+const RENEW = "if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('PEXPIRE', KEYS[1], ARGV[2]) else return 0 end";
+const RELEASE = "if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) else return 0 end";
+
+/** Worker pickup asks before any paid work. An unavailable store throws, never admits. */
+export async function readBusy(redis) {
+	return Boolean(await redis.get(CANARY_LOCK_KEY));
+}
+
+/** The lease owner alone may renew or release it; a timed-out owner cannot erase a successor. */
+export async function acquireCanaryLease(redis, { token = randomUUID() } = {}) {
+	if (await redis.set(CANARY_LOCK_KEY, token, "PX", CANARY_LEASE_MS, "NX") !== "OK") return null;
+	return {
+		async renew() { return Number(await redis.eval(RENEW, 1, CANARY_LOCK_KEY, token, CANARY_LEASE_MS)) === 1; },
+		async release() { return Number(await redis.eval(RELEASE, 1, CANARY_LOCK_KEY, token)) === 1; },
+	};
+}
+
+/** Discover host queues as well as the shared queue, then ask BullMQ's public active-count API. */
+export async function canaryQueueNames(redis) {
+	const names = new Set(["pi-jobs"]);
+	let cursor = "0", scanned = 0, pages = 0;
+	do {
+		// MATCH limits returned keys, not the work SCAN does across a shared database.
+		if (++pages > 512) throw new Error("canary queue discovery exceeded its bound");
+		const next = await redis.scan(cursor, "MATCH", metaKeyPattern("pi-jobs"), "COUNT", 100);
+		cursor = next[0];
+		for (const key of next[1]) {
+			if (++scanned > 4096) throw new Error("canary queue discovery exceeded its bound");
+			// The queue's name between BullMQ's `bull:` and `:meta`, or null for a key the pattern's `*` matched that is no
+			// queue's meta key: a job's own hash whose three-part custom id ends in `meta` (`bull:pi-jobs:a:b:meta`).
+			const name = queueNameOfMetaKey(key);
+			if (name !== null && /^pi-jobs(?:@[A-Za-z0-9._-]{1,64})?$/.test(name)) names.add(name);
+		}
+	} while (cursor !== "0");
+	return [...names];
+}

package/src/canary-runtime.mjs

@@ -0,0 +1,237 @@
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import * as fs from "node:fs";
+import { join } from "node:path";
+import { randomUUID } from "node:crypto";
+import { loadConfig, ensureJobsDir } from "./config.mjs";
+import { readOverlay, resolveSettings } from "./runtime-settings.mjs";
+import { readManifest, sandboxExpiry, sandboxManifestNewer } from "./sandbox-store.mjs";
+import { builtinModel, isBuiltinModel } from "./model-catalog.mjs";
+import { resolveProviderCredential } from "./env-allowlist.mjs";
+import { splitModelEntry } from "./model-ref.mjs";
+import { makeImagePreflight } from "./image-preflight.mjs";
+import { makeRunContainer } from "./run-container.mjs";
+import { makeLogSink, makeRecordWriter, buildRecord, recordReadable } from "./run-history.mjs";
+import { materializePiDir } from "./materialize.mjs";
+import { runJob } from "./processor.mjs";
+import { dollarWindowCaps } from "./dollar-budget.mjs";
+import { makeRedisClient, parseConnection } from "./connection.mjs";
+import { makeQueue } from "./queue.mjs";
+import { acquireCanaryLease, canaryQueueNames } from "./canary-lock.mjs";
+import { MODEL_ROLE_RE } from "./model-roles.mjs";
+import { parseTriggersDocument } from "./triggers.mjs";
+import { CONTAINER_HOME } from "./container-spec.mjs";
+import { readAuthPolicy } from "./subscription-config.mjs";
+
+// env-internal PATH: host git executable lookup in a clean subprocess environment.
+const exec = promisify(execFile);
+// The same pinned artifact as deploy/docker-compose.yml. A contract test holds the copy to it.
+export const CANARY_PROXY_IMAGE = "ubuntu/squid@sha256:6a097f68bae708cedbabd6188d68c7e2e7a38cedd05a176e1cc0ba29e3bbe029";
+const PROVIDER_HOSTS = Object.freeze({ anthropic: "api.anthropic.com", openai: "api.openai.com" });
+const SHA = /^(?:[a-f0-9]{40}|[a-f0-9]{64})$/;
+const MAX_SOURCE_BYTES = 256 * 1024 * 1024;
+const fail = (message) => { throw new Error(message); };
+
+export function canaryProxyPolicy(provider) {
+	const host = PROVIDER_HOSTS[provider];
+	if (!host) fail("canary supports first-party anthropic and openai only; gateways and custom endpoints cannot establish a no-forge boundary");
+	return `http_port 3128\naccess_log none\ncache_log /dev/null\npid_filename /tmp/squid.pid\nacl CONNECT method CONNECT\nacl tls port 443\nacl provider dstdomain ${host}\nacl private dst 0.0.0.0/8 10.0.0.0/8 127.0.0.0/8 169.254.0.0/16 172.16.0.0/12 192.168.0.0/16 ::1 ::/128 fc00::/7 fe80::/10\nhttp_access deny provider private\nhttp_access allow CONNECT tls provider\nhttp_access deny all\ncache deny all\n`;
+}
+
+/** Read regular files without following a job-authored link, bounded before allocating its contents. */
+function regular(path, max = 8 * 1024 * 1024) {
+	const fd = fs.openSync(path, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK);
+	try {
+		const st = fs.fstatSync(fd);
+		if (!st.isFile() || st.size > max) fail("canary source contains an unsupported file");
+		return fs.readFileSync(fd);
+	} finally { fs.closeSync(fd); }
+}
+
+/** Only ordinary object files; no alternates, symlinks, hooks or original config enter the new repository. */
+export function copyCanaryObjects(source, destination) {
+	let bytes = 0, files = 0;
+	function copy(from, to, relative = "") {
+		if (!fs.lstatSync(from).isDirectory()) fail("canary source objects must be directories without links");
+		fs.mkdirSync(to, { recursive: true, mode: 0o700 });
+		for (const name of fs.readdirSync(from)) {
+			if (!/^[A-Za-z0-9._-]+$/.test(name) || name === ".." || name === ".") fail("canary source has an invalid object path");
+			const path = relative ? `${relative}/${name}` : name;
+			if (path === "info/alternates" || path === "info/http-alternates") fail("canary source uses alternate object stores; retain a self-contained source");
+			const src = join(from, name), dest = join(to, name), st = fs.lstatSync(src);
+			if (st.isDirectory()) copy(src, dest, path);
+			else {
+				if (!st.isFile() || ++files > 100_000 || (bytes += st.size) > MAX_SOURCE_BYTES) fail("canary source object store is unsupported or too large");
+				fs.writeFileSync(dest, regular(src, MAX_SOURCE_BYTES), { mode: 0o600 });
+			}
+		}
+	}
+	copy(source, destination);
+}
+
+function sourceFor(record, config, now) {
+	const source = readManifest({ sandboxDir: config.sandboxDir, jobId: record.jobId });
+	if (!source || source.jobId !== record.jobId || sandboxExpiry(source, { at: now(), retentionHours: config.sandboxRetentionHours }).expired) fail("a selected source sandbox is absent or expired; enable retention and run the flow again");
+	// Issue #761: a manifest a newer pi-dispatch kept may have reshaped the fields read below (its venue, its base commit,
+	// its workspace), the reason `sandboxSyncRefusal` refuses to reopen one. Before any of them is read.
+	if (sandboxManifestNewer(source)) fail("a selected source sandbox was kept by a newer pi-dispatch; run the canary with that release");
+	if (source.backend !== "local" || !SHA.test(source.baseSha ?? "")) fail("a selected source has no retained base commit on the local venue; run the flow again with this version");
+	if (!source.workspace || !fs.lstatSync(source.workspace).isDirectory() || !fs.lstatSync(join(source.workspace, ".git")).isDirectory()) fail("a selected source has no self-contained retained git repository");
+	// These files were mounted read-only to the original job. Never reconstruct a task from its PII-free record.
+	if (record.bundle || record.secretsProfile || record.session?.resumed === true || record.fork || record.recoveryOf) fail("canary cannot replay a source requiring secrets, a previous session, recovery or a fork");
+	const roles = JSON.parse(regular(join(source.dir, "model-roles.json")));
+	if (roles.version !== 1 || !roles.roles || !Object.hasOwn(roles.roles, record.modelRole)) fail("canary source has no retained role snapshot; run the flow again");
+	return { ...source, roles: roles.roles, prompt: regular(join(source.dir, "prompt.md")), event: regular(join(source.dir, "event.json")) };
+}
+
+export async function createCanaryRuntime({ env, options, now = Date.now, execCommand = exec, redisFactory = makeRedisClient, queueFactory = makeQueue, imagePreflightFactory = makeImagePreflight, containerFactory = makeRunContainer, runJobFn = runJob }) {
+	const config = loadConfig(env);
+	if (config.authConnectionsFile) {
+		let policy;
+		try { policy = readAuthPolicy(config.authConnectionsFile); }
+		catch { fail("canary cannot validate the subscription connection policy; fix the reviewed file before retrying"); }
+		if (policy.bindings[splitModelEntry(options.model)?.provider]) fail("canary does not support a bound subscription connection; it cannot substitute an environment API key");
+	}
+	const settings = resolveSettings(config, readOverlay(config.settingsFile));
+	if (settings.invalid) fail("canary settings overlay is invalid; run doctor");
+	if (config.backends.length !== 1 || config.defaultBackend !== "local") fail("canary currently requires the local Docker venue");
+	if (!process.getuid?.() || !process.getgid?.()) fail("canary requires a non-root local account");
+	if (config.envelopeFile || config.scopedLimitsFile || config.pauseWindowsFile !== null || env.PI_HOST_MEMORY_BUDGET || env.PI_HOST_CPU_BUDGET || env.PI_HOST_RESERVE_MEMORY || env.PI_HOST_RESERVE_CPUS) fail("canary cannot bypass envelope, scoped, pause-window or host-capacity admission; use a deployment without those policies");
+	if (env.PI_MODEL_ENDPOINTS_FILE) fail("canary cannot use declared custom endpoints; use a first-party-only deployment");
+	if (!MODEL_ROLE_RE.test(options.role)) fail("canary role is invalid");
+	let declared;
+	try { declared = parseTriggersDocument(fs.readFileSync(config.triggersFile ?? "triggers.json", "utf8"), config.triggersFile ?? "triggers.json").modelRoles; }
+	catch { fail("canary cannot read the reviewed modelRoles map; check triggers.json"); }
+	if (!Object.values(declared ?? {}).some((map) => Object.hasOwn(map, options.role))) fail("canary role is not declared in modelRoles");
+	const { provider, model } = splitModelEntry(options.model);
+	const policy = canaryProxyPolicy(provider);
+	const builtin = builtinModel(provider, model);
+	if (!isBuiltinModel(provider, model, { chatOnly: true }) || new URL(builtin.baseUrl).hostname !== PROVIDER_HOSTS[provider]) fail("canary candidate must be a pinned first-party chat model");
+	const credential = resolveProviderCredential({ provider, hostEnv: env, authFromPi: config.authFromPi });
+	if (!credential || !Object.values(credential).some(Boolean)) fail("canary provider credential is missing; configure the first-party provider key");
+	const providerEnv = credential;
+	let records;
+	// A record a newer pi-dispatch wrote is left out here (issue #761, `recordReadable`): the role and flow a source is chosen
+	// by are not core fields, so they may mean something else in it. `selectCanarySources` asks again, for any caller.
+	try { records = fs.readdirSync(config.logsDir).filter((n) => /^[A-Za-z0-9._-]+\.json$/.test(n)).flatMap((n) => { try { return [JSON.parse(regular(join(config.logsDir, n), 256 * 1024))]; } catch { return []; } }).filter((record) => recordReadable(record)); }
+	catch { fail("canary run history cannot be read; run this command on the host retaining the originals"); }
+	ensureJobsDir(config.jobsDir, { env });
+	const root = fs.mkdtempSync(join(config.jobsDir, "canary-"));
+	const name = `pi-dispatch-canary-${randomUUID()}`;
+	const proxy = `${name}-proxy`, outNetwork = `${name}-out`;
+	const run = async (args) => {
+		try { return await execCommand("docker", args, { timeout: 30_000, maxBuffer: 1024 * 1024 }); }
+		catch { fail("canary Docker operation failed; check the local daemon and pinned images"); }
+	};
+	let redis, queues = [], proxyMade = false, outMade = false, lease = null, renewTimer = null, leaseLost = false, interrupted = false;
+	const active = new Set();
+	const stops = new Set();
+	const removeContainer = async (container) => {
+		try { await run(["rm", "-f", container]); }
+		catch {
+			// --rm may already have removed it. Only a successful daemon listing proves absence.
+			const listed = await run(["ps", "-aq", "--filter", `name=^/${container}$`]);
+			if (listed.stdout.trim() !== "") fail("canary container remains after cleanup");
+		}
+	};
+	const close = async () => {
+		if (renewTimer) clearInterval(renewTimer);
+		const leftovers = [];
+		for (const container of active) await removeContainer(container).catch(() => leftovers.push(container));
+		if (proxyMade) await removeContainer(proxy).catch(() => leftovers.push(proxy));
+		if (outMade) await run(["network", "rm", outNetwork]).catch(() => leftovers.push(outNetwork));
+		for (const queue of queues) await queue.close().catch(() => {});
+		if (lease && !leftovers.length) await lease.release().catch(() => leftovers.push("canary lease (expires in 60 seconds)"));
+		redis?.disconnect();
+		if (leftovers.length) fail(`canary cleanup failed; remove these Docker resources before another experiment: ${leftovers.join(", ")}`);
+		fs.rmSync(root, { recursive: true, force: true });
+	};
+	try {
+		// Proxy validation and the model/image checks run before the first count or dollar reservation.
+		await run(["image", "inspect", CANARY_PROXY_IMAGE]);
+		const preflight = imagePreflightFactory({ image: config.jobImage });
+		const checked = await preflight({ kind: "local", models: [options.model], maxCostMicros: 1 });
+		if (!checked.ok || !["costCap", "modelPolicy", "modelRoles", "exitAuth", "anyUid"].every((c) => checked.capabilities?.includes(c))) fail("canary job image must declare costCap, modelPolicy, modelRoles, exitAuth and anyUid; rebuild the job image");
+		redis = redisFactory(config.valkeyUrl, { failFast: true, lazyConnect: true });
+		redis.on("error", () => {});
+		if (typeof redis.connect === "function") await redis.connect();
+		await redis.ping();
+		lease = await acquireCanaryLease(redis);
+		if (!lease) fail("another canary owns the deployment admission lease; wait for it to finish");
+		const loseLease = () => { leaseLost = true; for (const stop of stops) stop(); };
+		renewTimer = setInterval(() => { void lease.renew().then((ok) => { if (!ok) loseLease(); }, loseLease); }, 10_000);
+		queues = (await canaryQueueNames(redis)).map((name) => queueFactory(parseConnection(config.valkeyUrl, { failFast: true }), { name }));
+		for (const queue of queues) if ((await queue.getJobCounts("active")).active !== 0) fail("canary refused while jobs are active; wait for those runs to finish");
+		const unpaused = async () => {
+			if (interrupted) fail("canary experiment was stopped; no further replay was started");
+			if (leaseLost || !await lease.renew()) { loseLease(); fail("canary lost its admission lease; the experiment was stopped"); }
+			for (const queue of queues) if (await queue.isPaused()) fail("canary refused while dispatch is paused; resume it explicitly first");
+		};
+		await unpaused();
+		const sink = makeLogSink({ logsDir: config.logsDir, enabled: false });
+		const writer = makeRecordWriter({ logsDir: config.logsDir });
+		const createProxy = async () => {
+			if (proxyMade) return;
+			fs.writeFileSync(join(root, "squid.conf"), policy, { mode: 0o644 });
+			await run(["network", "create", outNetwork]); outMade = true;
+			await run(["run", "--pull=never", "--rm", "-d", "--name", proxy, "--network", outNetwork, "--user", "proxy", "--cap-drop=ALL", "--security-opt=no-new-privileges", "--entrypoint", "/usr/sbin/squid", "--mount", `type=bind,source=${join(root, "squid.conf")},target=/etc/squid/squid.conf,readonly`, CANARY_PROXY_IMAGE, "-f", "/etc/squid/squid.conf", "-NYC"]); proxyMade = true;
+			await run(["exec", proxy, "squid", "-k", "parse"]);
+		};
+		const prepareSource = async (source) => {
+			const jobDir = fs.mkdtempSync(join(root, "job-")), workspace = join(jobDir, "workspace");
+			const gitEnv = { PATH: process.env.PATH, HOME: root, GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null", GIT_TERMINAL_PROMPT: "0" };
+			const git = async (args) => { try { return await execCommand("git", args, { env: gitEnv, timeout: 30_000, maxBuffer: 16 * 1024 * 1024 }); } catch { fail("canary base commit cannot be restored from its retained object store"); } };
+			fs.mkdirSync(workspace);
+			await git(["init", "--template=", workspace]);
+			copyCanaryObjects(join(source.workspace, ".git", "objects"), join(workspace, ".git", "objects"));
+			await git(["-C", workspace, "-c", "core.hooksPath=/dev/null", "checkout", "--detach", source.baseSha]);
+			fs.writeFileSync(join(jobDir, "prompt.md"), source.prompt, { mode: 0o400 });
+			fs.writeFileSync(join(jobDir, "event.json"), source.event, { mode: 0o400 });
+			const materialized = await materializePiDir({ gitDir: workspace, sha: source.baseSha, destDir: jobDir });
+			if (materialized.outcome === "policy") fail("canary base instructions exceed the materializer limits");
+			try { regular(join(jobDir, "pi", "skills", options.flow, "SKILL.md")); }
+			catch { fail("canary flow must be present in the retained base repository; injected and overlay-only flows are unsupported"); }
+			return { ...source, prepared: { jobDir, workspace, sha: source.baseSha } };
+		};
+		return {
+			records, deploymentCapUsd: settings.maxCostUsd, close,
+			inspectSource: async (record) => {
+				const source = sourceFor(record, config, now);
+				for (const entry of Object.values({ ...source.roles, [options.role]: options.model })) {
+					const ref = splitModelEntry(entry);
+					if (ref.provider !== provider || !isBuiltinModel(ref.provider, ref.model, { chatOnly: true }) || new URL(builtinModel(ref.provider, ref.model).baseUrl).hostname !== PROVIDER_HOSTS[provider]) fail("canary source roles need endpoints outside the isolated provider policy; use a single first-party provider");
+				}
+				return prepareSource(source);
+			},
+			execute: async ({ source, original, jobId, maxCostMicros }) => {
+				await unpaused();
+				await createProxy();
+				const { jobDir, workspace } = source.prepared;
+				const modelRoles = { ...source.roles, [options.role]: options.model };
+				const job = { kind: "local", folder: workspace, flow: options.flow, provider, model, modelRole: options.role, modelRoles, models: [...new Set([options.model, ...Object.values(modelRoles)])], maxCostMicros, maxTurns: settings.maxTurns, maxTokens: settings.maxTokens, packages: false, canary: { sourceJobId: original.jobId, role: options.role, candidate: options.model } };
+				const startedAt = new Date(now()).toISOString();
+				let result, error;
+				try {
+					const runContainer = containerFactory({ image: config.jobImage, hostEnv: providerEnv, allowGlobalExtensions: false, packagePaths: [], egress: true, egressProxy: proxy, onOutput: () => {}, openJobLog: sink });
+					result = await runJobFn(job, { redis, caps: { day: settings.dailyCap, week: settings.weeklyCap, month: settings.monthlyCap }, softHoldPct: settings.softHoldPct, tokenCap: settings.dailyTokenCap, dollarCaps: dollarWindowCaps(settings), now: new Date(now()), imagePreflight: async () => checked, checkProviderCredential: () => ({ ok: true }), prepareWorkspace: async () => ({ jobDir, workspace, sha: source.baseSha }), runContainer: async (args) => {
+						const controller = new AbortController();
+						const stop = () => { interrupted = true; if (!controller.signal.aborted) { controller.abort(); void run(["stop", "-t", "5", jobId]).catch(() => {}); } };
+						stops.add(stop);
+						const timer = setTimeout(stop, 30 * 60_000);
+						const pausePoll = setInterval(() => { void unpaused().catch(stop); }, 1000);
+						process.once("SIGINT", stop); process.once("SIGTERM", stop);
+						active.add(jobId);
+						try { return await runContainer({ ...args, name: jobId, signal: controller.signal, exitAuth: true, jobId, user: `${process.getuid()}:${process.getgid()}`, home: CONTAINER_HOME }); }
+						finally { stops.delete(stop); clearTimeout(timer); clearInterval(pausePoll); process.removeListener("SIGINT", stop); process.removeListener("SIGTERM", stop); await removeContainer(jobId).then(() => active.delete(jobId), () => {}); }
+					}, comment: async () => {}, collectChain: async () => ({ enqueued: 0, refused: 0 }), cleanup: async () => {} });
+					if (result) result = { ...result, modelRole: options.role };
+					return result;
+				} catch (e) { error = e; throw new Error("canary replay failed; inspect its run record before retrying"); }
+				finally {
+					await writer(buildRecord({ job: { id: jobId, data: job }, result, error, startedAt, endedAt: new Date(now()).toISOString(), host: config.workerName, defaultBackend: "local" }));
+					if (!active.has(jobId)) fs.rmSync(jobDir, { recursive: true, force: true });
+				}
+			},
+		};
+	} catch (error) { await close(); throw error; }
+}

package/src/canary.mjs

@@ -0,0 +1,53 @@
+import { randomUUID } from "node:crypto";
+import { modelListProblem } from "./model-ref.mjs";
+import { parseUsdMicros, optionalUsdMicros } from "./money.mjs";
+import { MODEL_ROLE_RE } from "./model-roles.mjs";
+import { SKILL_NAME_RE } from "./flow-gate.mjs";
+import { recordReadable } from "./run-history.mjs";
+
+/** A canary pays for fresh isolated runs, never for a modification to the role map. */
+export function canaryOptions({ role, model, flow, count = 3, capUsd }) {
+	if (typeof role !== "string" || !MODEL_ROLE_RE.test(role) || role === "inherit") throw new Error("--role must name a configured model role");
+	if (modelListProblem([model]) !== null) throw new Error("--model must be provider/model");
+	if (typeof flow !== "string" || !SKILL_NAME_RE.test(flow)) throw new Error("--flow must name a flow");
+	if (!Number.isInteger(Number(count)) || Number(count) < 1 || Number(count) > 10) throw new Error("--count must be 1 to 10");
+	const capMicros = parseUsdMicros(capUsd, "--cap-usd");
+	return { role, model, flow, count: Number(count), capMicros };
+}
+
+/**
+ * The completed runs of `options.flow` under `options.role` a canary replays, newest first. A record a newer pi-dispatch
+ * wrote is skipped (issue #761, `recordReadable`): only its core fields keep their meaning in every version, and the
+ * role, the flow and the cost read here are not among them.
+ */
+export function selectCanarySources(records, options, { now, windowMs = 7 * 86400_000 }) {
+	return records.filter((r) => r && recordReadable(r) && !r.canary && r.modelRole === options.role && r.flow === options.flow && r.outcome === "completed" && typeof r.jobId === "string" && /^[A-Za-z0-9._:-]{1,200}$/.test(r.jobId) && Number.isFinite(Date.parse(r.endedAt)) && Date.parse(r.endedAt) <= now && Date.parse(r.endedAt) >= now - windowMs)
+		.sort((a, b) => Date.parse(b.endedAt) - Date.parse(a.endedAt) || a.jobId.localeCompare(b.jobId)).slice(0, options.count);
+}
+
+/**
+ * Allocate the whole experiment before starting. Unreported spend never buys another replay: each
+ * original gets at most its fixed share, even if a prior container died without a trustworthy meter.
+ * Every source is checked first, so an expired last source cannot strand a paid partial experiment.
+ */
+export async function executeCanary(options, { records, now = Date.now, inspectSource, execute, deploymentCapUsd = null, id = () => `canary-${randomUUID()}` }) {
+	const originals = selectCanarySources(records, options, { now: now() });
+	if (originals.length === 0) throw new Error("no completed jobs of this flow in the last seven days; run the flow with sandbox retention enabled first");
+	const sources = [];
+	for (const original of originals) sources.push(await inspectSource(original));
+	const share = Math.floor(options.capMicros / sources.length);
+	const perJob = Math.min(share, optionalUsdMicros(deploymentCapUsd, "maxCostUsd") ?? share);
+	if (perJob < 1) throw new Error("the canary cap is too small to allocate one micro-dollar per job");
+	const rows = [];
+	for (let i = 0; i < sources.length; i++) {
+		const jobId = id();
+		const result = await execute({ source: sources[i], original: originals[i], jobId, options, maxCostMicros: perJob });
+		rows.push({ sourceJobId: originals[i].jobId, jobId, original: { model: `${originals[i].provider}/${originals[i].model}`, outcome: originals[i].outcome, cost: originals[i].tokens?.cost ?? null }, candidate: { model: options.model, outcome: result.outcome, reason: result.reason ?? null, cost: result.tokens?.cost ?? null }, capMicros: perJob });
+	}
+	return { role: options.role, flow: options.flow, model: options.model, capMicros: options.capMicros, allocatedMicros: perJob * rows.length, whatIf: `/insights?whatif=${encodeURIComponent(options.model)}&flow=${encodeURIComponent(options.flow)}`, rows };
+}
+
+export function canaryText(report) {
+	const money = (value) => typeof value === "number" && Number.isFinite(value) ? `$${value.toFixed(6)}` : "unknown";
+	return [`Canary ${report.role}: ${report.model}, cap $${(report.capMicros / 1e6).toFixed(6)}`, "Original run | original cost / outcome | candidate run | candidate cost / outcome", ...report.rows.map((r) => `${r.sourceJobId} | ${money(r.original.cost)} / ${r.original.outcome} | ${r.jobId} | ${money(r.candidate.cost)} / ${r.candidate.outcome}${r.candidate.reason ? ` (${r.candidate.reason})` : ""}`), `What-if: ${report.whatIf}`, "The role map is unchanged. Workspaces are disposable; forge access is blocked.", ""].join("\n");
+}