@wjarka/cezarion 0.14.9-dev.933 → 0.14.9-dev.936

package/dist/ci-wait/controller.d.ts

@@ -41,17 +41,11 @@ export declare function ciToolRoutes(capabilities: Map<string, Capability>): imp
     "/api/v1/tools/ci-wait": {
         $post: {
             output: {
-                prUrl: string;
-                repository: string;
-                prNumber: number;
-                headSha: string;
-                waitId: string;
-                registeredAt: string;
-                deadline: string;
-                phase: "delivered" | "parked" | "registered" | "wake-pending" | "withdrawn";
+                code: "authentication" | "capacity" | "command_failed" | "gh_missing" | "inaccessible_pr" | "invalid_request" | "malformed_data" | "output_limit" | "persistence" | "query_timeout" | "unauthorized" | "unavailable" | "unsupported_host" | "wait_conflict";
+                message: string;
             };
             outputFormat: "json";
-            status: import("hono/utils/http-status").ContentfulStatusCode;
+            status: 503;
             input: {
                 json: {
                     pr: string;
@@ -60,11 +54,17 @@ export declare function ciToolRoutes(capabilities: Map<string, Capability>): imp
             };
         } | {
             output: {
-                code: "authentication" | "capacity" | "command_failed" | "gh_missing" | "inaccessible_pr" | "invalid_request" | "malformed_data" | "output_limit" | "persistence" | "query_timeout" | "unauthorized" | "unavailable" | "unsupported_host" | "wait_conflict";
-                message: string;
+                prUrl: string;
+                repository: string;
+                prNumber: number;
+                headSha: string;
+                waitId: string;
+                registeredAt: string;
+                deadline: string;
+                phase: "delivered" | "parked" | "registered" | "wake-pending" | "withdrawn";
             };
             outputFormat: "json";
-            status: 503;
+            status: import("hono/utils/http-status").ContentfulStatusCode;
             input: {
                 json: {
                     pr: string;

package/dist/delegation/process-liveness.d.ts

@@ -0,0 +1,66 @@
+/**
+ * Synchronous, dependency-free proof that a crashed worker generation's processes are gone
+ * (#469, spec 2026-09-26-worker-orphan-finalization). Sync so `continueRun` stays sync.
+ * Every uncertainty answers "alive" or "unknown", never "gone".
+ */
+export type RecordedProcess = {
+    pid: number;
+    startToken?: string;
+};
+export type WorkerProcessRecord = {
+    generation: string;
+    controller: RecordedProcess;
+    processes: RecordedProcess[];
+};
+export type GenerationLiveness = 'gone' | 'alive' | 'unknown';
+/** `/proc/<pid>/stat` field 22 (`starttime`); `comm` may hold spaces and `)`, so parse after the LAST `)`. */
+export declare function parseProcStat(stat: string): {
+    state: string;
+    startToken: string;
+} | undefined;
+/** One process incarnation, so a reused PID never matches. Absent when the platform cannot say. */
+export declare function processStartToken(pid: number, platform?: NodeJS.Platform): string | undefined;
+/** Live and the same incarnation. A token-less entry (or an unreadable current token) counts while the PID exists. */
+export declare function recordedProcessLive(entry: RecordedProcess): boolean;
+export declare function isCurrentProcess(entry: RecordedProcess): boolean;
+/** The `/proc` reads the Linux scan makes; injectable because EACCES cannot be staged for real. */
+export type ProcReader = {
+    readdir: () => string[];
+    readlink: (pid: string) => string;
+    ownerUid: (pid: string) => number | undefined;
+    startedAtMs: (pid: string) => number | undefined;
+};
+/** The darwin scan: `lsof` for working directories, `ps` for our own user's processes. */
+export type DarwinReader = {
+    lsof: () => {
+        ok: boolean;
+        stdout: string;
+    };
+    ownProcesses: () => Array<{
+        pid: number;
+        startedAtMs?: number;
+    }> | undefined;
+};
+/** PIDs (never this process) whose working directory is one of `dirs` or beneath it, in one scan
+ * pass; `unknown` when no scan can run. cezar's own short-lived git children in a worktree make
+ * this read "alive" briefly: conservative, and a retry self-heals. `since` (epoch ms) is the
+ * earliest moment the generation's processes can have started; see the EACCES rule below. */
+export declare function processesWithCwdUnder(dirs: string | readonly string[], platform?: NodeJS.Platform, proc?: ProcReader, since?: number, darwin?: DarwinReader): number[] | 'unknown';
+export type GenerationProbe = {
+    liveness: GenerationLiveness;
+    controller?: number;
+    pids: number[];
+};
+/** A missing record (legacy) relies on the working-directory scan alone. `paths` are the
+ * worktree and every scratch location, which finalization deletes. A live foreign controller
+ * short-circuits the scan; otherwise `pids` names every live recorded or scanned process. */
+export declare function inspectGeneration({ record, paths, since }: {
+    record?: WorkerProcessRecord;
+    paths: readonly string[];
+    since?: number;
+}): GenerationProbe;
+export declare function probeGeneration(input: {
+    record?: WorkerProcessRecord;
+    paths: readonly string[];
+    since?: number;
+}): GenerationLiveness;

package/dist/delegation/process-liveness.js

@@ -0,0 +1,215 @@
+import { spawnSync } from 'node:child_process';
+import { readdirSync, readFileSync, readlinkSync, realpathSync, statSync } from 'node:fs';
+import { resolve, sep } from 'node:path';
+/** `/proc/<pid>/stat` field 22 (`starttime`); `comm` may hold spaces and `)`, so parse after the LAST `)`. */
+export function parseProcStat(stat) {
+    const fields = stat.slice(stat.lastIndexOf(')') + 1).trim().split(/\s+/);
+    return stat.includes(')') && fields.length >= 20 && /^\d+$/.test(fields[19]) ? { state: fields[0], startToken: fields[19] } : undefined;
+}
+function procStat(pid) {
+    try {
+        return parseProcStat(readFileSync(`/proc/${pid}/stat`, 'utf8'));
+    }
+    catch {
+        return undefined;
+    }
+}
+let bootId;
+/** `starttime` counts ticks since boot, so it repeats across reboots; the boot id scopes it. */
+function linuxBootId() {
+    if (bootId === undefined) {
+        try {
+            bootId = readFileSync('/proc/sys/kernel/random/boot_id', 'utf8').trim() || null;
+        }
+        catch {
+            bootId = null;
+        }
+    }
+    return bootId ?? undefined;
+}
+/** One process incarnation, so a reused PID never matches. Absent when the platform cannot say. */
+export function processStartToken(pid, platform = process.platform) {
+    if (!Number.isSafeInteger(pid) || pid <= 0)
+        return undefined;
+    if (platform === 'linux') {
+        const start = procStat(pid)?.startToken;
+        const boot = linuxBootId();
+        return start && boot ? `${boot}:${start}` : start;
+    }
+    if (platform !== 'darwin')
+        return undefined;
+    // `lstart` is locale- and zone-formatted; pin both so the token is stable across cezar processes.
+    const ps = spawnSync('ps', ['-o', 'lstart=', '-p', String(pid)], { encoding: 'utf8', timeout: 2_000, env: { ...process.env, LC_ALL: 'C', TZ: 'UTC' } });
+    const token = ps.status === 0 ? ps.stdout.trim() : '';
+    return token || undefined;
+}
+function pidExists(pid) {
+    if (!Number.isSafeInteger(pid) || pid <= 0)
+        return false;
+    try {
+        process.kill(pid, 0);
+    }
+    catch (error) {
+        return error.code === 'EPERM';
+    }
+    // A zombie has exited; only its parent's wait remains.
+    return process.platform !== 'linux' || procStat(pid)?.state !== 'Z';
+}
+const LINUX_TOKEN = /^(?:[0-9a-f-]{36}:)?(\d+)$/;
+/** Liveness-only comparison: when exactly one Linux token lacks the boot id (it was unreadable on one
+ * side), the `starttime` suffix decides. Reaping still requires an exact match. */
+function sameIncarnation(recorded, current) {
+    if (recorded === current)
+        return true;
+    const a = LINUX_TOKEN.exec(recorded), b = LINUX_TOKEN.exec(current);
+    return !!a && !!b && recorded.includes(':') !== current.includes(':') && a[1] === b[1];
+}
+/** Live and the same incarnation. A token-less entry (or an unreadable current token) counts while the PID exists. */
+export function recordedProcessLive(entry) {
+    if (!pidExists(entry.pid))
+        return false;
+    if (entry.startToken === undefined)
+        return true;
+    const current = processStartToken(entry.pid);
+    return current === undefined || sameIncarnation(entry.startToken, current);
+}
+export function isCurrentProcess(entry) {
+    if (entry.pid !== process.pid)
+        return false;
+    const own = processStartToken(process.pid);
+    return entry.startToken === undefined || own === undefined || sameIncarnation(entry.startToken, own);
+}
+let clockTicks;
+let bootTimeMs;
+/** Wall-clock start of a process: boot time (`/proc/stat` btime) plus its starttime ticks. */
+function procStartedAtMs(pid) {
+    try {
+        bootTimeMs ??= Number(/^btime (\d+)$/m.exec(readFileSync('/proc/stat', 'utf8'))?.[1]) * 1000;
+        clockTicks ??= Number(spawnSync('getconf', ['CLK_TCK'], { encoding: 'utf8', timeout: 2_000 }).stdout.trim()) || 100;
+        const start = procStat(Number(pid))?.startToken;
+        return start === undefined || !Number.isFinite(bootTimeMs) ? undefined : bootTimeMs + Number(start) / clockTicks * 1000;
+    }
+    catch {
+        return undefined;
+    }
+}
+const realDarwin = {
+    lsof: () => {
+        const lsof = spawnSync('lsof', ['-a', '-d', 'cwd', '-Fpn'], { encoding: 'utf8', timeout: 5_000, maxBuffer: 16 * 1024 * 1024 });
+        // Status 1 is ordinary (some process unreadable); completeness is judged against `ps` instead.
+        return { ok: !lsof.error && (lsof.status === 0 || lsof.status === 1) && !!lsof.stdout, stdout: lsof.stdout ?? '' };
+    },
+    ownProcesses: () => {
+        const uid = process.getuid?.();
+        if (uid === undefined)
+            return undefined;
+        const ps = spawnSync('ps', ['-U', String(uid), '-o', 'pid=,lstart='], { encoding: 'utf8', timeout: 2_000, env: { ...process.env, LC_ALL: 'C', TZ: 'UTC' } });
+        if (ps.error || ps.status !== 0 || !ps.stdout)
+            return undefined;
+        return ps.stdout.split('\n').flatMap(line => {
+            const match = /^\s*(\d+)\s+(.+?)\s*$/.exec(line);
+            // `ps` lists itself, and lsof (which ran first) never saw it.
+            if (!match || Number(match[1]) === ps.pid)
+                return [];
+            const startedAtMs = Date.parse(`${match[2]} UTC`);
+            return [{ pid: Number(match[1]), ...(Number.isFinite(startedAtMs) ? { startedAtMs } : {}) }];
+        });
+    },
+};
+const realProc = {
+    readdir: () => readdirSync('/proc'),
+    readlink: pid => readlinkSync(`/proc/${pid}/cwd`),
+    ownerUid: pid => { try {
+        return statSync(`/proc/${pid}`).uid;
+    }
+    catch {
+        return undefined;
+    } },
+    startedAtMs: procStartedAtMs,
+};
+/** PIDs (never this process) whose working directory is one of `dirs` or beneath it, in one scan
+ * pass; `unknown` when no scan can run. cezar's own short-lived git children in a worktree make
+ * this read "alive" briefly: conservative, and a retry self-heals. `since` (epoch ms) is the
+ * earliest moment the generation's processes can have started; see the EACCES rule below. */
+export function processesWithCwdUnder(dirs, platform = process.platform, proc = realProc, since, darwin = realDarwin) {
+    const targets = (typeof dirs === 'string' ? [dirs] : dirs).map(dir => { try {
+        return realpathSync(dir);
+    }
+    catch {
+        return resolve(dir);
+    } });
+    const under = (cwd) => { const path = cwd.replace(/ \(deleted\)$/, ''); return targets.some(target => path === target || path.startsWith(target + sep)); };
+    const found = [];
+    if (platform === 'linux') {
+        let entries;
+        try {
+            entries = proc.readdir();
+        }
+        catch {
+            return 'unknown';
+        }
+        const uid = process.getuid?.();
+        for (const entry of entries) {
+            if (!/^\d+$/.test(entry) || Number(entry) === process.pid)
+                continue;
+            try {
+                if (under(proc.readlink(entry)))
+                    found.push(Number(entry));
+            }
+            catch (error) {
+                // A vanished process is gone; another user's is unreadable by design and skipped. Our own
+                // user's non-dumpable processes are unreadable too (systemd --user, sshd, gpg-agent, which
+                // every host has), so they cannot all block the proof: one that started before the worker
+                // existed cannot be its descendant. A later one is a possible holder, never signalled.
+                if (error.code === 'ENOENT' || uid === undefined || proc.ownerUid(entry) !== uid)
+                    continue;
+                const started = since === undefined ? undefined : proc.startedAtMs(entry);
+                if (started === undefined || started >= since)
+                    found.push(Number(entry));
+            }
+        }
+        return found;
+    }
+    if (platform !== 'darwin')
+        return 'unknown';
+    const lsof = darwin.lsof();
+    if (!lsof.ok)
+        return 'unknown';
+    const seen = new Set();
+    let pid;
+    for (const line of lsof.stdout.split('\n')) {
+        if (line.startsWith('p')) {
+            pid = Number(line.slice(1));
+            seen.add(pid);
+        }
+        else if (line.startsWith('n') && pid !== undefined && pid !== process.pid && under(line.slice(1)))
+            found.push(pid);
+    }
+    // lsof silently omits what it cannot read. An own-user process it omitted is judged by the
+    // Linux EACCES rule: a possible holder unless it predates the worker. Other users' are skipped.
+    const own = darwin.ownProcesses();
+    if (!own)
+        return 'unknown';
+    for (const entry of own) {
+        if (seen.has(entry.pid) || entry.pid === process.pid)
+            continue;
+        if (since === undefined || entry.startedAtMs === undefined || entry.startedAtMs >= since)
+            found.push(entry.pid);
+    }
+    return [...new Set(found)];
+}
+/** A missing record (legacy) relies on the working-directory scan alone. `paths` are the
+ * worktree and every scratch location, which finalization deletes. A live foreign controller
+ * short-circuits the scan; otherwise `pids` names every live recorded or scanned process. */
+export function inspectGeneration({ record, paths, since }) {
+    if (record && !isCurrentProcess(record.controller) && recordedProcessLive(record.controller))
+        return { liveness: 'alive', controller: record.controller.pid, pids: [] };
+    const recorded = record?.processes.filter(recordedProcessLive).map(entry => entry.pid) ?? [];
+    const scan = processesWithCwdUnder(paths, process.platform, realProc, since);
+    const pids = [...new Set([...recorded, ...(scan === 'unknown' ? [] : scan)])];
+    return { liveness: pids.length ? 'alive' : scan === 'unknown' ? 'unknown' : 'gone', pids };
+}
+export function probeGeneration(input) {
+    return inspectGeneration(input).liveness;
+}
+//# sourceMappingURL=process-liveness.js.map

package/dist/delegation/process-liveness.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"process-liveness.js","sourceRoot":"","sources":["../../src/delegation/process-liveness.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC1F,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAWzC,8GAA8G;AAC9G,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACzE,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,IAAI,EAAE,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAE,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAE,EAAE,UAAU,EAAE,MAAM,CAAC,EAAE,CAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;AAC7I,CAAC;AAED,SAAS,QAAQ,CAAC,GAAW;IAC3B,IAAI,CAAC;QAAC,OAAO,aAAa,CAAC,YAAY,CAAC,SAAS,GAAG,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;IAAC,CAAC;IAAC,MAAM,CAAC;QAAC,OAAO,SAAS,CAAC;IAAC,CAAC;AACtG,CAAC;AAED,IAAI,MAAiC,CAAC;AACtC,gGAAgG;AAChG,SAAS,WAAW;IAClB,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QAAC,IAAI,CAAC;YAAC,MAAM,GAAG,YAAY,CAAC,iCAAiC,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,MAAM,GAAG,IAAI,CAAC;QAAC,CAAC;IAAC,CAAC;IAC/I,OAAO,MAAM,IAAI,SAAS,CAAC;AAC7B,CAAC;AAED,mGAAmG;AACnG,MAAM,UAAU,iBAAiB,CAAC,GAAW,EAAE,QAAQ,GAAoB,OAAO,CAAC,QAAQ;IACzF,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IAC7D,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,EAAE,UAAU,CAAC;QAAC,MAAM,IAAI,GAAG,WAAW,EAAE,CAAC;QACpE,OAAO,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;IACpD,CAAC;IACD,IAAI,QAAQ,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAC5C,kGAAkG;IAClG,MAAM,EAAE,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;IACxJ,MAAM,KAAK,GAAG,EAAE,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACtD,OAAO,KAAK,IAAI,SAAS,CAAC;AAC5B,CAAC;AAED,SAAS,SAAS,CAAC,GAAW;IAC5B,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACzD,IAAI,CAAC;QAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;IAAC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAAC,OAAQ,KAA+B,CAAC,IAAI,KAAK,OAAO,CAAC;IAAC,CAAC;IACzG,uDAAuD;IACvD,OAAO,OAAO,CAAC,QAAQ,KAAK,OAAO,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE,KAAK,KAAK,GAAG,CAAC;AACtE,CAAC;AAED,MAAM,WAAW,GAAG,4BAA4B,CAAC;AACjD;mFACmF;AACnF,SAAS,eAAe,CAAC,QAAgB,EAAE,OAAe;IACxD,IAAI,QAAQ,KAAK,OAAO;QAAE,OAAO,IAAI,CAAC;IACtC,MAAM,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACzF,CAAC;AAED,sHAAsH;AACtH,MAAM,UAAU,mBAAmB,CAAC,KAAsB;IACxD,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACxC,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IAChD,MAAM,OAAO,GAAG,iBAAiB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC7C,OAAO,OAAO,KAAK,SAAS,IAAI,eAAe,CAAC,KAAK,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;AAC7E,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAsB;IACrD,IAAI,KAAK,CAAC,GAAG,KAAK,OAAO,CAAC,GAAG;QAAE,OAAO,KAAK,CAAC;IAC5C,MAAM,GAAG,GAAG,iBAAiB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC3C,OAAO,KAAK,CAAC,UAAU,KAAK,SAAS,IAAI,GAAG,KAAK,SAAS,IAAI,eAAe,CAAC,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;AACvG,CAAC;AAOD,IAAI,UAA8B,CAAC;AACnC,IAAI,UAA8B,CAAC;AACnC,8FAA8F;AAC9F,SAAS,eAAe,CAAC,GAAW;IAClC,IAAI,CAAC;QACH,UAAU,KAAK,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;QAC7F,UAAU,KAAK,MAAM,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC,SAAS,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,IAAI,GAAG,CAAC;QACpH,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC;QAChD,OAAO,KAAK,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,UAAU,GAAG,IAAI,CAAC;IAC1H,CAAC;IAAC,MAAM,CAAC;QAAC,OAAO,SAAS,CAAC;IAAC,CAAC;AAC/B,CAAC;AAGD,MAAM,UAAU,GAAiB;IAC/B,IAAI,EAAE,GAAG,EAAE;QACT,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC,CAAC;QAC/H,+FAA+F;QAC/F,OAAO,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;IACrH,CAAC;IACD,YAAY,EAAE,GAAG,EAAE;QACjB,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QAC/B,IAAI,GAAG,KAAK,SAAS;YAAE,OAAO,SAAS,CAAC;QACxC,MAAM,EAAE,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,cAAc,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;QAC7J,IAAI,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM;YAAE,OAAO,SAAS,CAAC;QAChE,OAAO,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;YAC1C,MAAM,KAAK,GAAG,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjD,8DAA8D;YAC9D,IAAI,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,GAAG;gBAAE,OAAO,EAAE,CAAC;YACrD,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAClD,OAAO,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QAC/F,CAAC,CAAC,CAAC;IACL,CAAC;CACF,CAAC;AACF,MAAM,QAAQ,GAAe;IAC3B,OAAO,EAAE,GAAG,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC;IACnC,QAAQ,EAAE,GAAG,CAAC,EAAE,CAAC,YAAY,CAAC,SAAS,GAAG,MAAM,CAAC;IACjD,QAAQ,EAAE,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC;QAAC,OAAO,QAAQ,CAAC,SAAS,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC;IAAC,CAAC;IAAC,MAAM,CAAC;QAAC,OAAO,SAAS,CAAC;IAAC,CAAC,CAAC,CAAC;IAC7F,WAAW,EAAE,eAAe;CAC7B,CAAC;AAEF;;;6FAG6F;AAC7F,MAAM,UAAU,qBAAqB,CAAC,IAAgC,EAAE,QAAQ,GAAoB,OAAO,CAAC,QAAQ,EAAE,IAAI,GAAe,QAAQ,EAAE,KAAc,EAAE,MAAM,GAAiB,UAAU;IAClM,MAAM,OAAO,GAAG,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC;QAAC,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC;IAAC,CAAC;IAAC,MAAM,CAAC;QAAC,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC;IAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5I,MAAM,KAAK,GAAG,CAAC,GAAW,EAAE,EAAE,GAAG,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,KAAK,MAAM,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACnK,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;QACzB,IAAI,OAAiB,CAAC;QACtB,IAAI,CAAC;YAAC,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,OAAO,SAAS,CAAC;QAAC,CAAC;QAC7D,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QAC/B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,OAAO,CAAC,GAAG;gBAAE,SAAS;YACpE,IAAI,CAAC;gBAAC,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAAC,CAAC;YACnE,OAAO,KAAK,EAAE,CAAC;gBACb,0FAA0F;gBAC1F,2FAA2F;gBAC3F,0FAA0F;gBAC1F,uFAAuF;gBACvF,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,IAAI,GAAG,KAAK,SAAS,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,GAAG;oBAAE,SAAS;gBACtH,MAAM,OAAO,GAAG,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;gBAC1E,IAAI,OAAO,KAAK,SAAS,IAAI,OAAO,IAAI,KAAM;oBAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAC5E,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,QAAQ,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAC5C,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;IAC3B,IAAI,CAAC,IAAI,CAAC,EAAE;QAAE,OAAO,SAAS,CAAC;IAC/B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,IAAI,GAAuB,CAAC;IAC5B,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3C,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAAC,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;aACpE,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,OAAO,CAAC,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACrH,CAAC;IACD,2FAA2F;IAC3F,gGAAgG;IAChG,MAAM,GAAG,GAAG,MAAM,CAAC,YAAY,EAAE,CAAC;IAClC,IAAI,CAAC,GAAG;QAAE,OAAO,SAAS,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,GAAG,EAAE,CAAC;QACxB,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,GAAG,KAAK,OAAO,CAAC,GAAG;YAAE,SAAS;QAC/D,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,WAAW,KAAK,SAAS,IAAI,KAAK,CAAC,WAAW,IAAI,KAAK;YAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAClH,CAAC;IACD,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;AAC7B,CAAC;AAID;;6FAE6F;AAC7F,MAAM,UAAU,iBAAiB,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAA8E;IACpI,IAAI,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,mBAAmB,CAAC,MAAM,CAAC,UAAU,CAAC;QAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;IACxK,MAAM,QAAQ,GAAG,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;IAC7F,MAAM,IAAI,GAAG,qBAAqB,CAAC,KAAK,EAAE,OAAO,CAAC,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IAC7E,MAAM,IAAI,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,EAAE,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9E,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC;AAC7F,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,KAAiF;IAC/G,OAAO,iBAAiB,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC;AAC3C,CAAC"}

package/dist/delegation/routes.d.ts

@@ -82,13 +82,13 @@ export declare function createDelegationRoutes(service: DelegationService, crede
             status: 200;
             input: {
                 json: {
-                    requestIds: string[];
-                    mode?: "all" | "any" | "one" | undefined;
-                    timeoutSeconds?: number | undefined;
-                } | {
                     workerIds: string[];
                     timeoutSeconds?: number | undefined;
                     mode?: "all" | "any" | "one" | undefined;
+                } | {
+                    requestIds: string[];
+                    mode?: "all" | "any" | "one" | undefined;
+                    timeoutSeconds?: number | undefined;
                 };
             };
         };
@@ -439,18 +439,6 @@ export declare function createDelegationRoutes(service: DelegationService, crede
 } & {
     "/:workerId": {
         $get: {
-            output: {
-                error: string;
-                code?: "invalid_input" | undefined;
-            };
-            outputFormat: "json";
-            status: 400;
-            input: {
-                param: {
-                    workerId: string;
-                };
-            };
-        } | {
             output: {
                 workerId: string;
                 parentRunId: string;
@@ -527,11 +515,7 @@ export declare function createDelegationRoutes(service: DelegationService, crede
                     workerId: string;
                 };
             };
-        };
-    };
-} & {
-    "/:workerId/collect": {
-        $post: {
+        } | {
             output: {
                 error: string;
                 code?: "invalid_input" | undefined;
@@ -542,10 +526,12 @@ export declare function createDelegationRoutes(service: DelegationService, crede
                 param: {
                     workerId: string;
                 };
-            } & {
-                json: Record<string, never>;
             };
-        } | {
+        };
+    };
+} & {
+    "/:workerId/collect": {
+        $post: {
             output: {
                 workerId: string;
                 parentRunId: string;
@@ -647,11 +633,7 @@ export declare function createDelegationRoutes(service: DelegationService, crede
             } & {
                 json: Record<string, never>;
             };
-        };
-    };
-} & {
-    "/:workerId/steer": {
-        $post: {
+        } | {
             output: {
                 error: string;
                 code?: "invalid_input" | undefined;
@@ -663,11 +645,13 @@ export declare function createDelegationRoutes(service: DelegationService, crede
                     workerId: string;
                 };
             } & {
-                json: {
-                    text: string;
-                };
+                json: Record<string, never>;
             };
-        } | {
+        };
+    };
+} & {
+    "/:workerId/steer": {
+        $post: {
             output: {
                 workerId: string;
                 state: "delivered" | "queued";
@@ -683,11 +667,7 @@ export declare function createDelegationRoutes(service: DelegationService, crede
                     text: string;
                 };
             };
-        };
-    };
-} & {
-    "/:workerId/stop": {
-        $post: {
+        } | {
             output: {
                 error: string;
                 code?: "invalid_input" | undefined;
@@ -699,9 +679,15 @@ export declare function createDelegationRoutes(service: DelegationService, crede
                     workerId: string;
                 };
             } & {
-                json: Record<string, never>;
+                json: {
+                    text: string;
+                };
             };
-        } | {
+        };
+    };
+} & {
+    "/:workerId/stop": {
+        $post: {
             output: {
                 workerId: string;
                 state: "stopping" | "terminated";
@@ -715,11 +701,7 @@ export declare function createDelegationRoutes(service: DelegationService, crede
             } & {
                 json: Record<string, never>;
             };
-        };
-    };
-} & {
-    "/:workerId/destroy": {
-        $post: {
+        } | {
             output: {
                 error: string;
                 code?: "invalid_input" | undefined;
@@ -733,7 +715,11 @@ export declare function createDelegationRoutes(service: DelegationService, crede
             } & {
                 json: Record<string, never>;
             };
-        } | {
+        };
+    };
+} & {
+    "/:workerId/destroy": {
+        $post: {
             output: {
                 workerId: string;
                 state: "complete" | "incomplete";
@@ -756,6 +742,20 @@ export declare function createDelegationRoutes(service: DelegationService, crede
             } & {
                 json: Record<string, never>;
             };
+        } | {
+            output: {
+                error: string;
+                code?: "invalid_input" | undefined;
+            };
+            outputFormat: "json";
+            status: 400;
+            input: {
+                param: {
+                    workerId: string;
+                };
+            } & {
+                json: Record<string, never>;
+            };
         } | {
             output: {
                 workerId: string;
@@ -785,11 +785,13 @@ export declare function createDelegationRoutes(service: DelegationService, crede
     "/:workerId/diff": {
         $get: {
             output: {
-                error: string;
-                code?: "invalid_input" | undefined;
+                workerId: string;
+                baselineSha: string;
+                diff: string;
+                truncated: boolean;
             };
             outputFormat: "json";
-            status: 400;
+            status: 200;
             input: {
                 param: {
                     workerId: string;
@@ -797,13 +799,11 @@ export declare function createDelegationRoutes(service: DelegationService, crede
             };
         } | {
             output: {
-                workerId: string;
-                baselineSha: string;
-                diff: string;
-                truncated: boolean;
+                error: string;
+                code?: "invalid_input" | undefined;
             };
             outputFormat: "json";
-            status: 200;
+            status: 400;
             input: {
                 param: {
                     workerId: string;

package/dist/delegation/service.d.ts

@@ -13,6 +13,8 @@ export declare const delegationEnabled: () => boolean;
 export declare class DelegationService {
     private projects;
     private serial;
+    /** How long destroy waits for proven termination; private and overridable so tests need not wait it out. */
+    private terminationTimeoutMs;
     registerProject(project: DelegationProject): () => void;
     private context;
     private serialized;

package/dist/delegation/service.js

@@ -80,6 +80,8 @@ export const delegationEnabled = () => process.env.CEZ_DELEGATION === '1';
 export class DelegationService {
     projects = new Map();
     serial = new Map();
+    /** How long destroy waits for proven termination; private and overridable so tests need not wait it out. */
+    terminationTimeoutMs = 30_000;
     registerProject(project) {
         const existing = this.projects.get(project.id);
         if (existing?.store === project.store && existing.manager === project.manager)
@@ -452,9 +454,10 @@ export class DelegationService {
         });
     }
     async inspect(caller, params) {
-        const { worker, parent } = this.target(caller, params, 'inspect');
+        const { project, worker, parent } = this.target(caller, params, 'inspect');
         if (worker.delegation?.role !== 'worker')
             throw new DelegationPolicyError('denied_scope', 'Worker scope denied');
+        project.manager.settleOrphanedWorkerExecution(worker.id); // #469: an orphan that died after recovery settles
         const outcome = workerOutcome(worker, new Date().toISOString());
         const wait = parent.delegation?.role === 'root' ? parent.delegation.wait : undefined;
         return { workerId: worker.id, parentRunId: parent.id, status: worker.status, workspace: worker.delegation.workspace,
@@ -477,6 +480,7 @@ export class DelegationService {
             return project.store.commitWorkerResult(caller.runId, evidence, project.store.readWorkerResultDiff(caller.runId, workerId));
         }
         authorizeWorker(caller, worker, 'inspect', parent, project.id);
+        project.manager.settleOrphanedWorkerExecution(workerId); // #469: before `settled` is computed
         if (parent?.delegation?.role === 'root' && parent.delegation.receipts.some(receipt => receipt.workerId === workerId && receipt.deletion?.phase === 'pending')) {
             const retained = project.store.readWorkerResult(parent.id, workerId);
             if (!retained || !project.store.canDeleteRun(workerId))
@@ -576,8 +580,16 @@ export class DelegationService {
             persist('terminating', ['process', ...resources]);
             project.manager.requestWorkerStop(workerId);
             let result;
-            if (!await project.manager.awaitRunTermination(workerId, 30_000)) {
-                result = { workerId, state: 'incomplete', remaining: ['process', ...resources], error: 'Worker termination is not proven; retry cleanup later' };
+            if (!await project.manager.awaitRunTermination(workerId, this.terminationTimeoutMs, { reapOrphans: true })) {
+                // #469: name what blocks a crashed generation; other causes keep the generic message.
+                const taken = project.manager.takeWorkerTerminationBlocker(workerId);
+                const reason = taken && (taken.blocker.kind === 'unreadable' ? 'worker process record is unreadable; termination cannot be proven'
+                    : taken.blocker.kind === 'controller' ? `the worker is still controlled by a live cezar (pid ${taken.blocker.pid})`
+                        : `${taken.blocker.pids.length === 1 ? 'process' : 'processes'} ${taken.blocker.pids.join(', ')} still ${taken.blocker.pids.length === 1 ? 'holds' : 'hold'} the worker's worktree or scratch`);
+                if (taken?.changed)
+                    project.store.appendEvent(workerId, { type: 'lifecycle', message: `destroy blocked: ${reason}` });
+                result = { workerId, state: 'incomplete', remaining: ['process', ...resources], error: !taken ? 'Worker termination is not proven; retry cleanup later'
+                        : taken.blocker.kind === 'unreadable' ? reason : `Worker termination is not proven: ${reason}; retry cleanup later` };
             }
             else {
                 persist('cleaning', resources);