@lchase/kiln 0.3.0 → 0.4.0

package/dist/reliability.js

@@ -0,0 +1,441 @@
+import { spawnSync } from "node:child_process";
+import { randomBytes } from "node:crypto";
+import { appendFileSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, statSync, writeFileSync, } from "node:fs";
+import { dirname, join, resolve, sep } from "node:path";
+import { z } from "zod";
+import { jobBasename } from "./paths.js";
+import { FAILURE_REASONS } from "./reconcile.js";
+/**
+ * Attempt ledger under `<kilnDir>/attempts/<jobId>.json`.
+ * Job files stay free of stored state. Tick, startup, and `kiln retry` are
+ * the only writers. A second pass that sees the same run id does not append.
+ *
+ * Failure backoff follows Symphony's failure rule, adapted to kiln attempts:
+ * `min(10s × 2^(n-1), 5 minutes)` where n is the attempt that just failed.
+ */
+export const RETRY_BASE_MS = 10_000;
+export const MAX_RETRY_BACKOFF_MS = 5 * 60 * 1000;
+const AttemptSchema = z.object({
+    n: z.number().int().min(1),
+    at: z.string().min(1),
+    reason: z.enum(FAILURE_REASONS),
+    runId: z.string().min(1).optional(),
+    pid: z.number().int().nonnegative().optional(),
+    detail: z.string().min(1).optional(),
+});
+const LedgerSchema = z.object({
+    jobId: z.string().min(1),
+    attempts: z.array(AttemptSchema),
+    nextRetryAt: z.string().min(1).optional(),
+    /** Runs a manual retry has retired. Observe skips them. */
+    ignoredRunIds: z.array(z.string().min(1)).optional(),
+});
+const RUN_ID = /^[A-Za-z0-9][A-Za-z0-9._+-]*$/;
+/** Elapsed time must be strictly greater than the timeout. `<= 0` disables stall detection. */
+export function isStalled(activityMs, nowMs, timeoutMs) {
+    if (timeoutMs <= 0 || activityMs === undefined)
+        return false;
+    return nowMs - activityMs > timeoutMs;
+}
+/** Delay before the next launch after `attempt` failures. Attempt 1 waits 10s. */
+export function retryDelayMs(attempt) {
+    const n = Math.max(1, Math.floor(attempt));
+    const shift = Math.min(n - 1, 20);
+    return Math.min(RETRY_BASE_MS * 2 ** shift, MAX_RETRY_BACKOFF_MS);
+}
+export function attemptLedgerPath(kilnDir, jobId) {
+    return join(kilnDir, "attempts", `${jobBasename(jobId)}.json`);
+}
+export function readAttemptLedger(kilnDir, jobId) {
+    const path = attemptLedgerPath(kilnDir, jobId);
+    if (!existsSync(path))
+        return undefined;
+    try {
+        const parsed = LedgerSchema.parse(JSON.parse(readFileSync(path, "utf8")));
+        if (parsed.jobId !== jobId)
+            return "corrupt";
+        return parsed;
+    }
+    catch {
+        return "corrupt";
+    }
+}
+/**
+ * Clear the attempt budget and hide `runId` so the next tick can launch.
+ * A corrupt ledger is replaced: this is the explicit recovery path.
+ */
+export function resetAttemptLedger(kilnDir, jobId, runId) {
+    const current = readAttemptLedger(kilnDir, jobId);
+    const ignored = new Set(current && current !== "corrupt" ? (current.ignoredRunIds ?? []) : []);
+    if (runId)
+        ignored.add(runId);
+    const ledger = {
+        jobId,
+        attempts: [],
+        ...(ignored.size > 0 ? { ignoredRunIds: [...ignored] } : {}),
+    };
+    writeLedger(kilnDir, ledger);
+    return ledger;
+}
+/**
+ * Stall, orphan, crash, and non-zero-exit handling for one observation.
+ * Caller holds the tick lock. Safe to run twice: a run id is recorded once,
+ * and worktree cleanup no-ops once the tree is gone or must be kept.
+ */
+export function recoverRuns(config, jobs, options) {
+    const notes = [];
+    for (const job of jobs) {
+        if (job.error)
+            continue;
+        const existing = readAttemptLedger(config.kilnDir, job.jobId);
+        if (existing === "corrupt")
+            continue;
+        const pending = pendingFailure(job, config, options);
+        if (!pending)
+            continue;
+        if (pending.killPid !== undefined && !killTree(pending.killPid))
+            continue;
+        const recorded = recordFailure(config, job.jobId, existing, pending, options.now);
+        if (recorded.created) {
+            appendLog(job.logPath, recorded.ledger.attempts.at(-1));
+            notes.push({
+                jobId: job.jobId,
+                kind: kindFor(pending.reason),
+                reason: pending.reason,
+                attempt: recorded.ledger.attempts.length,
+                ...(recorded.ledger.nextRetryAt ? { nextRetryAt: recorded.ledger.nextRetryAt } : {}),
+            });
+        }
+        const checkout = job.run?.checkout ?? config.repos[job.repo]?.path;
+        const runId = pending.runId ?? job.run?.runId;
+        if (checkout && runId)
+            cleanupRunWorktree(checkout, runId);
+    }
+    return notes;
+}
+function pendingFailure(job, config, options) {
+    if (job.run?.gateDecision === "reject")
+        return undefined;
+    if (job.run?.status === "ok")
+        return undefined;
+    if (job.run?.status === "suspended")
+        return undefined;
+    if (job.run?.status === "failed" && job.run.finishedReason === "cancelled")
+        return undefined;
+    const now = options.now.getTime();
+    const launchingLive = job.launching && job.launchPid !== undefined && isPidAlive(job.launchPid);
+    if (launchingLive && job.launchPid !== undefined && isStalled(job.launchActivityMs, now, config.stallTimeoutMs)) {
+        return {
+            reason: "stalled",
+            pid: job.launchPid,
+            killPid: job.launchPid,
+            detail: "no yak event or log output",
+        };
+    }
+    if (launchingLive)
+        return undefined;
+    const run = job.run;
+    const runLive = run?.status === "alive" && run.gateDecision !== "approve" && run.pid !== undefined && isPidAlive(run.pid);
+    if (runLive && run?.pid !== undefined && isStalled(run.activityMs, now, config.stallTimeoutMs)) {
+        return {
+            reason: "stalled",
+            runId: run.runId,
+            pid: run.pid,
+            killPid: run.pid,
+            detail: "no yak event or log output",
+        };
+    }
+    if (runLive)
+        return undefined;
+    if (run && (run.status === "dead" || (run.status === "alive" && run.pid !== undefined && !isPidAlive(run.pid)))) {
+        return options.startup
+            ? { reason: "orphaned", runId: run.runId, pid: run.pid, detail: "process gone at startup" }
+            : { reason: "crashed", runId: run.runId, pid: run.pid, detail: "process exited" };
+    }
+    if (run?.status === "failed") {
+        return { reason: "failed", runId: run.runId, pid: run.pid, detail: "non-zero exit" };
+    }
+    if (job.launchFailed) {
+        return {
+            reason: "crashed",
+            ...(job.launchPid !== undefined ? { pid: job.launchPid } : {}),
+            detail: "launch exited before a journal",
+        };
+    }
+    return undefined;
+}
+function recordFailure(config, jobId, existing, pending, now) {
+    const ledger = existing ?? { jobId, attempts: [] };
+    if (alreadyRecorded(ledger, pending))
+        return { ledger, created: false };
+    const attempt = {
+        n: ledger.attempts.length + 1,
+        at: now.toISOString(),
+        reason: pending.reason,
+        ...(pending.runId ? { runId: pending.runId } : {}),
+        ...(pending.pid !== undefined ? { pid: pending.pid } : {}),
+        detail: pending.detail,
+    };
+    const attempts = [...ledger.attempts, attempt];
+    const exhausted = attempts.length >= config.maxAttempts;
+    const next = exhausted ? undefined : new Date(now.getTime() + retryDelayMs(attempts.length)).toISOString();
+    const written = {
+        jobId,
+        attempts,
+        ...(next ? { nextRetryAt: next } : {}),
+        ...(ledger.ignoredRunIds && ledger.ignoredRunIds.length > 0 ? { ignoredRunIds: ledger.ignoredRunIds } : {}),
+    };
+    writeLedger(config.kilnDir, written);
+    return { ledger: written, created: true };
+}
+function alreadyRecorded(ledger, pending) {
+    if (pending.runId)
+        return ledger.attempts.some((attempt) => attempt.runId === pending.runId);
+    if (pending.pid !== undefined) {
+        return ledger.attempts.some((attempt) => attempt.runId === undefined && attempt.pid === pending.pid);
+    }
+    return ledger.attempts.some((attempt) => attempt.runId === undefined && attempt.pid === undefined && attempt.reason === pending.reason);
+}
+function kindFor(reason) {
+    if (reason === "stalled")
+        return "stall";
+    if (reason === "orphaned")
+        return "orphan";
+    if (reason === "crashed")
+        return "crash";
+    return "fail";
+}
+function appendLog(logPath, attempt) {
+    if (!logPath || !attempt)
+        return;
+    try {
+        mkdirSync(dirname(logPath), { recursive: true });
+        appendFileSync(logPath, `[kiln] ${attempt.reason}: ${attempt.detail ?? "failed"}\n`);
+    }
+    catch {
+        // The ledger is the record. A missing log must not drop the attempt.
+    }
+}
+function writeLedger(kilnDir, ledger) {
+    const path = attemptLedgerPath(kilnDir, ledger.jobId);
+    mkdirSync(dirname(path), { recursive: true });
+    const tmp = `${path}.${process.pid}.${randomBytes(4).toString("hex")}.tmp`;
+    writeFileSync(tmp, `${JSON.stringify(LedgerSchema.parse(ledger), null, 2)}\n`);
+    renameSync(tmp, path);
+}
+/**
+ * Drop a failed run's worktree only when that loses no commits.
+ * A branch with commits that are not on HEAD and not on a remote is kept,
+ * and so is a dirty worktree. The checkout's own worktree is not removed.
+ */
+export function cleanupRunWorktree(checkout, runId) {
+    if (!safeRunId(runId))
+        return "kept";
+    const root = resolve(checkout);
+    const branch = `yak/${runId}`;
+    const worktree = resolve(root, ".yak", "worktrees", runId);
+    const worktreeRoot = resolve(root, ".yak", "worktrees");
+    if (worktree === root || !worktree.startsWith(`${worktreeRoot}${sep}`))
+        return "kept";
+    const branchExists = gitOk(root, ["show-ref", "--verify", "--quiet", `refs/heads/${branch}`]);
+    const worktreeExists = existsSync(worktree);
+    if (!branchExists && !worktreeExists)
+        return "absent";
+    if (checkedOut(root, branch))
+        return "kept";
+    if (branchExists && hasUnpushedCommits(root, branch))
+        return "kept";
+    if (worktreeExists && worktreeDirty(worktree))
+        return "kept";
+    if (worktreeExists && worktreeListed(root, worktree)) {
+        gitOk(root, ["worktree", "remove", worktree]);
+    }
+    if (existsSync(worktree))
+        return "kept";
+    if (branchExists)
+        gitOk(root, ["branch", "-D", branch]);
+    const stillBranch = gitOk(root, ["show-ref", "--verify", "--quiet", `refs/heads/${branch}`]);
+    return existsSync(worktree) || stillBranch ? "kept" : "removed";
+}
+function safeRunId(runId) {
+    return RUN_ID.test(runId) && !runId.includes("..");
+}
+function checkedOut(checkout, branch) {
+    const result = git(checkout, ["rev-parse", "--abbrev-ref", "HEAD"]);
+    return result.status === 0 && result.stdout.trim() === branch;
+}
+/** Commits on the branch that are neither in HEAD nor on any remote. Failure keeps the branch. */
+function hasUnpushedCommits(checkout, branch) {
+    const result = git(checkout, ["rev-list", "--count", branch, "--not", "HEAD", "--not", "--remotes"]);
+    if (result.status !== 0)
+        return true;
+    const count = Number(result.stdout.trim());
+    return !Number.isInteger(count) || count > 0;
+}
+function worktreeDirty(worktree) {
+    const result = git(worktree, ["status", "--porcelain"]);
+    if (result.status !== 0)
+        return true;
+    return result.stdout.trim().length > 0;
+}
+function worktreeListed(checkout, worktree) {
+    const result = git(checkout, ["worktree", "list", "--porcelain"]);
+    if (result.status !== 0)
+        return false;
+    return result.stdout.split("\n").some((line) => line.startsWith("worktree ") && resolve(line.slice("worktree ".length)) === worktree);
+}
+function gitOk(cwd, args) {
+    return git(cwd, args).status === 0;
+}
+function git(cwd, args) {
+    const result = spawnSync("git", args, {
+        cwd,
+        encoding: "utf8",
+        env: { ...process.env, GIT_TERMINAL_PROMPT: "0" },
+    });
+    return { status: result.status ?? 1, stdout: result.stdout ?? "" };
+}
+const PROTECTED = new Set([process.pid, process.ppid].filter((pid) => Number.isInteger(pid) && pid > 1));
+const KILL_WAIT_MS = 2_000;
+const KILL_POLL_MS = 20;
+/**
+ * Kill `pid` and its children. Returns true once the pid is gone or a zombie.
+ * SIGKILL is not instant: a single check can still see the old state, and a
+ * zombie answers `kill(pid, 0)` until it is reaped, so this polls.
+ */
+export function killTree(pid) {
+    if (!Number.isInteger(pid) || pid <= 1 || PROTECTED.has(pid))
+        return false;
+    const deadline = Date.now() + KILL_WAIT_MS;
+    do {
+        signalTree(pid);
+        if (!isPidAlive(pid))
+            return true;
+        sleepMs(KILL_POLL_MS);
+    } while (Date.now() < deadline);
+    signalTree(pid);
+    return !isPidAlive(pid);
+}
+function signalTree(pid) {
+    const victims = new Set();
+    const stack = [pid];
+    while (stack.length > 0) {
+        const current = stack.pop();
+        if (current === undefined || victims.has(current) || PROTECTED.has(current) || current <= 1)
+            continue;
+        victims.add(current);
+        for (const child of childPids(current))
+            stack.push(child);
+    }
+    const group = processGroup(pid);
+    if (group === pid && !PROTECTED.has(group))
+        signal(-pid, "SIGKILL");
+    for (const victim of victims)
+        signal(victim, "SIGKILL");
+}
+function childPids(pid) {
+    try {
+        const text = readFileSync(`/proc/${pid}/task/${pid}/children`, "utf8").trim();
+        if (text) {
+            return text
+                .split(/\s+/)
+                .map((part) => Number(part))
+                .filter((child) => Number.isInteger(child) && child > 1);
+        }
+    }
+    catch {
+        // Fall through to a PPid scan. Some kernels leave `children` empty.
+    }
+    return childrenByPpid(pid);
+}
+function childrenByPpid(parent) {
+    let entries;
+    try {
+        entries = readdirSync("/proc");
+    }
+    catch {
+        return [];
+    }
+    const found = [];
+    for (const entry of entries) {
+        if (!/^\d+$/.test(entry))
+            continue;
+        const child = Number(entry);
+        if (child <= 1 || child === parent)
+            continue;
+        if (Number(statusValue(child, "PPid")) === parent)
+            found.push(child);
+    }
+    return found;
+}
+function processGroup(pid) {
+    const state = procFields(pid);
+    const group = state ? Number(state[2]) : Number.NaN;
+    return Number.isInteger(group) ? group : undefined;
+}
+function procState(pid) {
+    return procFields(pid)?.[0] ?? statusValue(pid, "State")?.slice(0, 1);
+}
+function statusValue(pid, key) {
+    try {
+        const text = readFileSync(`/proc/${pid}/status`, "utf8");
+        const line = text.split("\n").find((entry) => entry.startsWith(`${key}:`));
+        return line?.slice(key.length + 1).trim();
+    }
+    catch {
+        return undefined;
+    }
+}
+/** Fields of `/proc/pid/stat` after the comm, starting at `state`. */
+function procFields(pid) {
+    try {
+        const stat = readFileSync(`/proc/${pid}/stat`, "utf8");
+        const end = stat.lastIndexOf(")");
+        if (end < 0)
+            return undefined;
+        return stat.slice(end + 2).split(" ");
+    }
+    catch {
+        return undefined;
+    }
+}
+function signal(pid, sig) {
+    try {
+        process.kill(pid, sig);
+    }
+    catch {
+        // Already gone, or not our process group.
+    }
+}
+/**
+ * True when `pid` is still running. A zombie is not alive: `kill(pid, 0)`
+ * succeeds until the parent reaps it, which would keep a killed run in flight.
+ * EPERM means the pid exists and we cannot inspect it, so it still counts.
+ */
+export function isPidAlive(pid) {
+    if (!Number.isInteger(pid) || pid <= 0)
+        return false;
+    try {
+        process.kill(pid, 0);
+    }
+    catch (error) {
+        return isErrno(error, "EPERM");
+    }
+    const state = procState(pid);
+    return state !== "Z" && state !== "X";
+}
+function sleepMs(ms) {
+    Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+}
+function isErrno(error, code) {
+    return typeof error === "object" && error !== null && "code" in error && error.code === code;
+}
+/** Latest mtime, folded with journal event times by the driver. */
+export function fileMtimeMs(path) {
+    try {
+        return statSync(path).mtimeMs;
+    }
+    catch {
+        return undefined;
+    }
+}

package/dist/report.js

@@ -1,6 +1,7 @@
 import { readFileSync } from "node:fs";
 import { createRequire } from "node:module";
 import { dirname, join } from "node:path";
+import { readBoardFiles, readReviewRounds } from "./board.js";
 import { INVESTIGATION_WORKFLOW } from "./findings.js";
 import { inFlightCount, isDone, isLanded, jobState, } from "./reconcile.js";
 export function shellQuote(value) {
@@ -29,6 +30,52 @@ export function gateAnswerCommand(checkout, runId, gate, choice, title) {
     const titlePrefix = title ? `KILN_TASK_TITLE=${shellQuote(title)} ` : "";
     return `cd ${shellQuote(checkout)} && mkdir -p ${pendingDir} && printf '%s\\n' ${shellQuote(answer)} > ${answerFile} && ${titlePrefix}${yakInvocation("resume", runId)}`;
 }
+/**
+ * Mark a workflow's leaf steps from yak journal events.
+ * A later `step.started` reopens a step that already completed (the next loop round).
+ * Events whose id is not in the plan are ignored when the plan is non-empty,
+ * so a loop container does not sit beside its body. An empty plan uses
+ * first-seen event order.
+ */
+export function stepProgress(plan, events) {
+    const order = [...plan];
+    const phase = new Map();
+    for (const id of order)
+        phase.set(id, "pending");
+    for (const event of events) {
+        if (!event.stepId)
+            continue;
+        if (!phase.has(event.stepId)) {
+            if (plan.length > 0)
+                continue;
+            order.push(event.stepId);
+            phase.set(event.stepId, "pending");
+        }
+        if (event.t === "step.started")
+            phase.set(event.stepId, "current");
+        else if (event.t === "step.completed")
+            phase.set(event.stepId, "done");
+        else if (event.t === "step.failed")
+            phase.set(event.stepId, "failed");
+    }
+    return order.map((id) => ({ id, phase: phase.get(id) ?? "pending" }));
+}
+/** Detail of the latest step that failed and has not started or completed since. */
+export function latestStepFailure(events) {
+    const settled = new Set();
+    for (let index = events.length - 1; index >= 0; index -= 1) {
+        const event = events[index];
+        if (!event)
+            continue;
+        if (event.t === "step.started" || event.t === "step.completed") {
+            settled.add(event.stepId);
+            continue;
+        }
+        if (event.t === "step.failed" && !settled.has(event.stepId) && event.detail)
+            return event.detail;
+    }
+    return undefined;
+}
 /**
  * Derived state for every job, or for one id.
  * Throws `No job <id>` when that id is not on disk, same as `kiln status`.
@@ -76,6 +123,16 @@ export function statusDetails(job) {
         lines.push(`log ${job.logPath}`);
     if (job.state === "failed" && job.launchError)
         lines.push(job.launchError);
+    if (job.ledgerError)
+        lines.push(job.ledgerError);
+    if (job.attemptCount > 0)
+        lines.push(`attempts ${job.attemptCount}`);
+    if (job.lastFailureReason)
+        lines.push(`reason ${job.lastFailureReason}`);
+    if (job.nextRetryAt)
+        lines.push(`next retry ${job.nextRetryAt}`);
+    if (job.state === "dead" && job.attemptCount > 0)
+        lines.push(`retry: kiln retry ${job.jobId}`);
     if (job.noReadableApproval)
         lines.push("no readable approval decision");
     if (job.cancelCommand)
@@ -122,6 +179,10 @@ function toStatusJob(job, observation) {
             ...(job.logPath ? { logPath: job.logPath } : {}),
             ...(job.findingsPath ? { findingsPath: job.findingsPath } : {}),
             launchFailed: job.launchFailed,
+            attemptCount: job.attemptCount ?? 0,
+            maxAttempts: observation.maxAttempts,
+            ...(job.lastFailureReason ? { lastFailureReason: job.lastFailureReason } : {}),
+            ...(job.nextRetryAt ? { nextRetryAt: job.nextRetryAt } : {}),
             invalidReason: job.error,
             prPending: false,
             prClosedUnmerged: false,
@@ -129,6 +190,10 @@ function toStatusJob(job, observation) {
             gates: [],
             canApprove: false,
             canReject: false,
+            steps: progressFor(job),
+            ...(job.createdAt ? { createdAt: job.createdAt } : {}),
+            ...(job.logTail ? { logTail: job.logTail } : {}),
+            ...boardFields(job),
         };
     }
     const state = jobState(job, observation);
@@ -136,6 +201,7 @@ function toStatusJob(job, observation) {
     const prNumber = job.delivery?.prNumber;
     const runId = job.run?.runId;
     const showBranch = (state === "done" || state === "in-review") && runId !== undefined;
+    const failureDetail = job.run?.stepEvents ? latestStepFailure(job.run.stepEvents) : undefined;
     return {
         jobId: job.jobId,
         title: job.title,
@@ -153,20 +219,54 @@ function toStatusJob(job, observation) {
         ...(job.specPath ? { specPath: job.specPath } : {}),
         ...(job.findingsPath ? { findingsPath: job.findingsPath } : {}),
         launchFailed: job.launchFailed,
+        attemptCount: job.attemptCount ?? 0,
+        maxAttempts: observation.maxAttempts,
+        ...(job.lastFailureReason ? { lastFailureReason: job.lastFailureReason } : {}),
+        ...(job.nextRetryAt && !job.retriesExhausted ? { nextRetryAt: job.nextRetryAt } : {}),
+        ...(job.ledgerError ? { ledgerError: job.ledgerError } : {}),
         ...(job.launchError ? { launchError: job.launchError } : {}),
         ...(job.delivery?.error ? { deliveryError: job.delivery.error } : {}),
         ...(job.delivery?.viewError ? { viewError: job.delivery.viewError } : {}),
         prPending: state === "in-review" && !prNumber,
         prClosedUnmerged: job.delivery?.prState === "CLOSED",
         noReadableApproval: state === "failed" && job.run?.status === "ok" && !job.run.gateDecision,
-        ...(state === "dead" && job.run
+        ...(job.run && (job.run.status === "dead" || state === "dead")
             ? { cancelCommand: `fix: cd ${shellQuote(job.run.checkout)} && ${yakInvocation("cancel", job.run.runId)}` }
             : {}),
         gates,
         canApprove: gates.some((gate) => gate.choices.includes("approve")),
         canReject: gates.some((gate) => gate.choices.includes("reject")),
+        steps: progressFor(job),
+        ...(job.createdAt ? { createdAt: job.createdAt } : {}),
+        ...(job.run?.startedAt ? { startedAt: job.run.startedAt } : {}),
+        ...(job.run?.finishedAt ? { finishedAt: job.run.finishedAt } : {}),
+        ...(job.logTail ? { logTail: job.logTail } : {}),
+        ...(job.diffSummary ? { diffSummary: job.diffSummary } : {}),
+        ...(failureDetail ? { failureDetail } : {}),
+        ...boardFields(job),
     };
 }
+function boardFields(job) {
+    const runDir = job.run ? join(job.run.checkout, ".runs", job.run.runId) : undefined;
+    const rounds = readReviewRounds(runDir);
+    const files = readBoardFiles({
+        ...(runDir ? { runDir } : {}),
+        ...(job.specPath ? { specPath: job.specPath } : {}),
+        ...(job.findingsPath ? { findingsPath: job.findingsPath } : {}),
+        ...(job.workflow !== INVESTIGATION_WORKFLOW && job.diffSummary ? { diffSummary: job.diffSummary } : {}),
+        ...(job.logTail ? { logTail: job.logTail } : {}),
+        ...(job.workflow ? { workflow: job.workflow } : {}),
+        gates: job.run?.gates ?? [],
+    });
+    return {
+        ...(rounds.length > 0 ? { rounds } : {}),
+        ...(files.length > 0 ? { files } : {}),
+    };
+}
+function progressFor(job) {
+    const plan = job.run?.stepPlan?.length ? job.run.stepPlan : (job.stepPlan ?? []);
+    return stepProgress(plan, job.run?.stepEvents ?? []);
+}
 function toGateView(gate) {
     const choices = gate.choices?.length ? gate.choices : ["approve", "reject"];
     return {
@@ -217,7 +317,7 @@ function blockReasons(job, observation) {
     }
     return reasons;
 }
-export function renderTick(observation, actions, launches, delivers = [], findings = []) {
+export function renderTick(observation, actions, launches, delivers = [], findings = [], notes = []) {
     const lines = [`tick  inFlight=${inFlightCount(observation)}  maxConcurrent=${observation.maxConcurrent}`];
     for (const job of observation.jobs) {
         if (job.error)
@@ -237,14 +337,18 @@ export function renderTick(observation, actions, launches, delivers = [], findin
             ? `findings ${recorded.jobId}  ${recorded.path}`
             : `findings ${recorded.jobId}  missing`);
     }
+    for (const note of notes) {
+        const next = note.nextRetryAt ? `  next ${note.nextRetryAt}` : "  dead";
+        lines.push(`${note.kind} ${note.jobId}  ${note.reason}  attempt ${note.attempt}${next}`);
+    }
     for (const launch of launches) {
-        lines.push(`launch ${launch.jobId}  ${launch.repo}`);
+        lines.push(`${launch.retry ? "retry" : "launch"} ${launch.jobId}  ${launch.repo}`);
         for (const line of launch.output.split("\n")) {
             if (line.trim())
                 lines.push(`  ${line}`);
         }
     }
-    if (!actions.some((action) => action.type === "launch") &&
+    if (!actions.some((action) => action.type === "launch" || action.type === "retry") &&
         launches.length === 0 &&
         delivers.length === 0 &&
         findings.length === 0) {

package/dist/schemas.js

@@ -26,9 +26,20 @@ export const JiraConfigSchema = z.object({
     email: z.string().optional(),
     apiToken: z.string().optional(),
 });
+/** No yak event and no log output for this long stalls a running job. `<= 0` disables it. */
+export const DEFAULT_STALL_TIMEOUT_MS = 15 * 60 * 1000;
+/** Transient failures retry until this many attempts, then the job is dead. */
+export const DEFAULT_MAX_ATTEMPTS = 3;
 export const KilnConfigFileSchema = z.object({
     note: z.string().min(1).optional(),
     maxConcurrent: z.number().int().min(1).optional(),
+    /**
+     * Milliseconds without a yak event or log output before a running job is
+     * killed and retried. Default 15 minutes. `0` or negative disables stall detection.
+     */
+    stallTimeoutMs: z.number().int().optional(),
+    /** Transient failures before the job is dead. Default 3. */
+    maxAttempts: z.number().int().min(1).optional(),
     adapter: z.enum(["mock", "claude-code"]).optional(),
     repos: z.record(z.string(), RepoEntrySchema),
     jira: JiraConfigSchema.optional(),
@@ -79,7 +90,9 @@ export const JobSchema = z.object({
  * `deliver=pr`. It does not enter `in-review`. A reject, a failure, a dead
  * process, and an ok run with no readable approval do not unblock an
  * after-merged dependent. A stacked job unblocks when its one upstream has
- * landed, which includes `in-review`.
+ * landed, which includes `in-review`. `dead` is also a job whose transient
+ * failures have used `maxAttempts`. Those attempts live under the kiln
+ * directory, not in this file.
  */
 export const JOB_STATES = [
     "queued",