@lchase/kiln 0.3.0 → 0.5.0

package/dist/driver.js

@@ -1,16 +1,19 @@
 import { spawn, spawnSync } from "node:child_process";
 import { randomUUID } from "node:crypto";
-import { closeSync, existsSync, linkSync, mkdirSync, openSync, readdirSync, readFileSync, rmSync, writeFileSync, } from "node:fs";
+import { closeSync, existsSync, linkSync, mkdirSync, openSync, readdirSync, readFileSync, readSync, rmSync, statSync, writeFileSync, } from "node:fs";
 import { basename, dirname, isAbsolute, join } from "node:path";
+import { buildAttention } from "./attention.js";
 import { clearLaunchError, deliverJob, readDelivery, readLaunchError, resolveLaunchBase, writeDelivery, writeLaunchError, isMerged, } from "./delivery.js";
+import { feedbackSpec, openFeedbackRound, planFeedback, readFeedbackLedger, writeFeedbackLedger, } from "./feedback.js";
 import { appendInvestigationFindings, existingFindingsPath, findingsFilePath, INVESTIGATION_WORKFLOW, readFindingsMarkdown, writeFindings, } from "./findings.js";
 import { defaultHost } from "./host.js";
 import { readJobSpec, resolveJobSpec } from "./jobs.js";
 import { jobBasename, packageRoot } from "./paths.js";
 import { decide, isLanded, jobState, } from "./reconcile.js";
+import { fileMtimeMs, isPidAlive, readAttemptLedger, recoverRuns, resetAttemptLedger, } from "./reliability.js";
 import { gateAnswerCommand, renderStatus, renderTick, statusView, yakCliPath, } from "./report.js";
 import { JobSchema } from "./schemas.js";
-import { materializeWorkflow, workflowInputs, workflowNeedsTestCommand } from "./workflow.js";
+import { leafStepIds, materializeWorkflow, workflowInputs, workflowNeedsTestCommand, workflowStepPlan } from "./workflow.js";
 const TAG_PREFIX = "kiln:";
 export function jobTag(jobId) {
     return `${TAG_PREFIX}${jobId}`;
@@ -25,16 +28,30 @@ export function parseJobTag(tag) {
 }
 /** The derived view `kiln status` prints and `kiln serve` renders. */
 export function readStatus(config, jobId, host = defaultHost()) {
-    return statusView(observe(config, host).observation, jobId);
+    const view = statusView(observe(config, host).observation, jobId);
+    return { ...view, attention: buildAttention(view.jobs) };
 }
 export function runStatus(config, jobId, host = defaultHost()) {
     return renderStatus(readStatus(config, jobId, host));
 }
-export function runTick(config, host = defaultHost()) {
+export function runTick(config, host = defaultHost(), options = {}) {
+    const startup = options.startup ?? true;
+    const now = options.now ?? new Date();
     return withTickLock(config.kilnDir, () => {
+        const notes = recoverRuns(config, observe(config, host).observation.jobs, { startup, now });
+        const midway = observe(config, host);
+        bindOpenFixRuns(config, midway);
+        const published = publishReadyFixes(config, host, midway);
         const world = observe(config, host);
-        const actions = decide(world.observation);
-        return apply(config, world, actions, host);
+        const feedbackNotes = watchPullRequests(config, world, host);
+        const actions = decide(world.observation, now);
+        return apply(config, world, actions, host, notes, now, [...published, ...feedbackNotes]);
+    });
+}
+/** Startup pass shared by `kiln tick` (via `runTick`) and `kiln serve`. Idempotent. */
+export function startupRecovery(config, host = defaultHost(), now = new Date()) {
+    withTickLock(config.kilnDir, () => {
+        recoverRuns(config, observe(config, host).observation.jobs, { startup: true, now });
     });
 }
 export function answerJob(config, jobId, choice, host = defaultHost()) {
@@ -65,6 +82,74 @@ export function answerJob(config, jobId, choice, host = defaultHost()) {
     }
     return output.endsWith("\n") || output.length === 0 ? output : `${output}\n`;
 }
+/**
+ * Run a failed or dead job again. The board Retry button and `kiln retry`
+ * both call this.
+ * A landed job whose only failure is opening the pull request is handed to
+ * the same deliver path tick uses. Everything else is a new `yak run`, the
+ * same launch tick uses. The attempt ledger is cleared either way, and the
+ * current run id is ignored so a dead job is not stuck on the old journal.
+ * A dead journal is cancelled first.
+ */
+export function retryJob(config, jobId, host = defaultHost()) {
+    return withTickLock(config.kilnDir, () => {
+        const world = observe(config, host);
+        const observed = world.observation.jobs.find((candidate) => candidate.jobId === jobId);
+        if (!observed)
+            throw new Error(`No job ${jobId}.`);
+        if (observed.error)
+            throw new Error(`Job ${jobId} is invalid: ${observed.error}`);
+        const state = jobState(observed, world.observation);
+        if (state !== "failed" && state !== "dead") {
+            throw new Error(`Job ${jobId} is ${state}, not failed or dead.`);
+        }
+        const job = world.jobs.get(jobId);
+        if (!job)
+            throw new Error(`No job ${jobId}.`);
+        const crumb = readBreadcrumbs(config.kilnDir).find((candidate) => candidate.jobId === jobId);
+        if (crumb && pidAlive(crumb.pid)) {
+            throw new Error(`Job ${jobId} is already launching.`);
+        }
+        const cleared = observed.attemptCount ?? 0;
+        resetAttemptLedger(config.kilnDir, jobId, observed.run?.runId);
+        rmSync(breadcrumbPath(config.kilnDir, jobId), { force: true });
+        clearLaunchError(config.kilnDir, jobId);
+        const noun = cleared === 1 ? "attempt" : "attempts";
+        const clearedLine = `cleared ${cleared} ${noun}\n`;
+        if (shouldRedeliver(observed)) {
+            const report = deliverObserved(config, world, jobId, host);
+            if (!report.ok)
+                throw new Error(report.detail);
+            const detail = report.detail.endsWith("\n") || report.detail.length === 0 ? report.detail : `${report.detail}\n`;
+            return `retry ${jobId}  ${job.repo}\n${clearedLine}${detail}`;
+        }
+        if (state === "dead" && observed.run) {
+            try {
+                spawnSync(process.execPath, [yakCliPath(), "cancel", observed.run.runId], {
+                    cwd: observed.run.checkout,
+                    encoding: "utf8",
+                    timeout: 20_000,
+                });
+            }
+            catch {
+                // Cancel is the same cleanup status prints. The new run still supersedes the journal.
+            }
+        }
+        const logPath = join(config.kilnDir, "logs", `${jobBasename(job.id)}.log`);
+        if (existsSync(logPath))
+            writeFileSync(logPath, "");
+        const report = launchJob(config, world, job, host);
+        const output = report.output.endsWith("\n") || report.output.length === 0 ? report.output : `${report.output}\n`;
+        return `retry ${report.jobId}  ${report.repo}\n${clearedLine}${output}`;
+    });
+}
+function shouldRedeliver(job) {
+    if (!isLanded(job))
+        return false;
+    if (job.delivery?.prState === "CLOSED")
+        return false;
+    return Boolean(job.delivery?.error) && !job.delivery?.prNumber;
+}
 function gateForChoice(gates, choice) {
     return (gates.find((gate) => gate.choices?.includes(choice)) ??
         gates.find((gate) => !gate.choices?.length));
@@ -86,27 +171,53 @@ function observe(config, host) {
         if (entry.job)
             records.set(entry.job.id, entry.job);
         const tag = jobTag(entry.meta.jobId);
-        const run = runs.get(tag);
+        const ledgerResult = readAttemptLedger(config.kilnDir, entry.meta.jobId);
+        const ledger = ledgerResult && ledgerResult !== "corrupt" ? ledgerResult : undefined;
+        const feedbackResult = readFeedbackLedger(config.kilnDir, entry.meta.jobId);
+        const feedback = feedbackResult && feedbackResult !== "corrupt" ? feedbackResult : undefined;
+        const repoKnown = entry.job ? Boolean(config.repos[entry.job.repo]) : false;
+        const repo = entry.job ? config.repos[entry.job.repo] : undefined;
+        const deliver = repo?.deliver ?? "local";
+        const deliveryRecord = entry.job && repo && deliver === "pr" ? readDelivery(config.kilnDir, entry.job.id) : undefined;
+        const picked = selectJobRun(runs.get(tag) ?? [], ledger, feedback, deliveryRecord?.runId);
+        const run = picked.run;
         const crumb = breadcrumbs.find((candidate) => candidate.jobId === entry.meta.jobId);
-        if (crumb && run)
+        if (crumb && run && !picked.retired)
             breadcrumbsToClear.push({ path: crumb.path });
         const childAlive = crumb ? pidAlive(crumb.pid) : false;
-        const repoKnown = entry.job ? Boolean(config.repos[entry.job.repo]) : false;
+        const hideBase = picked.fixInProgress && !run;
+        const launching = Boolean(crumb) && childAlive && (hideBase || !run || picked.retired);
+        const launchFailed = Boolean(crumb) && !childAlive && (hideBase || !run || picked.retired);
         const repoError = entry.job && !repoKnown ? `Repo ${entry.job.repo} is not registered.` : undefined;
-        const repo = entry.job ? config.repos[entry.job.repo] : undefined;
-        const deliver = repo?.deliver ?? "local";
+        const deliveryRunId = picked.fixInProgress ? deliveryRecord?.runId : run?.runId;
         const delivery = entry.job && repo && deliver === "pr"
-            ? observedDelivery(config.kilnDir, host, repo.path, entry.job.id, run?.runId)
+            ? observedDelivery(config.kilnDir, host, repo.path, entry.job.id, deliveryRunId)
             : undefined;
         const launchError = entry.job && !run ? readLaunchError(config.kilnDir, entry.job.id) : undefined;
         const stackError = entry.job?.stack && (entry.job.dependsOn?.length ?? 0) !== 1
             ? "stack needs exactly one dependency."
             : undefined;
         const logPath = crumb?.logPath || existingLog(config.kilnDir, entry.meta.jobId);
+        if (run && logPath)
+            run.activityMs = later(run.activityMs, fileMtimeMs(logPath));
+        const launchActivityMs = later(crumb ? fileMtimeMs(crumb.path) : undefined, logPath ? fileMtimeMs(logPath) : undefined);
+        const attemptCount = ledger?.attempts.length ?? 0;
+        const lastFailure = ledger?.attempts.at(-1);
+        const exhausted = attemptCount >= config.maxAttempts &&
+            attemptCount > 0 &&
+            !launching &&
+            (picked.retired || !run || run.status === "dead" || run.status === "failed");
         const specPath = entry.job ? resolveJobSpec(config.kilnDir, entry.job.spec) : undefined;
         const findingsPath = entry.job?.workflow === INVESTIGATION_WORKFLOW
             ? existingFindingsPath(config.kilnDir, entry.job.id)
             : undefined;
+        const logTail = readLogTail(logPath);
+        const stepPlan = entry.job ? workflowStepPlan(entry.job.workflow) : undefined;
+        const diffSummary = run &&
+            run.status === "suspended" &&
+            entry.job?.workflow !== INVESTIGATION_WORKFLOW
+            ? readDiffSummary(run.checkout, run.runId)
+            : undefined;
         jobs.push({
             jobId: entry.meta.jobId,
             repo: entry.job?.repo ?? "",
@@ -119,11 +230,29 @@ function observe(config, host) {
             ...(entry.job?.stack ? { stack: true } : {}),
             ...(delivery ? { delivery } : {}),
             ...(findingsPath ? { findingsPath } : {}),
-            launching: Boolean(crumb) && !run && childAlive,
-            launchFailed: Boolean(crumb) && !run && !childAlive,
+            launching,
+            launchFailed,
             ...(launchError ? { launchError } : {}),
+            ...(crumb ? { launchPid: crumb.pid } : {}),
+            ...(launchActivityMs !== undefined ? { launchActivityMs } : {}),
             ...(logPath ? { logPath } : {}),
+            ...(entry.job?.createdAt ? { createdAt: entry.job.createdAt } : {}),
+            ...(logTail ? { logTail } : {}),
+            ...(diffSummary ? { diffSummary } : {}),
+            ...(stepPlan && stepPlan.length > 0 ? { stepPlan } : {}),
             ...(run ? { run } : {}),
+            attemptCount,
+            ...(lastFailure ? { lastFailureReason: lastFailure.reason } : {}),
+            ...(ledger?.nextRetryAt && !exhausted ? { nextRetryAt: ledger.nextRetryAt } : {}),
+            ...(exhausted ? { retriesExhausted: true } : {}),
+            ...(picked.retired ? { runRetired: true } : {}),
+            ...(ledgerResult === "corrupt" ? { ledgerError: "attempt ledger is unreadable" } : {}),
+            ...(feedback?.rounds.length ? { feedbackRounds: feedback.rounds } : {}),
+            ...(feedback?.exhausted ? { feedbackExhausted: true } : {}),
+            ...(feedback?.ambiguous ? { feedbackAmbiguous: true } : {}),
+            ...(feedbackResult === "corrupt" ? { feedbackLedgerError: "feedback ledger is unreadable" } : {}),
+            ...(picked.fixInProgress ? { fixInProgress: true } : {}),
+            ...(picked.fixAwaitingPublish ? { fixAwaitingPublish: true } : {}),
             ...(entry.meta.error || repoError || stackError
                 ? { error: entry.meta.error ?? repoError ?? stackError }
                 : {}),
@@ -132,13 +261,14 @@ function observe(config, host) {
     return {
         observation: {
             maxConcurrent: config.maxConcurrent,
+            maxAttempts: config.maxAttempts,
             jobs,
             breadcrumbsToClear,
         },
         jobs: records,
     };
 }
-function apply(config, world, actions, host) {
+function apply(config, world, actions, host, notes = [], now = new Date(), feedbackNotes = []) {
     const launches = [];
     const delivers = [];
     const findings = [];
@@ -157,13 +287,28 @@ function apply(config, world, actions, host) {
                 findings.push(recorded);
             continue;
         }
+        if (action.type === "mark-feedback") {
+            markFeedback(config.kilnDir, action.jobId, action.reason);
+            feedbackNotes.push(`${action.reason} ${action.jobId}`);
+            continue;
+        }
+        if (action.type === "start-fix") {
+            const job = world.jobs.get(action.jobId);
+            if (!job) {
+                throw new Error(`Tick planned a fix for ${action.jobId}, which is not in the observed jobs.`);
+            }
+            const report = startFix(config, world, job, host, now);
+            launches.push({ ...report, kind: "fix" });
+            continue;
+        }
         const job = world.jobs.get(action.jobId);
         if (!job) {
             throw new Error(`Tick planned a launch for ${action.jobId}, which is not in the observed jobs.`);
         }
-        launches.push(launchJob(config, world, job, host));
+        const report = launchJob(config, world, job, host);
+        launches.push(action.type === "retry" ? { ...report, retry: true } : report);
     }
-    return renderTick(world.observation, actions, launches, delivers, findings);
+    return renderTick(world.observation, actions, launches, delivers, findings, notes, feedbackNotes);
 }
 /**
  * Copy an approved investigation's markdown into `<kilnDir>/findings`.
@@ -296,11 +441,357 @@ function observedDelivery(kilnDir, host, checkout, jobId, runId) {
         ...(viewError ? { viewError } : {}),
     };
 }
+/**
+ * Attach a fix run id to an open round once its journal exists.
+ * A fix run that the attempt ledger has already recorded is replaced by a
+ * newer journal, so a retry does not stay pointed at the failed run.
+ */
+function bindOpenFixRuns(config, world) {
+    const runs = scanRuns(config);
+    for (const job of world.observation.jobs) {
+        const ledger = readFeedbackLedger(config.kilnDir, job.jobId);
+        if (!ledger || ledger === "corrupt")
+            continue;
+        const open = openFeedbackRound(ledger);
+        if (!open)
+            continue;
+        const attempts = readAttemptLedger(config.kilnDir, job.jobId);
+        const retired = new Set();
+        if (attempts && attempts !== "corrupt") {
+            for (const id of attempts.ignoredRunIds ?? [])
+                retired.add(id);
+            for (const attempt of attempts.attempts) {
+                if (attempt.runId)
+                    retired.add(attempt.runId);
+            }
+        }
+        if (open.fixRunId && !retired.has(open.fixRunId))
+            continue;
+        const baseRunId = readDelivery(config.kilnDir, job.jobId)?.runId;
+        const settled = new Set(ledger.rounds.flatMap((round) => (round.outcome === "pushed" || round.outcome === "fixed") && round.fixRunId ? [round.fixRunId] : []));
+        const candidates = (runs.get(jobTag(job.jobId)) ?? []).filter((run) => {
+            if (run.runId === baseRunId || settled.has(run.runId) || retired.has(run.runId))
+                return false;
+            return true;
+        });
+        candidates.sort((left, right) => (left.runId < right.runId ? 1 : left.runId > right.runId ? -1 : 0));
+        const next = candidates[0];
+        if (!next)
+            continue;
+        open.fixRunId = next.runId;
+        writeFeedbackLedger(config.kilnDir, ledger);
+    }
+}
+/**
+ * Fast-forward the open pull request branch to a fix run that finished ok.
+ * A missing branch or a history that is not a descendant is ambiguous.
+ * A rejected push that is not a fast-forward is ambiguous. Any other push
+ * error is left for the next tick. Kiln does not force-push.
+ */
+function publishReadyFixes(config, host, world) {
+    const notes = [];
+    for (const job of world.observation.jobs) {
+        if (!job.fixAwaitingPublish || !job.run)
+            continue;
+        const ledger = readFeedbackLedger(config.kilnDir, job.jobId);
+        if (!ledger || ledger === "corrupt")
+            continue;
+        const open = openFeedbackRound(ledger);
+        if (!open || open.fixRunId !== job.run.runId)
+            continue;
+        const record = readDelivery(config.kilnDir, job.jobId);
+        const checkout = job.run.checkout;
+        const source = `yak/${job.run.runId}`;
+        if (!record?.branch) {
+            notes.push(`feedback ${job.jobId} failed: no pull request branch to publish`);
+            continue;
+        }
+        let sha;
+        try {
+            sha = host.git.revParse(checkout, source);
+        }
+        catch (error) {
+            markFeedback(config.kilnDir, job.jobId, "feedback-ambiguous");
+            notes.push(`feedback-ambiguous ${job.jobId}  ${errorMessage(error)}`);
+            continue;
+        }
+        if (sha === open.headSha) {
+            open.outcome = "fixed";
+            writeFeedbackLedger(config.kilnDir, ledger);
+            notes.push(`fixed ${job.jobId}  ${sha}`);
+            continue;
+        }
+        const ancestor = host.git.isAncestor(checkout, open.headSha, sha);
+        if (!ancestor) {
+            markFeedback(config.kilnDir, job.jobId, "feedback-ambiguous");
+            notes.push(`feedback-ambiguous ${job.jobId}  ${source} is not based on ${open.headSha}`);
+            continue;
+        }
+        try {
+            host.git.publishBranch(checkout, source, record.branch, record.base);
+        }
+        catch (error) {
+            const message = errorMessage(error);
+            if (isNonFastForward(message)) {
+                markFeedback(config.kilnDir, job.jobId, "feedback-ambiguous");
+                notes.push(`feedback-ambiguous ${job.jobId}  ${message}`);
+            }
+            else {
+                notes.push(`feedback ${job.jobId} failed: ${message}`);
+            }
+            continue;
+        }
+        writeDelivery(config.kilnDir, {
+            jobId: record.jobId,
+            runId: record.runId,
+            branch: record.branch,
+            base: record.base,
+            sha,
+            pushed: true,
+            ...(record.prNumber ? { prNumber: record.prNumber } : {}),
+            ...(record.prUrl ? { prUrl: record.prUrl } : {}),
+        });
+        open.outcome = "pushed";
+        writeFeedbackLedger(config.kilnDir, ledger);
+        notes.push(`pushed ${job.jobId}  ${record.branch}  ${sha}`);
+    }
+    return notes;
+}
+function isNonFastForward(message) {
+    return /non-fast-forward|\[rejected\]|failed to push some refs/i.test(message);
+}
+/**
+ * Read each open pull request once per tick and attach a plan.
+ * `readStatus` does not call this. A host with no feedback reader does not watch.
+ */
+function watchPullRequests(config, world, host) {
+    const notes = [];
+    if (!host.feedback)
+        return notes;
+    for (const job of world.observation.jobs) {
+        if (job.feedbackLedgerError && job.deliver === "pr" && job.delivery?.prNumber) {
+            notes.push(`${job.feedbackLedgerError} ${job.jobId}`);
+        }
+        if (!shouldWatch(job) || !job.run)
+            continue;
+        const ledger = readFeedbackLedger(config.kilnDir, job.jobId);
+        if (ledger === "corrupt")
+            continue;
+        let snapshot;
+        try {
+            snapshot = host.feedback.read(job.run.checkout, job.delivery?.prNumber ?? 0);
+        }
+        catch (error) {
+            notes.push(`feedback ${job.jobId} failed: ${errorMessage(error)}`);
+            continue;
+        }
+        job.feedbackPlan = planFeedback({
+            maxFixRounds: config.maxFixRounds,
+            ledger,
+            snapshot,
+        });
+    }
+    return notes;
+}
+function shouldWatch(job) {
+    if (job.deliver !== "pr" || job.workflow === INVESTIGATION_WORKFLOW)
+        return false;
+    if (job.error || job.feedbackLedgerError || job.feedbackExhausted || job.feedbackAmbiguous)
+        return false;
+    if (job.fixInProgress || job.fixAwaitingPublish || job.launching)
+        return false;
+    if (!job.delivery?.prNumber)
+        return false;
+    if (job.delivery.merged || job.delivery.prState === "MERGED" || job.delivery.prState === "CLOSED")
+        return false;
+    return isLanded(job);
+}
+function markFeedback(kilnDir, jobId, reason) {
+    const existing = readFeedbackLedger(kilnDir, jobId);
+    const ledger = existing && existing !== "corrupt" ? existing : { jobId, rounds: [] };
+    if (reason === "feedback-exhausted")
+        ledger.exhausted = true;
+    else
+        ledger.ambiguous = true;
+    writeFeedbackLedger(kilnDir, ledger);
+}
+/**
+ * Open one feedback round and start the fix workflow on the pull request head.
+ * The round is written only after that commit resolves, so a missing sha
+ * retries on the next tick without consuming a round.
+ */
+function startFix(config, world, job, host, now) {
+    const observed = world.observation.jobs.find((candidate) => candidate.jobId === job.id);
+    const plan = observed?.feedbackPlan;
+    const repo = config.repos[job.repo];
+    if (!repo || !plan || plan.action !== "fix") {
+        return { jobId: job.id, repo: job.repo, output: `Job ${job.id} has no fix to start.` };
+    }
+    let headSha;
+    try {
+        headSha = resolveCommit(host, repo.path, plan.headSha);
+    }
+    catch (error) {
+        return { jobId: job.id, repo: job.repo, output: errorMessage(error) };
+    }
+    const existing = readFeedbackLedger(config.kilnDir, job.id);
+    const ledger = existing && existing !== "corrupt" ? existing : { jobId: job.id, rounds: [] };
+    if (openFeedbackRound(ledger)) {
+        return { jobId: job.id, repo: job.repo, output: `Job ${job.id} already has an open fix round.` };
+    }
+    const round = ledger.rounds.length + 1;
+    ledger.rounds.push({
+        round,
+        headSha,
+        trigger: plan.trigger,
+        items: plan.items,
+        at: now.toISOString(),
+    });
+    writeFeedbackLedger(config.kilnDir, ledger);
+    let specText;
+    try {
+        specText = feedbackSpec(specHandedToYak(config, world, job), plan.items);
+    }
+    catch (error) {
+        const message = errorMessage(error);
+        writeLaunchError(config.kilnDir, job.id, message);
+        const logPath = join(config.kilnDir, "logs", `${jobBasename(job.id)}.log`);
+        mkdirSync(dirname(logPath), { recursive: true });
+        writeFileSync(logPath, `${message}\n`);
+        writeBreadcrumb(breadcrumbPath(config.kilnDir, job.id), { jobId: job.id, pid: 0, logPath });
+        return { jobId: job.id, repo: job.repo, kind: "fix", output: message };
+    }
+    const report = launchFixRun(config, job, host, repo.path, headSha, specText);
+    return { ...report, output: `round ${round}\n${report.output}` };
+}
+/** Relaunch the open fix round. Does not append a round. Failures become a dead launch so retry can see them. */
+function launchOpenFix(config, world, job, host, checkout, open) {
+    let specText;
+    let headSha;
+    try {
+        clearLaunchError(config.kilnDir, job.id);
+        specText = feedbackSpec(specHandedToYak(config, world, job), open.items);
+        headSha = resolveCommit(host, checkout, open.headSha);
+    }
+    catch (error) {
+        const message = errorMessage(error);
+        writeLaunchError(config.kilnDir, job.id, message);
+        const logPath = join(config.kilnDir, "logs", `${jobBasename(job.id)}.log`);
+        mkdirSync(dirname(logPath), { recursive: true });
+        writeFileSync(logPath, `${message}\n`);
+        writeBreadcrumb(breadcrumbPath(config.kilnDir, job.id), { jobId: job.id, pid: 0, logPath });
+        return { jobId: job.id, repo: job.repo, kind: "fix", output: message };
+    }
+    return launchFixRun(config, job, host, checkout, headSha, specText);
+}
+function resolveCommit(host, checkout, sha) {
+    try {
+        return host.git.revParse(checkout, sha);
+    }
+    catch {
+        host.git.fetch(checkout);
+        return host.git.revParse(checkout, sha);
+    }
+}
+/** `workflows/fix.yaml` on `headSha`. Commits on `yak/<run-id>`. Does not push. */
+function launchFixRun(config, job, host, checkout, headSha, specText) {
+    const sourceWorkflow = readFileSync(join(packageRoot(), "workflows", "fix.yaml"), "utf8");
+    const workflow = materializeWorkflow(config.kilnDir, job.id, "fix");
+    const repo = config.repos[job.repo];
+    const logPath = join(config.kilnDir, "logs", `${jobBasename(job.id)}.log`);
+    mkdirSync(dirname(logPath), { recursive: true });
+    ensureCheckoutExcludes(checkout);
+    const input = workflowInputs(sourceWorkflow, {
+        title: job.title,
+        specText,
+        runNonce: randomUUID(),
+        baseRef: headSha,
+    });
+    const args = agentArgs(config, job, workflow, headSha, input);
+    const env = {
+        ...process.env,
+        ...(repo?.test ? { KILN_TEST: repo.test } : {}),
+        KILN_TASK_TITLE: job.title,
+        KILN_VERIFY_WRAPPER: join(packageRoot(), "dist", "truncate-output.js"),
+    };
+    return spawnLogged(config, host, {
+        jobId: job.id,
+        repo: job.repo,
+        kind: "fix",
+        cwd: checkout,
+        args,
+        env,
+        logPath,
+    });
+}
+function agentArgs(config, job, workflow, headSha, input) {
+    const args = [
+        yakCliPath(),
+        "run",
+        workflow,
+        "--adapter",
+        config.adapter,
+        "--isolation",
+        "worktree",
+        "--from",
+        headSha,
+        "--tag",
+        jobTag(job.id),
+    ];
+    for (const [key, value] of Object.entries(input)) {
+        args.push("--input", `${key}=${value}`);
+    }
+    return args;
+}
+function spawnLogged(config, host, spec) {
+    const crumbPath = breadcrumbPath(config.kilnDir, spec.jobId);
+    writeBreadcrumb(crumbPath, { jobId: spec.jobId, pid: 0, logPath: spec.logPath });
+    const logFd = openSync(spec.logPath, "a");
+    try {
+        const started = spawnAgent(host, { cwd: spec.cwd, args: spec.args, env: spec.env, logFd });
+        const pid = started.pid ?? 0;
+        writeBreadcrumb(crumbPath, { jobId: spec.jobId, pid, logPath: spec.logPath });
+        return {
+            jobId: spec.jobId,
+            repo: spec.repo,
+            ...(spec.kind ? { kind: spec.kind } : {}),
+            output: `pid ${pid}\nlog ${spec.logPath}`,
+        };
+    }
+    catch (error) {
+        writeFileSync(spec.logPath, `${errorMessage(error)}\n`);
+        return {
+            jobId: spec.jobId,
+            repo: spec.repo,
+            ...(spec.kind ? { kind: spec.kind } : {}),
+            output: `pid 0\nlog ${spec.logPath}`,
+        };
+    }
+    finally {
+        closeSync(logFd);
+    }
+}
+function spawnAgent(host, spec) {
+    if (host.startAgent)
+        return host.startAgent(spec);
+    const child = spawn(process.execPath, spec.args, {
+        cwd: spec.cwd,
+        detached: true,
+        stdio: ["ignore", spec.logFd, spec.logFd],
+        env: spec.env,
+    });
+    child.unref();
+    return { pid: child.pid ?? 0 };
+}
 function launchJob(config, world, job, host) {
     const repo = config.repos[job.repo];
     if (!repo) {
         throw new Error(`Repo ${job.repo} is not registered. Run kiln repo add ${job.repo} <path>.`);
     }
+    const feedback = readFeedbackLedger(config.kilnDir, job.id);
+    const open = feedback && feedback !== "corrupt" ? openFeedbackRound(feedback) : undefined;
+    if (open)
+        return launchOpenFix(config, world, job, host, repo.path, open);
     const sourceWorkflow = readFileSync(join(packageRoot(), "workflows", `${job.workflow}.yaml`), "utf8");
     const testCommand = repo.test;
     if (workflowNeedsTestCommand(sourceWorkflow) && !testCommand?.trim()) {
@@ -309,7 +800,6 @@ function launchJob(config, world, job, host) {
     const workflow = materializeWorkflow(config.kilnDir, job.id, job.workflow);
     const logPath = join(config.kilnDir, "logs", `${jobBasename(job.id)}.log`);
     mkdirSync(dirname(logPath), { recursive: true });
-    const crumbPath = breadcrumbPath(config.kilnDir, job.id);
     ensureCheckoutExcludes(repo.path);
     let baseSha;
     let specText;
@@ -338,49 +828,14 @@ function launchJob(config, world, job, host) {
         runNonce: randomUUID(),
         baseRef: baseSha,
     });
-    const args = [
-        yakCliPath(),
-        "run",
-        workflow,
-        "--adapter",
-        config.adapter,
-        "--isolation",
-        "worktree",
-        "--from",
-        baseSha,
-        "--tag",
-        jobTag(job.id),
-    ];
-    for (const [key, value] of Object.entries(input)) {
-        args.push("--input", `${key}=${value}`);
-    }
+    const args = agentArgs(config, job, workflow, baseSha, input);
     const env = {
         ...process.env,
         KILN_TEST: testCommand,
         KILN_TASK_TITLE: job.title,
         KILN_VERIFY_WRAPPER: join(packageRoot(), "dist", "truncate-output.js"),
     };
-    writeBreadcrumb(crumbPath, { jobId: job.id, pid: 0, logPath });
-    const logFd = openSync(logPath, "a");
-    try {
-        const child = spawn(process.execPath, args, {
-            cwd: repo.path,
-            detached: true,
-            stdio: ["ignore", logFd, logFd],
-            env,
-        });
-        child.unref();
-        const pid = child.pid ?? 0;
-        writeBreadcrumb(crumbPath, { jobId: job.id, pid, logPath });
-        return { jobId: job.id, repo: job.repo, output: `pid ${pid}\nlog ${logPath}` };
-    }
-    catch (error) {
-        writeFileSync(logPath, `${errorMessage(error)}\n`);
-        return { jobId: job.id, repo: job.repo, output: `pid 0\nlog ${logPath}` };
-    }
-    finally {
-        closeSync(logFd);
-    }
+    return spawnLogged(config, host, { jobId: job.id, repo: job.repo, cwd: repo.path, args, env, logPath });
 }
 function loadJobs(config) {
     if (!existsSync(config.jobsDir))
@@ -429,6 +884,12 @@ function scanRuns(config) {
             const status = classify(events);
             const gateDecision = readApproveDecision(runDir);
             const title = readRunTitle(runDir);
+            const finished = [...events].reverse().find((event) => event.t === "run.finished");
+            const stepPlan = readRunStepPlan(runDir);
+            const stepEvents = stepEventsFrom(events);
+            const pid = livePid(events);
+            const activityMs = eventActivityMs(events) ?? fileMtimeMs(join(runDir, "journal.jsonl"));
+            const finishedReason = finished && typeof finished.reason === "string" ? finished.reason : undefined;
             const run = {
                 runId,
                 checkout: repo.path,
@@ -436,14 +897,211 @@ function scanRuns(config) {
                 gates: status === "suspended" ? openGates(runDir, events) : [],
                 ...(gateDecision ? { gateDecision } : {}),
                 ...(title ? { title } : {}),
+                ...(typeof started?.at === "string" ? { startedAt: started.at } : {}),
+                ...(typeof finished?.at === "string" ? { finishedAt: finished.at } : {}),
+                ...(stepPlan.length > 0 ? { stepPlan } : {}),
+                ...(stepEvents.length > 0 ? { stepEvents } : {}),
+                ...(pid !== undefined ? { pid } : {}),
+                ...(activityMs !== undefined ? { activityMs } : {}),
+                ...(finishedReason ? { finishedReason } : {}),
             };
-            const previous = byTag.get(started.tag);
-            if (!previous || run.runId > previous.runId)
-                byTag.set(started.tag, run);
+            const list = byTag.get(started.tag) ?? [];
+            list.push(run);
+            byTag.set(started.tag, list);
         }
     }
     return byTag;
 }
+function stepEventsFrom(events) {
+    const found = [];
+    for (const event of events) {
+        if (event.t !== "step.started" && event.t !== "step.completed" && event.t !== "step.failed")
+            continue;
+        if (!event.stepId)
+            continue;
+        const detail = event.t === "step.failed" ? formatFailure(event.failure) : undefined;
+        found.push({
+            t: event.t,
+            stepId: event.stepId,
+            ...(detail ? { detail } : {}),
+        });
+    }
+    return found;
+}
+function formatFailure(failure) {
+    if (!failure)
+        return undefined;
+    const detail = typeof failure.detail === "string" ? failure.detail.trim() : "";
+    const reason = typeof failure.reason === "string" ? failure.reason.trim() : "";
+    const text = detail && reason ? `${detail} (${reason})` : detail || reason;
+    return text || undefined;
+}
+function readRunStepPlan(runDir) {
+    const file = join(runDir, "workflow.json");
+    if (!existsSync(file))
+        return [];
+    try {
+        const parsed = JSON.parse(readFileSync(file, "utf8"));
+        return leafStepIds(parsed.steps);
+    }
+    catch {
+        return [];
+    }
+}
+function readDiffSummary(checkout, runId) {
+    const worktree = join(checkout, ".yak", "worktrees", runId);
+    if (existsSync(worktree)) {
+        const result = spawnSync("git", ["diff", "--stat", "HEAD"], {
+            cwd: worktree,
+            encoding: "utf8",
+            timeout: 5_000,
+        });
+        const text = result.status === 0 ? result.stdout.trim() : "";
+        if (text)
+            return clipText(text, 1500);
+    }
+    return readBuildSummary(join(checkout, ".runs", runId));
+}
+function readBuildSummary(runDir) {
+    const dir = join(runDir, "artifacts");
+    if (!existsSync(dir))
+        return undefined;
+    let files;
+    try {
+        files = readdirSync(dir);
+    }
+    catch {
+        return undefined;
+    }
+    let best;
+    for (const file of files) {
+        const match = /^build(?:\.(\d+))?\.json$/.exec(file);
+        if (!match)
+            continue;
+        const iter = match[1] ? Number(match[1]) : 0;
+        if (!best || iter >= best.iter)
+            best = { iter, file };
+    }
+    if (!best)
+        return undefined;
+    try {
+        const parsed = JSON.parse(readFileSync(join(dir, best.file), "utf8"));
+        if (typeof parsed.summary !== "string" || !parsed.summary.trim())
+            return undefined;
+        return clipText(parsed.summary.trim(), 280);
+    }
+    catch {
+        return undefined;
+    }
+}
+function readLogTail(logPath) {
+    if (!logPath)
+        return undefined;
+    let size;
+    try {
+        size = statSync(logPath).size;
+    }
+    catch {
+        return undefined;
+    }
+    if (size <= 0)
+        return undefined;
+    const take = Math.min(size, 64_000);
+    let fd;
+    try {
+        fd = openSync(logPath, "r");
+        const buffer = Buffer.alloc(take);
+        readSync(fd, buffer, 0, take, size - take);
+        const text = buffer.toString("utf8");
+        const lines = text.split(/\r?\n/);
+        if (size > take)
+            lines.shift();
+        while (lines.length > 0 && lines[lines.length - 1] === "")
+            lines.pop();
+        const tail = lines.slice(-40).join("\n").trim();
+        return tail || undefined;
+    }
+    catch {
+        return undefined;
+    }
+    finally {
+        if (fd !== undefined)
+            closeSync(fd);
+    }
+}
+/**
+ * Prefer a run that is not already a recorded failure, so a retry is visible
+ * even when its id does not sort after the failed one. Otherwise keep the
+ * latest id, which is what a single-run job has always shown.
+ */
+function selectJobRun(runs, ledger, feedback, baseRunId) {
+    const settled = new Set();
+    for (const round of feedback?.rounds ?? []) {
+        if ((round.outcome === "pushed" || round.outcome === "fixed") && round.fixRunId)
+            settled.add(round.fixRunId);
+    }
+    const ignored = new Set(ledger?.ignoredRunIds ?? []);
+    const failed = new Set((ledger?.attempts ?? []).flatMap((attempt) => (attempt.runId ? [attempt.runId] : [])));
+    const open = openFeedbackRound(feedback);
+    if (open) {
+        const bound = open.fixRunId && !ignored.has(open.fixRunId) && !failed.has(open.fixRunId) ? open.fixRunId : undefined;
+        const fixRuns = runs.filter((run) => {
+            if (settled.has(run.runId) || ignored.has(run.runId))
+                return false;
+            if (bound)
+                return run.runId === bound;
+            if (failed.has(run.runId))
+                return false;
+            return baseRunId ? run.runId !== baseRunId : true;
+        });
+        fixRuns.sort((left, right) => (left.runId < right.runId ? 1 : left.runId > right.runId ? -1 : 0));
+        const fix = fixRuns[0];
+        if (fix) {
+            const retired = failed.has(fix.runId);
+            return {
+                run: fix,
+                retired,
+                fixInProgress: true,
+                fixAwaitingPublish: fix.status === "ok" && !retired,
+            };
+        }
+        return { retired: false, fixInProgress: true, fixAwaitingPublish: false };
+    }
+    const picked = selectRun(runs.filter((run) => !settled.has(run.runId)), ledger);
+    return { ...picked, fixInProgress: false, fixAwaitingPublish: false };
+}
+function selectRun(runs, ledger) {
+    const ignored = new Set(ledger?.ignoredRunIds ?? []);
+    const failed = new Set((ledger?.attempts ?? []).flatMap((attempt) => (attempt.runId ? [attempt.runId] : [])));
+    const visible = runs.filter((run) => !ignored.has(run.runId));
+    visible.sort((left, right) => (left.runId < right.runId ? 1 : left.runId > right.runId ? -1 : 0));
+    const fresh = visible.filter((run) => !failed.has(run.runId));
+    if (fresh[0])
+        return { run: fresh[0], retired: false };
+    if (visible[0])
+        return { run: visible[0], retired: true };
+    return { retired: false };
+}
+function later(left, right) {
+    if (left === undefined)
+        return right;
+    if (right === undefined)
+        return left;
+    return Math.max(left, right);
+}
+function eventActivityMs(events) {
+    let latest;
+    for (const event of events) {
+        if (!event.at)
+            continue;
+        const parsed = Date.parse(event.at);
+        if (!Number.isFinite(parsed))
+            continue;
+        if (latest === undefined || parsed > latest)
+            latest = parsed;
+    }
+    return latest;
+}
 function listRunDirs(checkout) {
     const runsDir = join(checkout, ".runs");
     if (!existsSync(runsDir))
@@ -807,15 +1465,7 @@ function releaseLock(lockPath) {
     }
 }
 function pidAlive(pid) {
-    if (!Number.isInteger(pid) || pid <= 0)
-        return false;
-    try {
-        process.kill(pid, 0);
-        return true;
-    }
-    catch (error) {
-        return isErrno(error, "EPERM");
-    }
+    return isPidAlive(pid);
 }
 function isErrno(error, code) {
     return typeof error === "object" && error !== null && "code" in error && error.code === code;