@lchase/kiln 0.3.0 → 0.4.0

package/dist/driver.js

@@ -1,16 +1,18 @@
 import { spawn, spawnSync } from "node:child_process";
 import { randomUUID } from "node:crypto";
-import { closeSync, existsSync, linkSync, mkdirSync, openSync, readdirSync, readFileSync, rmSync, writeFileSync, } from "node:fs";
+import { closeSync, existsSync, linkSync, mkdirSync, openSync, readdirSync, readFileSync, readSync, rmSync, statSync, writeFileSync, } from "node:fs";
 import { basename, dirname, isAbsolute, join } from "node:path";
+import { buildAttention } from "./attention.js";
 import { clearLaunchError, deliverJob, readDelivery, readLaunchError, resolveLaunchBase, writeDelivery, writeLaunchError, isMerged, } from "./delivery.js";
 import { appendInvestigationFindings, existingFindingsPath, findingsFilePath, INVESTIGATION_WORKFLOW, readFindingsMarkdown, writeFindings, } from "./findings.js";
 import { defaultHost } from "./host.js";
 import { readJobSpec, resolveJobSpec } from "./jobs.js";
 import { jobBasename, packageRoot } from "./paths.js";
 import { decide, isLanded, jobState, } from "./reconcile.js";
+import { fileMtimeMs, isPidAlive, readAttemptLedger, recoverRuns, resetAttemptLedger, } from "./reliability.js";
 import { gateAnswerCommand, renderStatus, renderTick, statusView, yakCliPath, } from "./report.js";
 import { JobSchema } from "./schemas.js";
-import { materializeWorkflow, workflowInputs, workflowNeedsTestCommand } from "./workflow.js";
+import { leafStepIds, materializeWorkflow, workflowInputs, workflowNeedsTestCommand, workflowStepPlan } from "./workflow.js";
 const TAG_PREFIX = "kiln:";
 export function jobTag(jobId) {
     return `${TAG_PREFIX}${jobId}`;
@@ -25,16 +27,26 @@ export function parseJobTag(tag) {
 }
 /** The derived view `kiln status` prints and `kiln serve` renders. */
 export function readStatus(config, jobId, host = defaultHost()) {
-    return statusView(observe(config, host).observation, jobId);
+    const view = statusView(observe(config, host).observation, jobId);
+    return { ...view, attention: buildAttention(view.jobs) };
 }
 export function runStatus(config, jobId, host = defaultHost()) {
     return renderStatus(readStatus(config, jobId, host));
 }
-export function runTick(config, host = defaultHost()) {
+export function runTick(config, host = defaultHost(), options = {}) {
+    const startup = options.startup ?? true;
+    const now = options.now ?? new Date();
     return withTickLock(config.kilnDir, () => {
+        const notes = recoverRuns(config, observe(config, host).observation.jobs, { startup, now });
         const world = observe(config, host);
-        const actions = decide(world.observation);
-        return apply(config, world, actions, host);
+        const actions = decide(world.observation, now);
+        return apply(config, world, actions, host, notes);
+    });
+}
+/** Startup pass shared by `kiln tick` (via `runTick`) and `kiln serve`. Idempotent. */
+export function startupRecovery(config, host = defaultHost(), now = new Date()) {
+    withTickLock(config.kilnDir, () => {
+        recoverRuns(config, observe(config, host).observation.jobs, { startup: true, now });
     });
 }
 export function answerJob(config, jobId, choice, host = defaultHost()) {
@@ -65,6 +77,74 @@ export function answerJob(config, jobId, choice, host = defaultHost()) {
     }
     return output.endsWith("\n") || output.length === 0 ? output : `${output}\n`;
 }
+/**
+ * Run a failed or dead job again. The board Retry button and `kiln retry`
+ * both call this.
+ * A landed job whose only failure is opening the pull request is handed to
+ * the same deliver path tick uses. Everything else is a new `yak run`, the
+ * same launch tick uses. The attempt ledger is cleared either way, and the
+ * current run id is ignored so a dead job is not stuck on the old journal.
+ * A dead journal is cancelled first.
+ */
+export function retryJob(config, jobId, host = defaultHost()) {
+    return withTickLock(config.kilnDir, () => {
+        const world = observe(config, host);
+        const observed = world.observation.jobs.find((candidate) => candidate.jobId === jobId);
+        if (!observed)
+            throw new Error(`No job ${jobId}.`);
+        if (observed.error)
+            throw new Error(`Job ${jobId} is invalid: ${observed.error}`);
+        const state = jobState(observed, world.observation);
+        if (state !== "failed" && state !== "dead") {
+            throw new Error(`Job ${jobId} is ${state}, not failed or dead.`);
+        }
+        const job = world.jobs.get(jobId);
+        if (!job)
+            throw new Error(`No job ${jobId}.`);
+        const crumb = readBreadcrumbs(config.kilnDir).find((candidate) => candidate.jobId === jobId);
+        if (crumb && pidAlive(crumb.pid)) {
+            throw new Error(`Job ${jobId} is already launching.`);
+        }
+        const cleared = observed.attemptCount ?? 0;
+        resetAttemptLedger(config.kilnDir, jobId, observed.run?.runId);
+        rmSync(breadcrumbPath(config.kilnDir, jobId), { force: true });
+        clearLaunchError(config.kilnDir, jobId);
+        const noun = cleared === 1 ? "attempt" : "attempts";
+        const clearedLine = `cleared ${cleared} ${noun}\n`;
+        if (shouldRedeliver(observed)) {
+            const report = deliverObserved(config, world, jobId, host);
+            if (!report.ok)
+                throw new Error(report.detail);
+            const detail = report.detail.endsWith("\n") || report.detail.length === 0 ? report.detail : `${report.detail}\n`;
+            return `retry ${jobId}  ${job.repo}\n${clearedLine}${detail}`;
+        }
+        if (state === "dead" && observed.run) {
+            try {
+                spawnSync(process.execPath, [yakCliPath(), "cancel", observed.run.runId], {
+                    cwd: observed.run.checkout,
+                    encoding: "utf8",
+                    timeout: 20_000,
+                });
+            }
+            catch {
+                // Cancel is the same cleanup status prints. The new run still supersedes the journal.
+            }
+        }
+        const logPath = join(config.kilnDir, "logs", `${jobBasename(job.id)}.log`);
+        if (existsSync(logPath))
+            writeFileSync(logPath, "");
+        const report = launchJob(config, world, job, host);
+        const output = report.output.endsWith("\n") || report.output.length === 0 ? report.output : `${report.output}\n`;
+        return `retry ${report.jobId}  ${report.repo}\n${clearedLine}${output}`;
+    });
+}
+function shouldRedeliver(job) {
+    if (!isLanded(job))
+        return false;
+    if (job.delivery?.prState === "CLOSED")
+        return false;
+    return Boolean(job.delivery?.error) && !job.delivery?.prNumber;
+}
 function gateForChoice(gates, choice) {
     return (gates.find((gate) => gate.choices?.includes(choice)) ??
         gates.find((gate) => !gate.choices?.length));
@@ -86,11 +166,16 @@ function observe(config, host) {
         if (entry.job)
             records.set(entry.job.id, entry.job);
         const tag = jobTag(entry.meta.jobId);
-        const run = runs.get(tag);
+        const ledgerResult = readAttemptLedger(config.kilnDir, entry.meta.jobId);
+        const ledger = ledgerResult && ledgerResult !== "corrupt" ? ledgerResult : undefined;
+        const picked = selectRun(runs.get(tag) ?? [], ledger);
+        const run = picked.run;
         const crumb = breadcrumbs.find((candidate) => candidate.jobId === entry.meta.jobId);
-        if (crumb && run)
+        if (crumb && run && !picked.retired)
             breadcrumbsToClear.push({ path: crumb.path });
         const childAlive = crumb ? pidAlive(crumb.pid) : false;
+        const launching = Boolean(crumb) && childAlive && (!run || picked.retired);
+        const launchFailed = Boolean(crumb) && !childAlive && (!run || picked.retired);
         const repoKnown = entry.job ? Boolean(config.repos[entry.job.repo]) : false;
         const repoError = entry.job && !repoKnown ? `Repo ${entry.job.repo} is not registered.` : undefined;
         const repo = entry.job ? config.repos[entry.job.repo] : undefined;
@@ -103,10 +188,26 @@ function observe(config, host) {
             ? "stack needs exactly one dependency."
             : undefined;
         const logPath = crumb?.logPath || existingLog(config.kilnDir, entry.meta.jobId);
+        if (run && logPath)
+            run.activityMs = later(run.activityMs, fileMtimeMs(logPath));
+        const launchActivityMs = later(crumb ? fileMtimeMs(crumb.path) : undefined, logPath ? fileMtimeMs(logPath) : undefined);
+        const attemptCount = ledger?.attempts.length ?? 0;
+        const lastFailure = ledger?.attempts.at(-1);
+        const exhausted = attemptCount >= config.maxAttempts &&
+            attemptCount > 0 &&
+            !launching &&
+            (picked.retired || !run || run.status === "dead" || run.status === "failed");
         const specPath = entry.job ? resolveJobSpec(config.kilnDir, entry.job.spec) : undefined;
         const findingsPath = entry.job?.workflow === INVESTIGATION_WORKFLOW
             ? existingFindingsPath(config.kilnDir, entry.job.id)
             : undefined;
+        const logTail = readLogTail(logPath);
+        const stepPlan = entry.job ? workflowStepPlan(entry.job.workflow) : undefined;
+        const diffSummary = run &&
+            run.status === "suspended" &&
+            entry.job?.workflow !== INVESTIGATION_WORKFLOW
+            ? readDiffSummary(run.checkout, run.runId)
+            : undefined;
         jobs.push({
             jobId: entry.meta.jobId,
             repo: entry.job?.repo ?? "",
@@ -119,11 +220,23 @@ function observe(config, host) {
             ...(entry.job?.stack ? { stack: true } : {}),
             ...(delivery ? { delivery } : {}),
             ...(findingsPath ? { findingsPath } : {}),
-            launching: Boolean(crumb) && !run && childAlive,
-            launchFailed: Boolean(crumb) && !run && !childAlive,
+            launching,
+            launchFailed,
             ...(launchError ? { launchError } : {}),
+            ...(crumb ? { launchPid: crumb.pid } : {}),
+            ...(launchActivityMs !== undefined ? { launchActivityMs } : {}),
             ...(logPath ? { logPath } : {}),
+            ...(entry.job?.createdAt ? { createdAt: entry.job.createdAt } : {}),
+            ...(logTail ? { logTail } : {}),
+            ...(diffSummary ? { diffSummary } : {}),
+            ...(stepPlan && stepPlan.length > 0 ? { stepPlan } : {}),
             ...(run ? { run } : {}),
+            attemptCount,
+            ...(lastFailure ? { lastFailureReason: lastFailure.reason } : {}),
+            ...(ledger?.nextRetryAt && !exhausted ? { nextRetryAt: ledger.nextRetryAt } : {}),
+            ...(exhausted ? { retriesExhausted: true } : {}),
+            ...(picked.retired ? { runRetired: true } : {}),
+            ...(ledgerResult === "corrupt" ? { ledgerError: "attempt ledger is unreadable" } : {}),
             ...(entry.meta.error || repoError || stackError
                 ? { error: entry.meta.error ?? repoError ?? stackError }
                 : {}),
@@ -132,13 +245,14 @@ function observe(config, host) {
     return {
         observation: {
             maxConcurrent: config.maxConcurrent,
+            maxAttempts: config.maxAttempts,
             jobs,
             breadcrumbsToClear,
         },
         jobs: records,
     };
 }
-function apply(config, world, actions, host) {
+function apply(config, world, actions, host, notes = []) {
     const launches = [];
     const delivers = [];
     const findings = [];
@@ -161,9 +275,10 @@ function apply(config, world, actions, host) {
         if (!job) {
             throw new Error(`Tick planned a launch for ${action.jobId}, which is not in the observed jobs.`);
         }
-        launches.push(launchJob(config, world, job, host));
+        const report = launchJob(config, world, job, host);
+        launches.push(action.type === "retry" ? { ...report, retry: true } : report);
     }
-    return renderTick(world.observation, actions, launches, delivers, findings);
+    return renderTick(world.observation, actions, launches, delivers, findings, notes);
 }
 /**
  * Copy an approved investigation's markdown into `<kilnDir>/findings`.
@@ -429,6 +544,12 @@ function scanRuns(config) {
             const status = classify(events);
             const gateDecision = readApproveDecision(runDir);
             const title = readRunTitle(runDir);
+            const finished = [...events].reverse().find((event) => event.t === "run.finished");
+            const stepPlan = readRunStepPlan(runDir);
+            const stepEvents = stepEventsFrom(events);
+            const pid = livePid(events);
+            const activityMs = eventActivityMs(events) ?? fileMtimeMs(join(runDir, "journal.jsonl"));
+            const finishedReason = finished && typeof finished.reason === "string" ? finished.reason : undefined;
             const run = {
                 runId,
                 checkout: repo.path,
@@ -436,14 +557,175 @@ function scanRuns(config) {
                 gates: status === "suspended" ? openGates(runDir, events) : [],
                 ...(gateDecision ? { gateDecision } : {}),
                 ...(title ? { title } : {}),
+                ...(typeof started?.at === "string" ? { startedAt: started.at } : {}),
+                ...(typeof finished?.at === "string" ? { finishedAt: finished.at } : {}),
+                ...(stepPlan.length > 0 ? { stepPlan } : {}),
+                ...(stepEvents.length > 0 ? { stepEvents } : {}),
+                ...(pid !== undefined ? { pid } : {}),
+                ...(activityMs !== undefined ? { activityMs } : {}),
+                ...(finishedReason ? { finishedReason } : {}),
             };
-            const previous = byTag.get(started.tag);
-            if (!previous || run.runId > previous.runId)
-                byTag.set(started.tag, run);
+            const list = byTag.get(started.tag) ?? [];
+            list.push(run);
+            byTag.set(started.tag, list);
         }
     }
     return byTag;
 }
+function stepEventsFrom(events) {
+    const found = [];
+    for (const event of events) {
+        if (event.t !== "step.started" && event.t !== "step.completed" && event.t !== "step.failed")
+            continue;
+        if (!event.stepId)
+            continue;
+        const detail = event.t === "step.failed" ? formatFailure(event.failure) : undefined;
+        found.push({
+            t: event.t,
+            stepId: event.stepId,
+            ...(detail ? { detail } : {}),
+        });
+    }
+    return found;
+}
+function formatFailure(failure) {
+    if (!failure)
+        return undefined;
+    const detail = typeof failure.detail === "string" ? failure.detail.trim() : "";
+    const reason = typeof failure.reason === "string" ? failure.reason.trim() : "";
+    const text = detail && reason ? `${detail} (${reason})` : detail || reason;
+    return text || undefined;
+}
+function readRunStepPlan(runDir) {
+    const file = join(runDir, "workflow.json");
+    if (!existsSync(file))
+        return [];
+    try {
+        const parsed = JSON.parse(readFileSync(file, "utf8"));
+        return leafStepIds(parsed.steps);
+    }
+    catch {
+        return [];
+    }
+}
+function readDiffSummary(checkout, runId) {
+    const worktree = join(checkout, ".yak", "worktrees", runId);
+    if (existsSync(worktree)) {
+        const result = spawnSync("git", ["diff", "--stat", "HEAD"], {
+            cwd: worktree,
+            encoding: "utf8",
+            timeout: 5_000,
+        });
+        const text = result.status === 0 ? result.stdout.trim() : "";
+        if (text)
+            return clipText(text, 1500);
+    }
+    return readBuildSummary(join(checkout, ".runs", runId));
+}
+function readBuildSummary(runDir) {
+    const dir = join(runDir, "artifacts");
+    if (!existsSync(dir))
+        return undefined;
+    let files;
+    try {
+        files = readdirSync(dir);
+    }
+    catch {
+        return undefined;
+    }
+    let best;
+    for (const file of files) {
+        const match = /^build(?:\.(\d+))?\.json$/.exec(file);
+        if (!match)
+            continue;
+        const iter = match[1] ? Number(match[1]) : 0;
+        if (!best || iter >= best.iter)
+            best = { iter, file };
+    }
+    if (!best)
+        return undefined;
+    try {
+        const parsed = JSON.parse(readFileSync(join(dir, best.file), "utf8"));
+        if (typeof parsed.summary !== "string" || !parsed.summary.trim())
+            return undefined;
+        return clipText(parsed.summary.trim(), 280);
+    }
+    catch {
+        return undefined;
+    }
+}
+function readLogTail(logPath) {
+    if (!logPath)
+        return undefined;
+    let size;
+    try {
+        size = statSync(logPath).size;
+    }
+    catch {
+        return undefined;
+    }
+    if (size <= 0)
+        return undefined;
+    const take = Math.min(size, 64_000);
+    let fd;
+    try {
+        fd = openSync(logPath, "r");
+        const buffer = Buffer.alloc(take);
+        readSync(fd, buffer, 0, take, size - take);
+        const text = buffer.toString("utf8");
+        const lines = text.split(/\r?\n/);
+        if (size > take)
+            lines.shift();
+        while (lines.length > 0 && lines[lines.length - 1] === "")
+            lines.pop();
+        const tail = lines.slice(-40).join("\n").trim();
+        return tail || undefined;
+    }
+    catch {
+        return undefined;
+    }
+    finally {
+        if (fd !== undefined)
+            closeSync(fd);
+    }
+}
+/**
+ * Prefer a run that is not already a recorded failure, so a retry is visible
+ * even when its id does not sort after the failed one. Otherwise keep the
+ * latest id, which is what a single-run job has always shown.
+ */
+function selectRun(runs, ledger) {
+    const ignored = new Set(ledger?.ignoredRunIds ?? []);
+    const failed = new Set((ledger?.attempts ?? []).flatMap((attempt) => (attempt.runId ? [attempt.runId] : [])));
+    const visible = runs.filter((run) => !ignored.has(run.runId));
+    visible.sort((left, right) => (left.runId < right.runId ? 1 : left.runId > right.runId ? -1 : 0));
+    const fresh = visible.filter((run) => !failed.has(run.runId));
+    if (fresh[0])
+        return { run: fresh[0], retired: false };
+    if (visible[0])
+        return { run: visible[0], retired: true };
+    return { retired: false };
+}
+function later(left, right) {
+    if (left === undefined)
+        return right;
+    if (right === undefined)
+        return left;
+    return Math.max(left, right);
+}
+function eventActivityMs(events) {
+    let latest;
+    for (const event of events) {
+        if (!event.at)
+            continue;
+        const parsed = Date.parse(event.at);
+        if (!Number.isFinite(parsed))
+            continue;
+        if (latest === undefined || parsed > latest)
+            latest = parsed;
+    }
+    return latest;
+}
 function listRunDirs(checkout) {
     const runsDir = join(checkout, ".runs");
     if (!existsSync(runsDir))
@@ -807,15 +1089,7 @@ function releaseLock(lockPath) {
     }
 }
 function pidAlive(pid) {
-    if (!Number.isInteger(pid) || pid <= 0)
-        return false;
-    try {
-        process.kill(pid, 0);
-        return true;
-    }
-    catch (error) {
-        return isErrno(error, "EPERM");
-    }
+    return isPidAlive(pid);
 }
 function isErrno(error, code) {
     return typeof error === "object" && error !== null && "code" in error && error.code === code;

package/dist/reconcile.js

@@ -1,4 +1,6 @@
 import { INVESTIGATION_WORKFLOW } from "./findings.js";
+/** Why a transient attempt failed. Written under `<kilnDir>/attempts`, not on the job. */
+export const FAILURE_REASONS = ["stalled", "orphaned", "crashed", "failed"];
 export function inFlightCount(observation) {
     return observation.jobs.filter((job) => inFlight(job, observation)).length;
 }
@@ -18,33 +20,48 @@ function inFlight(job, observation) {
  * An investigation is `done` on approve and never `in-review`.
  */
 export function jobState(job, observation) {
-    if (job.launchFailed && !job.run)
-        return "failed";
-    if (job.run) {
-        if (isLanded(job))
-            return landedState(job);
-        if (job.run.status === "ok" && job.run.gateDecision === "reject")
-            return "rejected";
-        if (job.run.status === "ok")
-            return "failed";
-        if (job.run.status === "failed")
-            return "failed";
-        if (job.run.status === "dead")
-            return "dead";
-        if (job.run.status === "suspended")
-            return "needs-human";
-        if (job.run.gateDecision === "approve")
-            return "approved";
+    if (job.launching && (!job.run || job.runRetired))
         return "running";
-    }
+    if (job.run && !job.runRetired)
+        return activeRunState(job);
+    if (job.run && isLanded(job))
+        return landedState(job);
+    if (job.run?.status === "ok" && job.run.gateDecision === "reject")
+        return "rejected";
+    if (job.run?.status === "suspended")
+        return "needs-human";
+    if (job.retriesExhausted)
+        return "dead";
+    if (job.runRetired || job.run?.status === "failed" || job.run?.status === "dead")
+        return "failed";
     if (job.launching)
         return "running";
     if (!dependenciesMet(job, observation))
         return "blocked";
-    if (job.launchError)
+    if (job.launchError || job.launchFailed)
         return "failed";
     return "queued";
 }
+/** State of the current run, before the attempt ledger retires it. */
+function activeRunState(job) {
+    if (!job.run)
+        return "queued";
+    if (isLanded(job))
+        return landedState(job);
+    if (job.run.status === "ok" && job.run.gateDecision === "reject")
+        return "rejected";
+    if (job.run.status === "ok")
+        return "failed";
+    if (job.run.status === "failed")
+        return "failed";
+    if (job.run.status === "dead")
+        return "dead";
+    if (job.run.status === "suspended")
+        return "needs-human";
+    if (job.run.gateDecision === "approve")
+        return "approved";
+    return "running";
+}
 /**
  * After-merged (the default) needs every upstream `done`.
  * Stacked needs its one upstream landed (approved and the run finished ok),
@@ -113,7 +130,24 @@ function canLaunch(job, observation) {
         return false;
     return dependenciesMet(job, observation);
 }
-export function decide(observation) {
+/** A recorded transient failure whose backoff has elapsed. One retry per tick, ahead of a new job. */
+function canRetry(job, observation, now) {
+    if (job.error || job.launching || job.retriesExhausted || job.ledgerError)
+        return false;
+    if (job.run && !job.runRetired)
+        return false;
+    if (!job.nextRetryAt)
+        return false;
+    const due = Date.parse(job.nextRetryAt);
+    if (!Number.isFinite(due) || due > now.getTime())
+        return false;
+    if (job.run?.gateDecision === "reject" || isLanded(job))
+        return false;
+    if (job.run?.status === "suspended")
+        return false;
+    return dependenciesMet(job, observation);
+}
+export function decide(observation, now = new Date()) {
     const actions = observation.breadcrumbsToClear.map((crumb) => ({
         type: "clear-breadcrumb",
         path: crumb.path,
@@ -132,6 +166,11 @@ export function decide(observation) {
     }
     if (inFlightCount(observation) >= observation.maxConcurrent)
         return actions;
+    const retry = observation.jobs.find((job) => canRetry(job, observation, now));
+    if (retry) {
+        actions.push({ type: "retry", jobId: retry.jobId, repo: retry.repo });
+        return actions;
+    }
     const ready = observation.jobs.filter((job) => canLaunch(job, observation));
     const healthy = ready.filter((job) => !job.launchError);
     const picked = healthy[0] ?? ready[0];