@lchase/kiln 0.8.0 → 0.10.0

package/dist/driver.js

@@ -4,14 +4,20 @@ import { closeSync, existsSync, linkSync, mkdirSync, openSync, readdirSync, read
 import { basename, dirname, isAbsolute, join } from "node:path";
 import { claudeCredentialPreflight } from "./auth.js";
 import { buildAttention } from "./attention.js";
+import { attemptsFor, fixRoundsFor } from "./config.js";
 import { clearLaunchError, deliverJob, plannedRemoteBranch, readDelivery, readLaunchError, resolveLaunchBase, writeDelivery, writeLaunchError, isMerged, } from "./delivery.js";
 import { feedbackSpec, openFeedbackRound, planFeedback, readFeedbackLedger, writeFeedbackLedger, } from "./feedback.js";
+import { appendEvent, readEvents } from "./events.js";
+import { costFromJournal } from "./failure.js";
+import { ensureFixTrailers } from "./fix-commit.js";
+import { holdDetail, pausedFrom, recordDailyCap, reviseNotesSinceRun, spendSnapshot, stopReasonFromEvents, withReviseNotes, } from "./rails.js";
 import { appendInvestigationFindings, existingFindingsPath, findingsFilePath, INVESTIGATION_WORKFLOW, readFindingsMarkdown, writeFindings, } from "./findings.js";
 import { defaultHost } from "./host.js";
 import { readJobSpec, resolveJobSpec } from "./jobs.js";
 import { jobBasename, packageRoot } from "./paths.js";
 import { decide, isLanded, jobState, } from "./reconcile.js";
 import { fileMtimeMs, isPidAlive, readAttemptLedger, recoverRuns, resetAttemptLedger, } from "./reliability.js";
+import { projectRunJournal, promptsHash } from "./project.js";
 import { gateAnswerCommand, renderStatus, renderTick, statusView, yakCliPath, sandboxFailureDetail, } from "./report.js";
 import { AGENT_SANDBOX_IMAGE, injectAgentSandbox, jobIsSandboxed, probeSandboxEnvironment, PROXY_SANDBOX_IMAGE, requiredSandboxImages, resolveSandbox, sandboxIsEnabled, sandboxPreflight, sandboxPreflightFor, } from "./sandbox.js";
 import { JobSchema } from "./schemas.js";
@@ -40,16 +46,19 @@ export function runTick(config, host = defaultHost(), options = {}) {
     const startup = options.startup ?? true;
     const now = options.now ?? new Date();
     return withTickLock(config.kilnDir, () => {
-        const notes = recoverRuns(config, observe(config, host).observation.jobs, { startup, now });
+        const notes = recoverRuns(config, observe(config, host, now).observation.jobs, { startup, now });
         const facts = sandboxIsEnabled(config)
             ? (options.sandboxFacts ?? probeSandboxEnvironment(process.env, requiredSandboxImages(config)))
             : undefined;
-        const midway = observe(config, host);
+        const midway = observe(config, host, now);
         bindOpenFixRuns(config, midway);
         const published = publishReadyFixes(config, host, midway);
-        const world = observe(config, host);
+        const world = observe(config, host, now);
         const feedbackNotes = watchPullRequests(config, world, host);
         const actions = decide(world.observation, now);
+        if (world.observation.spendBlocked) {
+            recordDailyCap(config.kilnDir, spendSnapshot(readEvents(config.kilnDir, { warn: () => { } }), config.dailyDollarCap, now), now);
+        }
         const body = apply(config, world, actions, host, notes, now, [...published, ...feedbackNotes], facts);
         const heads = [
             claudeCredentialPreflight(process.env, config.adapter),
@@ -61,7 +70,7 @@ export function runTick(config, host = defaultHost(), options = {}) {
 /** Startup pass shared by `kiln tick` (via `runTick`) and `kiln serve`. Idempotent. */
 export function startupRecovery(config, host = defaultHost(), now = new Date()) {
     withTickLock(config.kilnDir, () => {
-        recoverRuns(config, observe(config, host).observation.jobs, { startup: true, now });
+        recoverRuns(config, observe(config, host, now).observation.jobs, { startup: true, now });
     });
 }
 export function answerJob(config, jobId, choice, host = defaultHost()) {
@@ -87,6 +96,20 @@ export function answerJob(config, jobId, choice, host = defaultHost()) {
     if (result.status !== 0) {
         throw new Error(output.trim() || `Could not ${choice} job ${jobId}.`);
     }
+    const noted = gateEdit(job.run.checkout, job.run.runId, gate.stepId);
+    appendEvent(config.kilnDir, {
+        job_id: jobId,
+        run_id: job.run.runId,
+        event: "gate.resolved",
+        step: gate.stepId,
+        properties: {
+            decision: choice,
+            by: "human",
+            repo: job.repo,
+            ...noted,
+            once: `gate.resolved:${gate.stepId}`,
+        },
+    });
     if (choice === "approve") {
         recordInvestigationFindings(config, observe(config, host), jobId);
     }
@@ -110,8 +133,8 @@ export function retryJob(config, jobId, host = defaultHost()) {
         if (observed.error)
             throw new Error(`Job ${jobId} is invalid: ${observed.error}`);
         const state = jobState(observed, world.observation);
-        if (state !== "failed" && state !== "dead") {
-            throw new Error(`Job ${jobId} is ${state}, not failed or dead.`);
+        if (!canManualRetry(state, observed)) {
+            throw new Error(`Job ${jobId} is ${state}, not failed, dead, or stopped for a person.`);
         }
         const job = world.jobs.get(jobId);
         if (!job)
@@ -120,6 +143,11 @@ export function retryJob(config, jobId, host = defaultHost()) {
         if (crumb && pidAlive(crumb.pid)) {
             throw new Error(`Job ${jobId} is already launching.`);
         }
+        appendEvent(config.kilnDir, {
+            job_id: jobId,
+            event: "job.retried",
+            properties: { by: "human", repo: job.repo },
+        });
         const cleared = observed.attemptCount ?? 0;
         resetAttemptLedger(config.kilnDir, jobId, observed.run?.runId);
         rmSync(breadcrumbPath(config.kilnDir, jobId), { force: true });
@@ -133,7 +161,7 @@ export function retryJob(config, jobId, host = defaultHost()) {
             const detail = report.detail.endsWith("\n") || report.detail.length === 0 ? report.detail : `${report.detail}\n`;
             return `retry ${jobId}  ${job.repo}\n${clearedLine}${detail}`;
         }
-        if (state === "dead" && observed.run) {
+        if ((state === "dead" || observed.stopReason) && observed.run) {
             try {
                 spawnSync(process.execPath, [yakCliPath(), "cancel", observed.run.runId], {
                     cwd: observed.run.checkout,
@@ -153,6 +181,64 @@ export function retryJob(config, jobId, host = defaultHost()) {
         return `retry ${report.jobId}  ${report.repo}\n${clearedLine}${output}`;
     });
 }
+function canManualRetry(state, job) {
+    if (state === "failed" || state === "dead")
+        return true;
+    return state === "needs-human" && Boolean(job.stopReason) && (job.run?.gates.length ?? 0) === 0;
+}
+/**
+ * Send a gated job back for another round. The note is stored on a
+ * `gate.resolved` event (redacted, otherwise verbatim) and appended to the
+ * spec the next launch passes into the prompt.
+ */
+export function reviseJob(config, jobId, note, host = defaultHost()) {
+    const trimmed = note.trim();
+    if (!trimmed)
+        throw new Error("A revise note is required.");
+    if (trimmed.length > 8_000)
+        throw new Error("A revise note must be 8000 characters or fewer.");
+    return withTickLock(config.kilnDir, () => {
+        const world = observe(config, host);
+        const observed = world.observation.jobs.find((candidate) => candidate.jobId === jobId);
+        if (!observed)
+            throw new Error(`No job ${jobId}.`);
+        if (observed.error)
+            throw new Error(`Job ${jobId} is invalid: ${observed.error}`);
+        const state = jobState(observed, world.observation);
+        const gate = observed.run?.gates[0];
+        if (state !== "needs-human" || !observed.run || !gate) {
+            throw new Error(`Job ${jobId} is ${state}, not waiting on a gate.`);
+        }
+        const job = world.jobs.get(jobId);
+        if (!job)
+            throw new Error(`No job ${jobId}.`);
+        appendEvent(config.kilnDir, {
+            job_id: jobId,
+            run_id: observed.run.runId,
+            event: "gate.resolved",
+            step: gate.stepId,
+            properties: {
+                decision: "revise",
+                by: "human",
+                repo: job.repo,
+                note: trimmed,
+            },
+        });
+        resetAttemptLedger(config.kilnDir, jobId, observed.run.runId);
+        rmSync(breadcrumbPath(config.kilnDir, jobId), { force: true });
+        clearLaunchError(config.kilnDir, jobId);
+        const events = readEvents(config.kilnDir, { warn: () => { } });
+        const spend = spendSnapshot(events, config.dailyDollarCap);
+        const paused = pausedFrom(events);
+        const hold = holdDetail(paused, spend);
+        if (hold)
+            return `revised ${jobId}\n${hold}\n`;
+        const again = observe(config, host);
+        const report = launchJob(config, again, job, host);
+        const output = report.output.endsWith("\n") || report.output.length === 0 ? report.output : `${report.output}\n`;
+        return `revised ${jobId}  ${report.repo}\n${output}`;
+    });
+}
 function shouldRedeliver(job) {
     if (!isLanded(job))
         return false;
@@ -164,9 +250,13 @@ function gateForChoice(gates, choice) {
     return (gates.find((gate) => gate.choices?.includes(choice)) ??
         gates.find((gate) => !gate.choices?.length));
 }
-function observe(config, host) {
+function observe(config, host, now = new Date()) {
     const loaded = loadJobs(config);
     const runs = scanRuns(config);
+    const events = readEvents(config.kilnDir, { warn: () => { } });
+    const paused = pausedFrom(events);
+    const spend = spendSnapshot(events, config.dailyDollarCap, now);
+    const waiting = holdDetail(paused, spend);
     const breadcrumbs = readBreadcrumbs(config.kilnDir);
     const jobs = [];
     const records = new Map();
@@ -213,10 +303,13 @@ function observe(config, host) {
         const launchActivityMs = later(crumb ? fileMtimeMs(crumb.path) : undefined, logPath ? fileMtimeMs(logPath) : undefined);
         const attemptCount = ledger?.attempts.length ?? 0;
         const lastFailure = ledger?.attempts.at(-1);
-        const exhausted = attemptCount >= config.maxAttempts &&
+        const attemptCap = attemptsFor(config, entry.job?.repo ?? "");
+        const exhausted = attemptCount >= attemptCap &&
             attemptCount > 0 &&
             !launching &&
             (picked.retired || !run || run.status === "dead" || run.status === "failed");
+        const eventStop = entry.job ? stopReasonFromEvents(events, entry.job.id) : undefined;
+        const stopReason = eventStop ?? (exhausted ? "attempt-cap" : undefined);
         const specPath = entry.job ? resolveJobSpec(config.kilnDir, entry.job.spec) : undefined;
         const findingsPath = entry.job?.workflow === INVESTIGATION_WORKFLOW
             ? existingFindingsPath(config.kilnDir, entry.job.id)
@@ -255,6 +348,8 @@ function observe(config, host) {
             ...(lastFailure ? { lastFailureReason: lastFailure.reason } : {}),
             ...(ledger?.nextRetryAt && !exhausted ? { nextRetryAt: ledger.nextRetryAt } : {}),
             ...(exhausted ? { retriesExhausted: true } : {}),
+            ...(stopReason ? { stopReason } : {}),
+            attemptCap,
             ...(picked.retired ? { runRetired: true } : {}),
             ...(ledgerResult === "corrupt" ? { ledgerError: "attempt ledger is unreadable" } : {}),
             ...(entry.job ? { sandboxed: jobIsSandboxed(config, entry.job.id, entry.job.workflow) } : {}),
@@ -275,6 +370,9 @@ function observe(config, host) {
             maxAttempts: config.maxAttempts,
             jobs,
             breadcrumbsToClear,
+            ...(paused ? { paused: true } : {}),
+            ...(spend.blocked ? { spendBlocked: true } : {}),
+            ...(waiting ? { holdDetail: waiting } : {}),
         },
         jobs: records,
     };
@@ -417,7 +515,8 @@ function specHandedToYak(config, world, job) {
         }
         sections.push({ title: dep.title, markdown: readFileSync(path, "utf8") });
     }
-    return appendInvestigationFindings(base, sections);
+    const noted = withReviseNotes(appendInvestigationFindings(base, sections), reviseNotesSinceRun(readEvents(config.kilnDir, { warn: () => { } }), job.id));
+    return noted;
 }
 /** Remote name the stacked pull request targets. Falls back to the local yak ref. */
 function upstreamBranch(world, job) {
@@ -529,6 +628,15 @@ function publishReadyFixes(config, host, world) {
         const record = readDelivery(config.kilnDir, job.jobId);
         const checkout = job.run.checkout;
         const source = `yak/${job.run.runId}`;
+        const worktree = join(checkout, ".yak", "worktrees", job.run.runId);
+        if (existsSync(worktree)) {
+            ensureFixTrailers(worktree, {
+                jobId: job.jobId,
+                round: open.round,
+                promptsHash: hashForRun(config.kilnDir, job.jobId, job.run.runId),
+                title: job.title,
+            }, open.headSha);
+        }
         if (!record?.branch) {
             notes.push(`feedback ${job.jobId} failed: no pull request branch to publish`);
             continue;
@@ -545,6 +653,7 @@ function publishReadyFixes(config, host, world) {
         if (sha === open.headSha) {
             open.outcome = "fixed";
             writeFeedbackLedger(config.kilnDir, ledger);
+            recordFeedbackOutcome(config.kilnDir, job.jobId, job.run.runId, open, "fixed");
             notes.push(`fixed ${job.jobId}  ${sha}`);
             continue;
         }
@@ -580,13 +689,62 @@ function publishReadyFixes(config, host, world) {
         });
         open.outcome = "pushed";
         writeFeedbackLedger(config.kilnDir, ledger);
+        recordFeedbackOutcome(config.kilnDir, job.jobId, job.run.runId, open, "pushed");
         notes.push(`pushed ${job.jobId}  ${record.branch}  ${sha}`);
     }
     return notes;
 }
+function hashForRun(kilnDir, jobId, runId) {
+    for (const event of readEvents(kilnDir, { warn: () => { } })) {
+        if (event.job_id === jobId && event.run_id === runId && event.event === "run.started") {
+            return event.properties.prompts_hash;
+        }
+    }
+    return promptsHash("fix", kilnDir, jobId);
+}
 function isNonFastForward(message) {
     return /non-fast-forward|\[rejected\]|failed to push some refs/i.test(message);
 }
+function recordFeedbackOutcome(kilnDir, jobId, runId, round, outcome) {
+    appendEvent(kilnDir, {
+        job_id: jobId,
+        run_id: runId,
+        event: "pr.feedback",
+        properties: {
+            round: round.round,
+            trigger: round.trigger,
+            outcome,
+            items: round.items.length,
+            once: `pr.feedback:${round.round}:${outcome}`,
+        },
+    });
+}
+/** An edit note the human left on the gate answer, when the answer file has one. */
+function gateEdit(checkout, runId, stepId) {
+    const paths = [
+        join(checkout, ".runs", runId, "pending", `${stepId}.answer.json`),
+        join(checkout, ".runs", runId, "artifacts", "approval.json"),
+    ];
+    for (const path of paths) {
+        if (!existsSync(path))
+            continue;
+        try {
+            const parsed = JSON.parse(readFileSync(path, "utf8"));
+            const note = typeof parsed.note === "string" ? parsed.note.trim() : "";
+            const edit = typeof parsed.edit === "string" ? parsed.edit.trim() : "";
+            if (!note && !edit)
+                continue;
+            return {
+                ...(note ? { note: note.slice(0, 240) } : {}),
+                ...(edit ? { edit: edit.slice(0, 240) } : {}),
+            };
+        }
+        catch {
+            // An unreadable answer is not an edit note.
+        }
+    }
+    return {};
+}
 /**
  * Read each open pull request once per tick and attach a plan.
  * `readStatus` does not call this. A host with no feedback reader does not watch.
@@ -613,7 +771,7 @@ function watchPullRequests(config, world, host) {
             continue;
         }
         job.feedbackPlan = planFeedback({
-            maxFixRounds: config.maxFixRounds,
+            maxFixRounds: fixRoundsFor(config, job.repo),
             ledger,
             snapshot,
         });
@@ -641,6 +799,11 @@ function markFeedback(kilnDir, jobId, reason) {
     else
         ledger.ambiguous = true;
     writeFeedbackLedger(kilnDir, ledger);
+    appendEvent(kilnDir, {
+        job_id: jobId,
+        event: "needs-human",
+        properties: { reason, once: `needs-human:${reason}` },
+    });
 }
 /**
  * Open one feedback round and start the fix workflow on the pull request head.
@@ -678,6 +841,16 @@ function startFix(config, world, job, host, now, sandboxFacts) {
         at: now.toISOString(),
     });
     writeFeedbackLedger(config.kilnDir, ledger);
+    appendEvent(config.kilnDir, {
+        job_id: job.id,
+        event: "pr.feedback",
+        properties: {
+            round,
+            trigger: plan.trigger,
+            items: plan.items.length,
+            once: `pr.feedback:${round}:open`,
+        },
+    });
     let specText;
     try {
         specText = feedbackSpec(specHandedToYak(config, world, job), plan.items);
@@ -691,7 +864,7 @@ function startFix(config, world, job, host, now, sandboxFacts) {
         writeBreadcrumb(breadcrumbPath(config.kilnDir, job.id), { jobId: job.id, pid: 0, logPath });
         return { jobId: job.id, repo: job.repo, kind: "fix", output: message };
     }
-    const report = launchFixRun(config, job, host, repo.path, headSha, specText, sandboxFacts);
+    const report = launchFixRun(config, job, host, repo.path, headSha, specText, round, sandboxFacts);
     return { ...report, output: `round ${round}\n${report.output}` };
 }
 /** Relaunch the open fix round. Does not append a round. Failures become a dead launch so retry can see them. */
@@ -712,7 +885,7 @@ function launchOpenFix(config, world, job, host, checkout, open, sandboxFacts) {
         writeBreadcrumb(breadcrumbPath(config.kilnDir, job.id), { jobId: job.id, pid: 0, logPath });
         return { jobId: job.id, repo: job.repo, kind: "fix", output: message };
     }
-    return launchFixRun(config, job, host, checkout, headSha, specText, sandboxFacts);
+    return launchFixRun(config, job, host, checkout, headSha, specText, open.round, sandboxFacts);
 }
 function resolveCommit(host, checkout, sha) {
     try {
@@ -724,7 +897,7 @@ function resolveCommit(host, checkout, sha) {
     }
 }
 /** `workflows/fix.yaml` on `headSha`. Commits on `yak/<run-id>`. Does not push. */
-function launchFixRun(config, job, host, checkout, headSha, specText, sandboxFacts) {
+function launchFixRun(config, job, host, checkout, headSha, specText, round, sandboxFacts) {
     const refused = claudeCredentialPreflight(process.env, config.adapter);
     if (refused)
         return { jobId: job.id, repo: job.repo, kind: "fix", output: refused };
@@ -736,6 +909,7 @@ function launchFixRun(config, job, host, checkout, headSha, specText, sandboxFac
     const blocked = commitAgentSandbox(config, job.id, "fix", workflow, logPath, sandboxFacts);
     if (blocked)
         return { jobId: job.id, repo: job.repo, kind: "fix", output: blocked };
+    const hash = promptsHash("fix", config.kilnDir, job.id);
     ensureCheckoutExcludes(checkout);
     const input = workflowInputs(sourceWorkflow, {
         title: job.title,
@@ -748,6 +922,9 @@ function launchFixRun(config, job, host, checkout, headSha, specText, sandboxFac
         ...process.env,
         ...(repo?.test ? { KILN_TEST: repo.test } : {}),
         KILN_TASK_TITLE: job.title,
+        KILN_JOB: job.id,
+        KILN_ROUND: String(round),
+        KILN_PROMPTS_HASH: hash,
         KILN_VERIFY_WRAPPER: join(packageRoot(), "dist", "truncate-output.js"),
     };
     return spawnLogged(config, host, {
@@ -943,6 +1120,11 @@ function loadJobFile(file) {
     return { meta: { jobId: job.id }, job };
 }
 function scanRuns(config) {
+    const jobs = new Map();
+    for (const entry of loadJobs(config)) {
+        if (entry.job)
+            jobs.set(entry.job.id, entry.job);
+    }
     const byTag = new Map();
     for (const repo of Object.values(config.repos)) {
         for (const runDir of listRunDirs(repo.path)) {
@@ -951,6 +1133,21 @@ function scanRuns(config) {
             if (!started?.tag || !parseJobTag(started.tag))
                 continue;
             const runId = basename(runDir);
+            const jobId = parseJobTag(started.tag);
+            if (jobId) {
+                const job = jobs.get(jobId);
+                const testCommand = job ? config.repos[job.repo]?.test : undefined;
+                projectRunJournal({
+                    kilnDir: config.kilnDir,
+                    jobId,
+                    runId,
+                    runDir,
+                    workflow: job?.workflow ?? "",
+                    repo: job?.repo ?? "",
+                    ...(testCommand ? { testCommand } : {}),
+                    events,
+                });
+            }
             const status = classify(events);
             const gateDecision = readApproveDecision(runDir);
             const title = readRunTitle(runDir);
@@ -958,6 +1155,7 @@ function scanRuns(config) {
             const stepPlan = readRunStepPlan(runDir);
             const stepEvents = stepEventsFrom(events);
             const sandboxError = sandboxFailureDetail(stepEvents);
+            const cost = costFromJournal(events);
             const pid = livePid(events);
             const activityMs = eventActivityMs(events) ?? fileMtimeMs(join(runDir, "journal.jsonl"));
             const finishedReason = finished && typeof finished.reason === "string" ? finished.reason : undefined;
@@ -976,6 +1174,7 @@ function scanRuns(config) {
                 ...(activityMs !== undefined ? { activityMs } : {}),
                 ...(finishedReason ? { finishedReason } : {}),
                 ...(sandboxError ? { sandboxError } : {}),
+                cost,
             };
             const list = byTag.get(started.tag) ?? [];
             list.push(run);