@lchase/kiln 0.9.0 → 0.11.0

package/dist/reliability.js

@@ -4,7 +4,10 @@ import { appendFileSync, closeSync, existsSync, mkdirSync, openSync, readdirSync
 import { dirname, join, resolve, sep } from "node:path";
 import { z } from "zod";
 import { authenticationFailureLine } from "./auth.js";
-import { appendEvent } from "./events.js";
+import { attemptsFor } from "./config.js";
+import { appendEvent, readEvents } from "./events.js";
+import { decideFailure, failureFingerprint, isSandboxStartFlake } from "./failure.js";
+import { fingerprintCount } from "./rails.js";
 import { jobBasename } from "./paths.js";
 import { FAILURE_REASONS } from "./reconcile.js";
 /**
@@ -88,25 +91,23 @@ export function resetAttemptLedger(kilnDir, jobId, runId) {
  */
 export function recoverRuns(config, jobs, options) {
     const notes = [];
+    const events = readEvents(config.kilnDir, { warn: () => { } });
     for (const job of jobs) {
         if (job.error)
             continue;
         const existing = readAttemptLedger(config.kilnDir, job.jobId);
         if (existing === "corrupt")
             continue;
-        if (job.run?.sandboxError && job.run.status !== "alive") {
-            suppressAutomaticRetry(config.kilnDir, job.jobId);
-            continue;
-        }
         const pending = pendingFailure(job, config, options);
         if (!pending)
             continue;
         if (pending.killPid !== undefined && !killTree(pending.killPid))
             continue;
-        const recorded = recordFailure(config, job.jobId, existing, pending, options.now);
+        const recorded = recordFailure(config, job, existing, pending, options.now, events);
         if (recorded.created) {
             appendLog(job.logPath, recorded.ledger.attempts.at(-1));
             const runId = pending.runId ?? job.run?.runId;
+            const cost = job.run?.cost ?? { tokens: null, usd: null };
             appendEvent(config.kilnDir, {
                 job_id: job.jobId,
                 run_id: runId ?? null,
@@ -114,15 +115,31 @@ export function recoverRuns(config, jobs, options) {
                 properties: {
                     reason: pending.reason,
                     ...(pending.detail ? { detail: pending.detail } : {}),
+                    class: recorded.decision.class,
+                    fingerprint: recorded.decision.fingerprint,
+                    tokens: cost.tokens,
+                    usd: cost.usd,
                     once: runId ? `run.failed:${runId}` : `run.failed:launch:${recorded.ledger.attempts.length}`,
                 },
             });
+            if (recorded.decision.needsHuman) {
+                appendEvent(config.kilnDir, {
+                    job_id: job.jobId,
+                    run_id: runId ?? null,
+                    event: "needs-human",
+                    properties: {
+                        reason: recorded.decision.needsHuman,
+                        once: `needs-human:${recorded.decision.needsHuman}:${runId ?? recorded.ledger.attempts.length}`,
+                    },
+                });
+            }
             notes.push({
                 jobId: job.jobId,
                 kind: kindFor(pending.reason),
                 reason: pending.reason,
                 attempt: recorded.ledger.attempts.length,
                 ...(recorded.ledger.nextRetryAt ? { nextRetryAt: recorded.ledger.nextRetryAt } : {}),
+                ...(recorded.decision.needsHuman ? { needsHuman: true } : {}),
             });
         }
         const checkout = job.run?.checkout ?? config.repos[job.repo]?.path;
@@ -141,18 +158,6 @@ export function recoverRuns(config, jobs, options) {
     }
     return notes;
 }
-/** Drop a scheduled retry so a sandbox failure stays in Needs human. */
-function suppressAutomaticRetry(kilnDir, jobId) {
-    const existing = readAttemptLedger(kilnDir, jobId);
-    if (!existing || existing === "corrupt" || !existing.nextRetryAt)
-        return;
-    const written = {
-        jobId: existing.jobId,
-        attempts: existing.attempts,
-        ...(existing.ignoredRunIds && existing.ignoredRunIds.length > 0 ? { ignoredRunIds: existing.ignoredRunIds } : {}),
-    };
-    writeLedger(kilnDir, written);
-}
 function pendingFailure(job, config, options) {
     if (job.run?.gateDecision === "reject")
         return undefined;
@@ -261,10 +266,24 @@ function readAuthLog(logPath) {
             closeSync(fd);
     }
 }
-function recordFailure(config, jobId, existing, pending, now) {
-    const ledger = existing ?? { jobId, attempts: [] };
+function recordFailure(config, job, existing, pending, now, events) {
+    const ledger = existing ?? { jobId: job.jobId, attempts: [] };
+    const text = failureText(job, pending);
+    const decision = decideFailure({
+        text,
+        operational: pending.reason === "stalled" ||
+            pending.reason === "orphaned" ||
+            pending.reason === "crashed" ||
+            pending.reason === "failed",
+        blocked: pending.reason === "auth-failed" ||
+            (Boolean(job.run?.sandboxError) && !isSandboxStartFlake(text)),
+        attempt: ledger.attempts.length + 1,
+        maxAttempts: attemptsFor(config, job.repo),
+        sameFingerprint: fingerprintCount(events, job.jobId, failureFingerprint(text)) + 1,
+        maxSameFingerprint: config.maxSameFingerprint,
+    });
     if (alreadyRecorded(ledger, pending))
-        return { ledger, created: false };
+        return { ledger, created: false, decision };
     const attempt = {
         n: ledger.attempts.length + 1,
         at: now.toISOString(),
@@ -274,16 +293,25 @@ function recordFailure(config, jobId, existing, pending, now) {
         detail: pending.detail,
     };
     const attempts = [...ledger.attempts, attempt];
-    const exhausted = attempts.length >= config.maxAttempts || pending.reason === "auth-failed";
-    const next = exhausted ? undefined : new Date(now.getTime() + retryDelayMs(attempts.length)).toISOString();
+    const next = decision.retry ? new Date(now.getTime() + retryDelayMs(attempts.length)).toISOString() : undefined;
     const written = {
-        jobId,
+        jobId: job.jobId,
         attempts,
         ...(next ? { nextRetryAt: next } : {}),
         ...(ledger.ignoredRunIds && ledger.ignoredRunIds.length > 0 ? { ignoredRunIds: ledger.ignoredRunIds } : {}),
     };
     writeLedger(config.kilnDir, written);
-    return { ledger: written, created: true };
+    return { ledger: written, created: true, decision };
+}
+function failureText(job, pending) {
+    const parts = [pending.detail];
+    if (job.run?.sandboxError)
+        parts.push(job.run.sandboxError);
+    for (const event of job.run?.stepEvents ?? []) {
+        if (event.detail)
+            parts.push(event.detail);
+    }
+    return parts.filter((part) => part.length > 0).join("\n");
 }
 function alreadyRecorded(ledger, pending) {
     if (pending.runId)

package/dist/report.js

@@ -159,6 +159,10 @@ export function statusDetails(job) {
         lines.push(`feedback ${job.feedbackRounds?.length}`);
     if (job.state === "needs-human" && job.feedbackReason)
         lines.push(`reason ${job.feedbackReason}`);
+    if (job.stopReason)
+        lines.push(`reason ${job.stopReason}`);
+    if (job.waitReason)
+        lines.push(job.waitReason);
     if (job.attemptCount > 0)
         lines.push(`attempts ${job.attemptCount}`);
     if (job.lastFailureReason)
@@ -167,8 +171,9 @@ export function statusDetails(job) {
         lines.push(AUTH_FAILED_LINE);
     if (job.nextRetryAt)
         lines.push(`next retry ${job.nextRetryAt}`);
-    if (job.state === "dead" && job.attemptCount > 0)
+    if ((job.state === "dead" || job.stopReason) && job.attemptCount > 0 && job.gates.length === 0) {
         lines.push(`retry: kiln retry ${job.jobId}`);
+    }
     if (job.noReadableApproval)
         lines.push("no readable approval decision");
     if (job.cancelCommand)
@@ -186,6 +191,9 @@ export function statusDetails(job) {
                 lines.push(`approve: kiln approve ${job.jobId}`);
             if (gate.choices.includes("reject"))
                 lines.push(`reject: kiln reject ${job.jobId}`);
+            if (gate.choices.includes("approve") || gate.choices.includes("revise")) {
+                lines.push(`revise: kiln revise ${job.jobId} --note <text>`);
+            }
         }
     }
     return lines;
@@ -216,7 +224,7 @@ function toStatusJob(job, observation) {
             ...(job.findingsPath ? { findingsPath: job.findingsPath } : {}),
             launchFailed: job.launchFailed,
             attemptCount: job.attemptCount ?? 0,
-            maxAttempts: observation.maxAttempts,
+            maxAttempts: job.attemptCap ?? observation.maxAttempts,
             ...(job.lastFailureReason ? { lastFailureReason: job.lastFailureReason } : {}),
             ...(job.nextRetryAt ? { nextRetryAt: job.nextRetryAt } : {}),
             invalidReason: job.error,
@@ -226,6 +234,7 @@ function toStatusJob(job, observation) {
             gates: [],
             canApprove: false,
             canReject: false,
+            canRevise: false,
             steps: progressFor(job),
             ...(job.deliver ? { deliver: job.deliver } : {}),
             ...(job.createdAt ? { createdAt: job.createdAt } : {}),
@@ -239,7 +248,7 @@ function toStatusJob(job, observation) {
         : job.feedbackAmbiguous
             ? "feedback-ambiguous"
             : undefined;
-    const gates = state === "needs-human" && job.run && !feedbackReason ? job.run.gates.map(toGateView) : [];
+    const gates = state === "needs-human" && job.run && !feedbackReason && !job.stopReason ? job.run.gates.map(toGateView) : [];
     const prNumber = job.delivery?.prNumber;
     const runId = job.run?.runId;
     const showBranch = runId !== undefined && (state === "done" || state === "in-review" || feedbackReason !== undefined);
@@ -263,8 +272,12 @@ function toStatusJob(job, observation) {
         ...(job.findingsPath ? { findingsPath: job.findingsPath } : {}),
         launchFailed: job.launchFailed,
         attemptCount: job.attemptCount ?? 0,
-        maxAttempts: observation.maxAttempts,
+        maxAttempts: job.attemptCap ?? observation.maxAttempts,
         ...(job.lastFailureReason ? { lastFailureReason: job.lastFailureReason } : {}),
+        ...(job.stopReason ? { stopReason: job.stopReason } : {}),
+        ...(observation.holdDetail && (state === "queued" || state === "blocked" || state === "failed")
+            ? { waitReason: observation.holdDetail }
+            : {}),
         ...(job.nextRetryAt && !job.retriesExhausted ? { nextRetryAt: job.nextRetryAt } : {}),
         ...(job.ledgerError ? { ledgerError: job.ledgerError } : {}),
         ...(job.feedbackLedgerError ? { feedbackLedgerError: job.feedbackLedgerError } : {}),
@@ -281,6 +294,7 @@ function toStatusJob(job, observation) {
         gates,
         canApprove: gates.some((gate) => gate.choices.includes("approve")),
         canReject: gates.some((gate) => gate.choices.includes("reject")),
+        canRevise: gates.some((gate) => gate.choices.includes("approve") || gate.choices.includes("revise")),
         steps: progressFor(job),
         ...(job.createdAt ? { createdAt: job.createdAt } : {}),
         ...(job.run?.startedAt ? { startedAt: job.run.startedAt } : {}),
@@ -402,7 +416,7 @@ export function renderTick(observation, actions, launches, delivers = [], findin
             : `findings ${recorded.jobId}  missing`);
     }
     for (const note of notes) {
-        const next = note.nextRetryAt ? `  next ${note.nextRetryAt}` : note.reason === "auth-failed" ? "" : "  dead";
+        const next = note.nextRetryAt ? `  next ${note.nextRetryAt}` : note.needsHuman ? "  needs human" : "";
         lines.push(`${note.kind} ${note.jobId}  ${note.reason}  attempt ${note.attempt}${next}`);
     }
     for (const note of feedbackNotes) {

package/dist/schemas.js

@@ -36,6 +36,18 @@ export const RepoEntrySchema = z.object({
      * The local branch stays `yak/<run-id>`.
      */
     branch: z.string().min(1, "Branch template is empty.").optional(),
+    /** Overrides the global attempt cap for jobs in this repo. */
+    maxAttempts: z
+        .number()
+        .int("Max attempts must be a whole number.")
+        .min(1, "Max attempts must be at least 1.")
+        .optional(),
+    /** Overrides the global fix-round cap for jobs in this repo. */
+    maxFixRounds: z
+        .number()
+        .int("Max fix rounds must be a whole number.")
+        .min(1, "Max fix rounds must be at least 1.")
+        .optional(),
 });
 /** Repo map shared by the config file and the settings API. */
 export const KilnReposSchema = z.record(z.string().regex(REPO_NAME, "must start with a letter or digit and contain only letters, digits, dots, underscores, and hyphens"), RepoEntrySchema);
@@ -75,21 +87,34 @@ export const EditableSandboxSchema = SandboxSettingsSchema.pick({
 const WORKFLOW_KEY = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
 /** No yak event and no log output for this long stalls a running job. `<= 0` disables it. */
 export const DEFAULT_STALL_TIMEOUT_MS = 15 * 60 * 1000;
-/** Transient failures retry until this many attempts, then the job is dead. */
+/** Transient failures retry until this many attempts, then the job needs a person. */
 export const DEFAULT_MAX_ATTEMPTS = 3;
 /** Pull request fix rounds before the job is needs-human (`feedback-exhausted`). */
 export const DEFAULT_MAX_FIX_ROUNDS = 2;
+/** Identical failure fingerprints before the job needs a person. */
+export const DEFAULT_MAX_SAME_FINGERPRINT = 3;
 /**
  * Milliseconds without a yak event or log output before a running job is
  * killed and retried. Default 15 minutes. `0` or negative disables stall detection.
  */
 const stallTimeoutMs = z.number().int("Stall timeout must be a whole number of milliseconds.").optional();
-/** Transient failures before the job is dead. Default 3. */
+/** Transient failures before the job needs a person. Default 3. Hitting the cap is a budget failure. */
 const maxAttempts = z
     .number()
     .int("Max attempts must be a whole number.")
     .min(1, "Max attempts must be at least 1.")
     .optional();
+/** Same failure fingerprint before Needs human. Default 3. */
+const maxSameFingerprint = z
+    .number()
+    .int("Fingerprint limit must be a whole number.")
+    .min(1, "Fingerprint limit must be at least 1.")
+    .optional();
+/**
+ * Dollars of recorded run cost allowed on the local calendar day.
+ * Omitted means no cap. `0` blocks new runs immediately.
+ */
+const dailyDollarCap = z.number().nonnegative("Daily dollar cap cannot be negative.").optional();
 /**
  * Fix rounds for an open pull request before the job is needs-human with
  * reason `feedback-exhausted`. Default 2. Kiln does not merge.
@@ -109,6 +134,8 @@ export const EditableKilnSettingsSchema = z.object({
     stallTimeoutMs,
     maxAttempts,
     maxFixRounds,
+    maxSameFingerprint,
+    dailyDollarCap,
     sandbox: EditableSandboxSchema.optional(),
 });
 /** PUT /api/settings body. `revision` is the sha256 of the file the client loaded. */
@@ -121,6 +148,8 @@ export const KilnConfigFileSchema = z.object({
     stallTimeoutMs,
     maxAttempts,
     maxFixRounds,
+    maxSameFingerprint,
+    dailyDollarCap,
     adapter: z.enum(["mock", "claude-code"]).optional(),
     sandbox: SandboxSettingsSchema.optional(),
     /** Per-workflow sandbox override. A set field replaces the global one; an omitted field inherits. */
@@ -176,9 +205,10 @@ export const JobSchema = z.object({
  * `deliver=pr`. It does not enter `in-review`. A reject, a failure, a dead
  * process, and an ok run with no readable approval do not unblock an
  * after-merged dependent. A stacked job unblocks when its one upstream has
- * landed, which includes `in-review`. `dead` is also a job whose transient
- * failures have used `maxAttempts`. Those attempts live under the kiln
- * directory, not in this file.
+ * landed, which includes `in-review`. A job that has used its attempt cap,
+ * fix-round cap, or fingerprint breaker is `needs-human`. `dead` is a run
+ * whose process died and has not been recorded yet. Attempts live under
+ * the kiln directory, not in this file.
  */
 export const JOB_STATES = [
     "queued",