@lchase/kiln 0.9.0 → 0.11.0

package/dist/events.js

@@ -9,7 +9,8 @@ import { redactDeep } from "./redact.js";
  * parses it. A write failure is reported and does not escape to the caller.
  *
  * `run_id` and `step` are null when the moment has none. `job_id` is null
- * only for `config.reloaded`, which is not a job.
+ * for `config.reloaded`, `kiln.paused`, `kiln.resumed`, and `budget.blocked`,
+ * which are not a job.
  */
 export const EVENT_NAMES = [
     "job.added",
@@ -27,6 +28,9 @@ export const EVENT_NAMES = [
     "worktree.kept",
     "config.reloaded",
     "needs-human",
+    "kiln.paused",
+    "kiln.resumed",
+    "budget.blocked",
 ];
 const ISO_UTC = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$/;
 const once = z.string().min(1).optional();
@@ -80,14 +84,24 @@ const GateResolved = z.object({
     edit: z.string().min(1).optional(),
     once,
 });
+const costFields = {
+    /** Sum of yak `budget.consumed` tokens. Null when the journal did not report any. */
+    tokens: z.number().nonnegative().nullable().optional(),
+    /** Sum of reported dollars. Null when the journal did not report any. Never invented. */
+    usd: z.number().nonnegative().nullable().optional(),
+};
 const RunFailed = z.object({
     reason: z.string().min(1).optional(),
     detail: z.string().min(1).optional(),
     status: z.string().min(1).optional(),
+    class: z.enum(["transient", "deterministic", "budget"]).optional(),
+    fingerprint: z.string().min(1).optional(),
+    ...costFields,
     once,
 });
 const RunFinished = z.object({
     status: z.literal("ok"),
+    ...costFields,
     once,
 });
 const PrOpened = z.object({
@@ -110,6 +124,13 @@ const NeedsHuman = z.object({
     reason: z.string().min(1),
     once,
 });
+const FactoryToggle = z.object({ once });
+const BudgetBlocked = z.object({
+    day: z.string().min(1),
+    usd: z.number().nonnegative(),
+    cap: z.number().nonnegative(),
+    once,
+});
 const envelope = {
     id: z.string().min(1),
     ts: z.string().regex(ISO_UTC, "ts must be ISO-8601 UTC"),
@@ -137,6 +158,24 @@ export const EventSchema = z.discriminatedUnion("event", [
         properties: ConfigReloaded,
     }),
     z.object({ ...envelope, event: z.literal("needs-human"), job_id: z.string().min(1), properties: NeedsHuman }),
+    z.object({
+        ...envelope,
+        event: z.literal("kiln.paused"),
+        job_id: z.string().min(1).nullable(),
+        properties: FactoryToggle,
+    }),
+    z.object({
+        ...envelope,
+        event: z.literal("kiln.resumed"),
+        job_id: z.string().min(1).nullable(),
+        properties: FactoryToggle,
+    }),
+    z.object({
+        ...envelope,
+        event: z.literal("budget.blocked"),
+        job_id: z.string().min(1).nullable(),
+        properties: BudgetBlocked,
+    }),
 ]);
 const LOCK_WAIT_MS = 5_000;
 const pauseBuf = new Int32Array(new SharedArrayBuffer(4));

package/dist/failure.js

@@ -0,0 +1,140 @@
+import { createHash } from "node:crypto";
+import { redact } from "./redact.js";
+/**
+ * One failed run is transient, deterministic, or budget.
+ * Transient may retry with backoff inside the attempt cap.
+ * Deterministic does not. A cap hit is budget and stops in Needs human.
+ */
+export const FAILURE_CLASSES = ["transient", "deterministic", "budget"];
+const TRANSIENT = [
+    /\b429\b/,
+    /rate limit/i,
+    /too many requests/i,
+    /\b5\d\d\b/,
+    /internal server error/i,
+    /bad gateway/i,
+    /service unavailable/i,
+    /gateway timeout/i,
+    /ECONNRESET|ECONNREFUSED|ENOTFOUND|ETIMEDOUT|EAI_AGAIN|EPIPE/i,
+    /socket hang up/i,
+    /network (?:error|unreachable|is down)/i,
+    /fetch failed/i,
+    /connection (?:reset|refused|timed out|closed)/i,
+    /container never started/i,
+    /failed to start (?:the )?(?:container|sandbox)/i,
+    /docker daemon/i,
+    /cannot connect to the docker/i,
+    /OCI runtime/i,
+    /error creating container/i,
+    /sandbox (?:start|flake)/i,
+];
+const DETERMINISTIC = [
+    /\berror TS\d+\b/,
+    /type error/i,
+    /cannot find name/i,
+    /is not assignable/i,
+    /syntax error/i,
+    /\bAssertionError\b/,
+    /\b(?:tests?|assertion) failed\b/i,
+    /\bfailing tests?\b/i,
+    /\bbuild failed\b/i,
+    /\bcompilation failed\b/i,
+    /\bcompile error\b/i,
+    /npm ERR!/i,
+];
+/** A container or sandbox that did not start. Those are transient. */
+export function isSandboxStartFlake(text) {
+    return /container never started|failed to start (?:the )?(?:container|sandbox)|docker daemon|cannot connect to the docker|OCI runtime|error creating container|sandbox (?:start|flake)/i.test(text);
+}
+/**
+ * Strip timestamps, temp and worktree paths, hex ids, and line numbers so
+ * the same failure hashes the same way across runs.
+ */
+export function normalizeFailureText(text) {
+    let out = redact(text).replace(/\r\n/g, "\n").trim();
+    out = out.replace(/\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?/g, "<ts>");
+    out = out.replace(/\b\d{2}:\d{2}:\d{2}(?:\.\d+)?\b/g, "<ts>");
+    out = out.replace(/(?:\/private)?\/(?:tmp|var\/folders)\/[^\s'")\]:]+/g, "<tmp>");
+    out = out.replace(/[^\s'")\]]*[/\\]\.yak[/\\]worktrees[/\\][^\s'")\]:]+/g, "<worktree>");
+    out = out.replace(/[^\s'")\]]+[/\\]worktrees[/\\][^\s'")\]:]+/g, "<worktree>");
+    out = out.replace(/\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/gi, "<id>");
+    out = out.replace(/\b[0-9a-f]{8,}\b/gi, "<id>");
+    out = out.replace(/:(\d+)(?::\d+)?\b/g, ":<line>");
+    out = out.replace(/\bline\s+\d+\b/gi, "line <line>");
+    out = out.replace(/[ \t]+/g, " ");
+    out = out.replace(/\n{3,}/g, "\n\n");
+    return out.toLowerCase();
+}
+/** Stable id for one normalized failure. */
+export function failureFingerprint(text) {
+    return createHash("sha256").update(normalizeFailureText(text)).digest("hex");
+}
+/**
+ * Transient when the text is a network, rate-limit, 5xx, or start flake.
+ * A test, build, or type error is deterministic and wins over a transient
+ * pattern in the same text. Anything else is deterministic.
+ */
+export function classifyFailureText(text) {
+    if (DETERMINISTIC.some((pattern) => pattern.test(text)))
+        return "deterministic";
+    if (TRANSIENT.some((pattern) => pattern.test(text)))
+        return "transient";
+    return "deterministic";
+}
+/**
+ * The only decision for what a recorded failure does next.
+ * Attempt cap wins over the fingerprint breaker when both trip on the same run.
+ */
+export function decideFailure(input) {
+    const fingerprint = failureFingerprint(input.text);
+    const deterministicHit = DETERMINISTIC.some((pattern) => pattern.test(input.text));
+    const transientHit = TRANSIENT.some((pattern) => pattern.test(input.text));
+    // Stall, crash, orphan, and a bare non-zero exit stay transient unless the
+    // text is a test, build, or type error, or the failure is blocked.
+    let failureClass = deterministicHit
+        ? "deterministic"
+        : transientHit || (input.operational && !input.blocked)
+            ? "transient"
+            : "deterministic";
+    let retry = failureClass === "transient" && !input.blocked;
+    let needsHuman;
+    if (input.attempt >= input.maxAttempts) {
+        failureClass = "budget";
+        retry = false;
+        needsHuman = "attempt-cap";
+    }
+    else if (input.sameFingerprint >= input.maxSameFingerprint) {
+        retry = false;
+        needsHuman = "fingerprint";
+    }
+    else if (input.blocked || failureClass === "deterministic") {
+        retry = false;
+    }
+    return { class: failureClass, fingerprint, retry, ...(needsHuman ? { needsHuman } : {}) };
+}
+/** Sum yak `budget.consumed` lines. Missing numbers stay null. Never invents a figure. */
+export function costFromJournal(events) {
+    let tokens = 0;
+    let usd = 0;
+    let sawTokens = false;
+    let sawUsd = false;
+    for (const event of events) {
+        if (!event || typeof event !== "object")
+            continue;
+        const record = event;
+        if (record.t !== "budget.consumed")
+            continue;
+        if (typeof record.tokens === "number" && Number.isFinite(record.tokens) && record.tokens >= 0) {
+            tokens += record.tokens;
+            sawTokens = true;
+        }
+        if (typeof record.usd === "number" && Number.isFinite(record.usd) && record.usd >= 0) {
+            usd += record.usd;
+            sawUsd = true;
+        }
+    }
+    return {
+        tokens: sawTokens ? tokens : null,
+        usd: sawUsd ? usd : null,
+    };
+}

package/dist/fix-commit.js

@@ -0,0 +1,73 @@
+import { spawnSync } from "node:child_process";
+export function fixTrailerLines(input) {
+    return [`Kiln-Job: ${input.jobId}`, `Kiln-Round: ${input.round}`, `Kiln-Prompts-Hash: ${input.promptsHash}`];
+}
+/**
+ * The fix workflow `land` step. `workflows/fix.yaml` keeps the same command.
+ * Launch sets `KILN_JOB`, `KILN_ROUND`, and `KILN_PROMPTS_HASH`.
+ */
+export const FIX_LAND_COMMAND = 'git add -A && git diff --cached --quiet || git commit -m "kiln: $KILN_TASK_TITLE" --trailer "Kiln-Job: $KILN_JOB" --trailer "Kiln-Round: $KILN_ROUND" --trailer "Kiln-Prompts-Hash: $KILN_PROMPTS_HASH"';
+/**
+ * Make the tip of `dir` carry the fix-round trailers.
+ * A dirty tree is committed. A tip that already moved and lacks the trailers
+ * is amended. A clean tree still on `baseSha` is left alone.
+ */
+export function ensureFixTrailers(dir, input, baseSha) {
+    if (!hasTrailers(dir, input)) {
+        if (isDirty(dir)) {
+            commit(dir, input, false);
+            return "committed";
+        }
+        const head = revParse(dir, "HEAD");
+        if (!head || (baseSha && head === baseSha))
+            return "clean";
+        commit(dir, input, true);
+        return "amended";
+    }
+    return "present";
+}
+export function commitMessageHasTrailers(message, input) {
+    return fixTrailerLines(input).every((line) => message.split("\n").some((entry) => entry.trim() === line));
+}
+function hasTrailers(dir, input) {
+    const message = git(dir, ["log", "-1", "--format=%B"]);
+    if (message.status !== 0)
+        return false;
+    return commitMessageHasTrailers(message.stdout, input);
+}
+function isDirty(dir) {
+    const status = git(dir, ["status", "--porcelain"]);
+    if (status.status !== 0)
+        return false;
+    return status.stdout.trim().length > 0;
+}
+function revParse(dir, ref) {
+    const result = git(dir, ["rev-parse", ref]);
+    if (result.status !== 0)
+        return undefined;
+    const sha = result.stdout.trim();
+    return sha || undefined;
+}
+function commit(dir, input, amend) {
+    if (!amend) {
+        const added = git(dir, ["add", "-A"]);
+        if (added.status !== 0)
+            return;
+    }
+    const args = ["commit"];
+    if (amend)
+        args.push("--amend", "--no-edit");
+    else
+        args.push("-m", `kiln: ${input.title}`);
+    for (const line of fixTrailerLines(input))
+        args.push("--trailer", line);
+    git(dir, args);
+}
+function git(cwd, args) {
+    const result = spawnSync("git", args, {
+        cwd,
+        encoding: "utf8",
+        env: { ...process.env, GIT_TERMINAL_PROMPT: "0" },
+    });
+    return { status: result.status ?? 1, stdout: result.stdout ?? "" };
+}

package/dist/ledger.js

@@ -1,5 +1,6 @@
 import { readEvents } from "./events.js";
 import { redact } from "./redact.js";
+import { formatSpend, spendSnapshot } from "./rails.js";
 /**
  * Interventions are derived from the event log. Nothing is stored beside it.
  * A human had to act when a gate was rejected, a job was retried by hand,
@@ -64,9 +65,14 @@ export function ledgerRows(events, options = {}) {
     return rows;
 }
 export function renderLedger(events, options = {}) {
+    const spend = options.spend ??
+        (options.dailyDollarCap !== undefined || options.now
+            ? spendSnapshot(events, options.dailyDollarCap, options.now ?? new Date())
+            : undefined);
+    const head = spend ? `${formatSpend(spend)}\n` : "";
     const rows = ledgerRows(events, options);
     if (rows.length === 0)
-        return redact("0 interventions\n");
+        return redact(`${head}0 interventions\n`);
     const lines = [];
     let grand = 0;
     let index = 0;
@@ -90,10 +96,12 @@ export function renderLedger(events, options = {}) {
     }
     lines.push("");
     lines.push(`${grand} ${grand === 1 ? "intervention" : "interventions"}`);
-    return redact(`${lines.join("\n")}\n`);
+    return redact(`${head}${lines.join("\n")}\n`);
 }
 export function renderLedgerFile(kilnDir, options = {}) {
-    return renderLedger(readEvents(kilnDir), options);
+    const events = readEvents(kilnDir);
+    const spend = spendSnapshot(events, options.dailyDollarCap, options.now ?? new Date());
+    return renderLedger(events, { ...options, spend });
 }
 function reasonsFor(event) {
     const reasons = [];

package/dist/project.js

@@ -3,6 +3,7 @@ import { existsSync, readdirSync, readFileSync } from "node:fs";
 import { createRequire } from "node:module";
 import { basename, join, relative, sep } from "node:path";
 import { appendEvent } from "./events.js";
+import { costFromJournal, decideFailure } from "./failure.js";
 import { jobBasename, packageRoot } from "./paths.js";
 /**
  * Project a yak journal into the kiln event log. Called when a run is
@@ -186,25 +187,40 @@ function recordGateResolved(input, event) {
     });
 }
 function recordRunFinished(input, event) {
+    const cost = costFromJournal(input.events);
     if (event.status === "ok") {
         appendEvent(input.kilnDir, {
             job_id: input.jobId,
             run_id: input.runId,
             event: "run.finished",
-            properties: { status: "ok", once: "run.finished" },
+            properties: { status: "ok", tokens: cost.tokens, usd: cost.usd, once: "run.finished" },
         });
         return;
     }
-    if (event.status !== "failed")
+    // A normal failure is classified in recoverRuns, which owns the retry decision.
+    // A cancelled run is not recovered, so the class is recorded here.
+    if (event.status !== "failed" || event.reason !== "cancelled")
         return;
-    const reason = typeof event.reason === "string" ? clip(event.reason) : undefined;
+    const decision = decideFailure({
+        text: "cancelled",
+        operational: false,
+        blocked: true,
+        attempt: 1,
+        maxAttempts: Number.MAX_SAFE_INTEGER,
+        sameFingerprint: 1,
+        maxSameFingerprint: Number.MAX_SAFE_INTEGER,
+    });
     appendEvent(input.kilnDir, {
         job_id: input.jobId,
         run_id: input.runId,
         event: "run.failed",
         properties: {
             status: "failed",
-            ...(reason ? { reason } : {}),
+            reason: "cancelled",
+            class: decision.class,
+            fingerprint: decision.fingerprint,
+            tokens: cost.tokens,
+            usd: cost.usd,
             once: `run.failed:${input.runId}`,
         },
     });

package/dist/rails.js

@@ -0,0 +1,164 @@
+import { appendEvent, readEvents } from "./events.js";
+export function localDay(now) {
+    const year = now.getFullYear();
+    const month = String(now.getMonth() + 1).padStart(2, "0");
+    const day = String(now.getDate()).padStart(2, "0");
+    return `${year}-${month}-${day}`;
+}
+export function isPaused(kilnDir) {
+    return pausedFrom(readEvents(kilnDir, { warn: () => { } }));
+}
+export function pausedFrom(events) {
+    let paused = false;
+    for (const event of events) {
+        if (event.event === "kiln.paused")
+            paused = true;
+        else if (event.event === "kiln.resumed")
+            paused = false;
+    }
+    return paused;
+}
+/** Append a pause event when the factory is running. Already paused is a no-op. */
+export function pauseFactory(kilnDir, now = new Date()) {
+    if (pausedFrom(readEvents(kilnDir, { warn: () => { } })))
+        return false;
+    appendEvent(kilnDir, {
+        job_id: null,
+        event: "kiln.paused",
+        properties: {},
+        ts: now.toISOString(),
+    });
+    return true;
+}
+/** Append a resume event when the factory is paused. Already running is a no-op. */
+export function resumeFactory(kilnDir, now = new Date()) {
+    if (!pausedFrom(readEvents(kilnDir, { warn: () => { } })))
+        return false;
+    appendEvent(kilnDir, {
+        job_id: null,
+        event: "kiln.resumed",
+        properties: {},
+        ts: now.toISOString(),
+    });
+    return true;
+}
+export function spendSnapshot(events, cap, now = new Date()) {
+    const day = localDay(now);
+    let usd = 0;
+    for (const event of events) {
+        if (localDay(new Date(event.ts)) !== day)
+            continue;
+        const amount = dollarCost(event);
+        if (amount === undefined)
+            continue;
+        usd += amount;
+    }
+    const limit = cap === undefined ? null : cap;
+    return {
+        day,
+        usd,
+        cap: limit,
+        blocked: limit !== null && usd >= limit,
+    };
+}
+export function spendFor(kilnDir, cap, now = new Date()) {
+    return spendSnapshot(readEvents(kilnDir, { warn: () => { } }), cap, now);
+}
+/** One event per local day when the dollar cap is blocking new runs. */
+export function recordDailyCap(kilnDir, spend, now = new Date()) {
+    if (!spend.blocked || spend.cap === null)
+        return;
+    appendEvent(kilnDir, {
+        job_id: null,
+        event: "budget.blocked",
+        properties: {
+            day: spend.day,
+            usd: spend.usd,
+            cap: spend.cap,
+            once: `daily-cap:${spend.day}`,
+        },
+        ts: now.toISOString(),
+    });
+}
+export function formatSpend(spend) {
+    const cap = spend.cap === null ? "no cap" : `$${spend.cap.toFixed(2)}`;
+    return `today ${spend.day}  $${spend.usd.toFixed(2)} / ${cap}`;
+}
+export function holdDetail(paused, spend) {
+    if (paused)
+        return "Paused. New runs wait until kiln resume.";
+    if (spend.blocked && spend.cap !== null) {
+        return `Daily cap reached ($${spend.usd.toFixed(2)} of $${spend.cap.toFixed(2)}). New runs wait until the local day rolls over.`;
+    }
+    return undefined;
+}
+/**
+ * Revise notes written since the latest `run.started` for this job.
+ * The next launch appends them to the spec, which is the next prompt's input.
+ */
+export function reviseNotesSinceRun(events, jobId) {
+    const notes = [];
+    for (const event of events) {
+        if (event.job_id !== jobId)
+            continue;
+        if (event.event === "run.started") {
+            notes.length = 0;
+            continue;
+        }
+        if (event.event === "gate.resolved" && event.properties.decision === "revise" && event.properties.note) {
+            notes.push(event.properties.note);
+        }
+    }
+    return notes;
+}
+export function withReviseNotes(spec, notes) {
+    if (notes.length === 0)
+        return spec;
+    return [
+        spec.trimEnd(),
+        "",
+        "## Human note",
+        "",
+        "Apply every note below on this round. A person typed it at the gate.",
+        "",
+        notes.join("\n\n"),
+        "",
+    ].join("\n");
+}
+/** Fingerprint or attempt-cap stop that a later manual retry has not cleared. */
+export function stopReasonFromEvents(events, jobId) {
+    let reason;
+    for (const event of events) {
+        if (event.job_id !== jobId)
+            continue;
+        if (event.event === "job.retried") {
+            reason = undefined;
+            continue;
+        }
+        if (event.event !== "needs-human")
+            continue;
+        if (event.properties.reason === "fingerprint" || event.properties.reason === "attempt-cap") {
+            reason = event.properties.reason;
+        }
+    }
+    return reason;
+}
+/** How many `run.failed` lines for this job already carry `fingerprint`. */
+export function fingerprintCount(events, jobId, fingerprint) {
+    let count = 0;
+    for (const event of events) {
+        if (event.job_id !== jobId || event.event !== "run.failed")
+            continue;
+        if (event.properties.fingerprint === fingerprint)
+            count += 1;
+    }
+    return count;
+}
+function dollarCost(event) {
+    if (event.event !== "run.finished" && event.event !== "run.failed")
+        return undefined;
+    const usd = event.properties.usd;
+    if (typeof usd !== "number" || !Number.isFinite(usd) || usd < 0)
+        return undefined;
+    return usd;
+}

package/dist/reconcile.js

@@ -10,10 +10,32 @@ export function inFlightCount(observation) {
 function inFlight(job, observation) {
     if (job.error || job.feedbackExhausted || job.feedbackAmbiguous)
         return false;
+    // A cap or fingerprint stop is waiting on a person. It does not hold a slot.
+    if (job.retriesExhausted || job.stopReason)
+        return false;
     const state = jobState(job, observation);
     return state === "running" || state === "needs-human" || state === "approved";
 }
 /**
+ * The only job-state transition. First match wins.
+ *
+ * | when | state |
+ * | fix awaiting publish | in-review |
+ * | launching, and the selected run is absent, retired, or a fix | running |
+ * | feedback exhausted or ambiguous, landed, pull request merged | done |
+ * | feedback exhausted or ambiguous, landed, pull request closed | failed |
+ * | feedback exhausted or ambiguous, landed, pull request still open | needs-human |
+ * | selected run not retired | that run: landed, rejected, failed, dead, needs-human, approved, running |
+ * | retired run that landed | done, in-review, or failed from the pull request |
+ * | retired run whose gate rejected | rejected |
+ * | retired suspended run | needs-human |
+ * | attempt cap or fingerprint breaker | needs-human |
+ * | retired, failed, or dead run | failed |
+ * | launching breadcrumb | running |
+ * | dependencies unmet | blocked |
+ * | launch error | failed |
+ * | otherwise | queued |
+ *
  * `deliver=local`: `done` is a finished run whose gate says approve.
  * `deliver=pr`: that landed run is `in-review` while the pull request is open
  * (and in the gap before tick opens it) and `done` only once it has merged.
@@ -21,6 +43,7 @@ function inFlight(job, observation) {
  * a slot and does not unblock dependents. An ok run with no readable
  * decision is `failed`, so a missing artifact cannot unblock the next job.
  * An investigation is `done` on approve and never `in-review`.
+ * A cap or fingerprint stop is `needs-human` and does not hold a slot.
  */
 export function jobState(job, observation) {
     if (job.fixAwaitingPublish)
@@ -42,8 +65,8 @@ export function jobState(job, observation) {
         return "rejected";
     if (job.run?.status === "suspended")
         return "needs-human";
-    if (job.retriesExhausted)
-        return "dead";
+    if ((job.retriesExhausted || job.stopReason) && !job.launching)
+        return "needs-human";
     if (job.runRetired || job.run?.status === "failed" || job.run?.status === "dead")
         return "failed";
     if (job.launching)
@@ -150,7 +173,7 @@ function canStartFix(job) {
 }
 /** A recorded transient failure whose backoff has elapsed. One retry per tick, ahead of a new job. */
 function canRetry(job, observation, now) {
-    if (job.error || job.launching || job.retriesExhausted || job.ledgerError)
+    if (job.error || job.launching || job.retriesExhausted || job.stopReason || job.ledgerError)
         return false;
     if (job.run && !job.runRetired)
         return false;
@@ -192,6 +215,9 @@ export function decide(observation, now = new Date()) {
     }
     if (inFlightCount(observation) >= observation.maxConcurrent)
         return actions;
+    // Pause and the daily dollar cap stop new runs. In-flight work is not touched.
+    if (observation.paused || observation.spendBlocked)
+        return actions;
     const retry = observation.jobs.find((job) => canRetry(job, observation, now));
     if (retry) {
         actions.push({ type: "retry", jobId: retry.jobId, repo: retry.repo });