@tokenfactory/acc-runner 0.14.0 → 0.18.0

package/dist/task-runner.js

@@ -22,7 +22,7 @@ import { fileURLToPath, pathToFileURL } from "node:url";
 import { execa } from "execa";
 import { loadProfile as defaultLoadProfile } from "./config.js";
 import { normalizeUsage, parseClaudeJson, priceUsdCents, toCliAlias, } from "./cost-pricing.js";
-import { classifyClaudeExit } from "./failure-classifier.js";
+import { classifyClaudeExit, extractResetTime, } from "./failure-classifier.js";
 import { startTaskGithubBudget, endTaskGithubBudget, } from "./github-client.js";
 import { git as defaultGit } from "./git.js";
 import { gh as defaultGh } from "./gh.js";
@@ -33,6 +33,7 @@ import { acquireTaskLock as defaultAcquireTaskLock, TaskLockHeldError, } from ".
 import { prepareTaskWorktree as defaultPrepareTaskWorktree, } from "./runtime/worktree.js";
 import { parseConflictMeta, resolveConflict, MAX_CONFLICT_ATTEMPTS, } from "./runtime/conflict-resolver.js";
 import { parseReworkMeta, renderReworkPrompt, fetchPrHeadRef as defaultFetchPrHeadRef, } from "./runtime/rework.js";
+import { assertResolutionIntegrity, isConflictResolutionTask, summariseFailures, } from "./runtime/resolution-integrity.js";
 const LOG_BATCH_BYTES = 4 * 1024;
 /**
  * v0.6.0 REG-296: build the argv for `claude --print`. Splitting this
@@ -85,6 +86,85 @@ export function expandHomePath(p) {
         return path.join(os.homedir(), p.slice(2));
     return p;
 }
+/**
+ * v0.63 T-63-1: default claude health probe. A silent instant-empty exit is
+ * ambiguous — it can be a broken host OR a momentarily-unavailable claude.
+ * `claude --version` is a cheap, deterministic answer to "is the binary/host
+ * actually broken?": if it returns 0 the host is fine (the exit was
+ * transient → claude_unavailable, retry), and any spawn failure / non-zero
+ * exit / timeout means the binary or its dynamic deps are genuinely broken
+ * (→ env_broken, quarantine). Short timeout so a probe never wedges the run.
+ */
+async function defaultHealthProbe() {
+    try {
+        const res = await execa("claude", ["--version"], {
+            reject: false,
+            timeout: 5_000,
+            env: process.env,
+        });
+        if (res.exitCode === 0) {
+            return { ok: true, detail: (res.stdout || "").trim().slice(0, 200) || "claude --version ok" };
+        }
+        return {
+            ok: false,
+            detail: `claude --version exited ${res.exitCode ?? "signal"}: ${(res.stderr || "").trim().slice(0, 200)}`,
+        };
+    }
+    catch (err) {
+        return { ok: false, detail: `claude --version spawn failed: ${err.message}` };
+    }
+}
+const DEFAULT_CLAUDE_UNAVAILABLE_KNOBS = {
+    windowMs: 15 * 60_000,
+    alertThreshold: 5,
+    backoffBaseMs: 30_000,
+    backoffMaxMs: 15 * 60_000,
+};
+// Process-level state. runTask() is a fresh closure per task, but the module
+// is loaded once, so consecutive task runs in the same `acc-runner watch`
+// process share this — the only place a cross-task streak can live without
+// touching watch.ts (out of this track's file scope). A clean (exit-0) claude
+// run resets it: claude has recovered.
+let claudeUnavailableKnobs = { ...DEFAULT_CLAUDE_UNAVAILABLE_KNOBS };
+let claudeUnavailableHits = [];
+let claudeUnavailableStreak = 0;
+/** v0.63 T-63-1: override the silent-unavailability knobs (used by tests). */
+export function configureClaudeUnavailable(partial) {
+    claudeUnavailableKnobs = { ...claudeUnavailableKnobs, ...partial };
+}
+/** v0.63 T-63-1: reset both the streak and the knobs (used by tests). */
+export function resetClaudeUnavailableState() {
+    claudeUnavailableHits = [];
+    claudeUnavailableStreak = 0;
+    claudeUnavailableKnobs = { ...DEFAULT_CLAUDE_UNAVAILABLE_KNOBS };
+}
+/**
+ * v0.63 T-63-1: record one silent instant-empty exit and compute the response
+ * (exponential backoff + whether the sustained-streak INFRA alert trips).
+ * Exported so the watch process and the unit suite share one definition.
+ */
+export function recordClaudeUnavailable(now) {
+    claudeUnavailableStreak += 1;
+    claudeUnavailableHits.push(now);
+    const cutoff = now - claudeUnavailableKnobs.windowMs;
+    claudeUnavailableHits = claudeUnavailableHits.filter((t) => t >= cutoff);
+    const windowCount = claudeUnavailableHits.length;
+    const alert = windowCount >= claudeUnavailableKnobs.alertThreshold;
+    const expo = claudeUnavailableKnobs.backoffBaseMs * 2 ** (claudeUnavailableStreak - 1);
+    const backoffMs = Math.min(expo, claudeUnavailableKnobs.backoffMaxMs);
+    return {
+        streak: claudeUnavailableStreak,
+        windowCount,
+        alert,
+        backoffMs,
+        resumeAt: new Date(now + backoffMs).toISOString(),
+    };
+}
+/** v0.63 T-63-1: a clean claude run proves the binary recovered — reset. */
+function clearClaudeUnavailableStreak() {
+    claudeUnavailableStreak = 0;
+    claudeUnavailableHits = [];
+}
 // v0.37-A HEAD-LOCK-GUARD: git worktree add/remove/prune can leave stale
 // .git/HEAD.lock or .git/index.lock behind. The virtiofs boundary keeps
 // the sandbox from clearing them, but the host runner process can. Call
@@ -447,6 +527,7 @@ export function runTask(taskId, deps) {
         ((args) => defaultPostRunnerStateMessage(deps.supabase, args));
     const loadProfile = deps.loadProfile ?? defaultLoadProfile;
     const loadPromptPreludeFn = deps.loadPromptPrelude ?? loadPromptPrelude;
+    const healthProbe = deps.healthProbe ?? defaultHealthProbe;
     // Silence the unused-binding lint for `checkoutBase` — v0.11-F supersedes
     // the v0.6.0 REG-301 pre-spawn `git checkout <baseBranch>` with the
     // worktree's `add -B <branch> <path> <baseBranch>` start-point semantics,
@@ -746,6 +827,102 @@ export function runTask(taskId, deps) {
             // return that follows.
             let worktree = null;
             let workdir = repoPath;
+            // v0.57 T-57-2: conflict-resolution integrity guardrail. For tasks whose
+            // job is to resolve a merge conflict (the Claude/worktree path that the
+            // mechanical resolver hands off to when it bails), re-verify the
+            // resolution from the runner's own vantage point before pushing —
+            // zero conflict markers, typecheck green, and no silently-dropped /
+            // regressed version bump. On failure the resolution is REJECTED (no
+            // push, not marked resolved) and the task fails with a diagnostic that
+            // names the offending assertion, so a broken merge is reworked rather
+            // than shipped. A no-op for every non-conflict task.
+            const runResolutionIntegrityGate = async () => {
+                if (!isConflictResolutionTask(task))
+                    return null;
+                const assertIntegrity = deps.assertResolutionIntegrity ?? assertResolutionIntegrity;
+                let result;
+                try {
+                    result = await assertIntegrity(workdir, {
+                        baseRef: integrationBranch,
+                        typecheckCmd: task.typecheck_cmd ?? undefined,
+                        // The #496 regression lived in the acc-runner package; also guard
+                        // the repo root. Each pair is skipped silently when its version is
+                        // unchanged or unreadable, so this never false-fails.
+                        versionTargets: [
+                            { packageJsonPath: "package.json", changelogPath: "CHANGELOG.md" },
+                            {
+                                packageJsonPath: "packages/acc-runner/package.json",
+                                changelogPath: "packages/acc-runner/CHANGELOG.md",
+                            },
+                        ],
+                    });
+                }
+                catch (err) {
+                    // A guardrail that itself crashed must fail closed: reject, never
+                    // wave a resolution through because the check errored.
+                    result = {
+                        ok: false,
+                        checked: [],
+                        failures: [
+                            {
+                                assertion: "typecheck",
+                                detail: `integrity check crashed: ${err.message?.slice(0, 200)}`,
+                            },
+                        ],
+                    };
+                }
+                if (result.ok) {
+                    await appendEvent(deps.supabase, taskId, "resolution_integrity_passed", {
+                        pr_number: task.pr_number ?? null,
+                        checked: result.checked,
+                    });
+                    return null;
+                }
+                const summary = summariseFailures(result.failures);
+                const detail = result.failures
+                    .map((f) => `${f.assertion}: ${f.detail}`)
+                    .join("\n");
+                await appendEvent(deps.supabase, taskId, "resolution_integrity_failed", {
+                    pr_number: task.pr_number ?? null,
+                    failed_assertions: result.failures.map((f) => f.assertion),
+                    files: result.failures.flatMap((f) => f.files ?? []),
+                    detail: detail.slice(-4000),
+                });
+                // Best-effort operator-facing activity event naming the failed assertion.
+                try {
+                    await deps.supabase.rpc("log_activity", {
+                        p_verb: "conflict.integrity_rejected",
+                        p_target_id: String(task.pr_number ?? taskId),
+                        p_payload: {
+                            task_id: taskId,
+                            repo: task.repo ?? null,
+                            failed_assertions: result.failures.map((f) => f.assertion),
+                        },
+                        p_target_type: task.pr_number ? "pr" : "task",
+                    });
+                }
+                catch {
+                    /* best-effort */
+                }
+                await postState_("blocked", "error_context", {
+                    task_id: taskId,
+                    phase: "review",
+                    error_class: "resolution_integrity_failed",
+                    stderr_tail: detail.slice(-4000),
+                });
+                // Reject: do NOT push, do NOT mark resolved. Fail the task so the
+                // resolution is reworked instead of merged broken.
+                await deps.supabase.rpc("transition_task", {
+                    p_task_id: taskId,
+                    p_new_status: "failed",
+                });
+                return {
+                    taskId,
+                    status: "failed",
+                    phase: "push",
+                    error: `conflict-resolution integrity guardrail rejected resolution (${summary})`,
+                };
+            };
             try {
                 // 3. Repo prep. v0.11-F: fetch on the shared clone (worktree
                 // shares the object store) then provision an isolated worktree
@@ -926,32 +1103,35 @@ export function runTask(taskId, deps) {
                         durationMs: Date.now() - spawnedAt,
                     };
                 };
-                // v0.53 T-53-4: quarantine tune. A lone instant-empty exit (the pd36
-                // fast-exit heuristic) is too weak a signal to bench a healthy box —
-                // v0.52 incident (d) quarantined a working machine on ONE 249ms empty
-                // exit. So when the FIRST exit is a *heuristic* env_broken we log it
-                // and retry the spawn once; only TWO consecutive instant-empty exits
-                // are treated as a real env_broken. Definitive failures (dyld/OOM/exit
-                // 127, usage/auth patterns — classified.heuristic falsey) skip the
-                // retry and quarantine immediately, unchanged from v0.48.
+                // v0.53 T-53-4 / v0.63 T-63-1: instant-empty tune. A lone instant-empty
+                // exit (the pd36 fast-exit fingerprint, now classified
+                // `claude_unavailable`) is too weak a signal to act on — v0.52 incident
+                // (d) quarantined a working machine on ONE 249ms empty exit. So when the
+                // FIRST exit is a *heuristic* claude_unavailable we log it and retry the
+                // spawn once; a single blip that the retry recovers stays silent. Only a
+                // CONFIRMED streak (two consecutive instant-empties) is acted on — and
+                // even then v0.63 routes it to pause + backoff + retry (NOT quarantine),
+                // gated by the health probe below. Definitive failures (dyld/OOM/exit
+                // 127, usage/auth patterns — classified.heuristic falsey) skip the retry
+                // and quarantine immediately, unchanged from v0.48.
                 let attempt = await runClaudeOnce();
                 const classifyIfFailed = (a) => a.exitCode === 0
                     ? null
                     : classifyClaudeExit(a.exitCode, a.stderr, a.stdout, a.durationMs);
                 let classified = classifyIfFailed(attempt);
-                let instantEmptyStreak = classified?.class === "env_broken" && classified.heuristic ? 1 : 0;
+                let instantEmptyStreak = classified?.class === "claude_unavailable" && classified.heuristic ? 1 : 0;
                 if (instantEmptyStreak === 1 && !cancelled) {
                     await appendEvent(deps.supabase, taskId, "log", {
                         phase: "claude_exit",
                         stream: "stderr",
-                        event: "env_broken_blip",
+                        event: "claude_unavailable_blip",
                         attempt: 1,
                         duration_ms: attempt.durationMs,
-                        message: "instant empty claude exit — single blip, retrying task once before quarantine",
+                        message: "instant empty claude exit — single blip, retrying task once before pausing",
                     });
                     attempt = await runClaudeOnce();
                     classified = classifyIfFailed(attempt);
-                    if (classified?.class === "env_broken" && classified.heuristic) {
+                    if (classified?.class === "claude_unavailable" && classified.heuristic) {
                         instantEmptyStreak = 2;
                     }
                 }
@@ -974,15 +1154,146 @@ export function runTask(taskId, deps) {
                 }
                 if (attempt.exitCode !== 0) {
                     // v0.48: classify the failure so the quarantine cause is consistent
-                    // across the event, the bus message, and the RunTaskOutcome.
-                    const cls = classified ??
+                    // across the event, the bus message, and the RunTaskOutcome. `let`
+                    // because v0.63's health probe can upgrade a claude_unavailable exit
+                    // to env_broken when the probe proves the host is genuinely broken.
+                    let cls = classified ??
                         classifyClaudeExit(attempt.exitCode, attempt.stderr, attempt.stdout, attempt.durationMs);
-                    // v0.53 T-53-4: a heuristic env_broken that did NOT reach 2
-                    // consecutive instant-empty exits is a transient blip, not a benched
-                    // machine — drop the quarantine cause so watch.ts handles it as a
-                    // normal failed task instead of quarantining the runner.
-                    const softUnconfirmed = cls.class === "env_broken" && cls.heuristic === true && instantEmptyStreak < 2;
-                    const quarantineCause = cls.class !== "task_error" && !softUnconfirmed ? cls.class : undefined;
+                    // v0.56 (T-56-1): capacity exhaustion is NOT a task failure. Do not
+                    // transition the task to 'failed' (that would dead-letter it and burn
+                    // the reviewer/automerge retry cap). Leave it 'running' with its
+                    // signal loop stopped (the outer finally does this) so the
+                    // stale-running sweep returns it to 'queued' with runner_id cleared —
+                    // a lossless requeue for any runner that still has capacity. Surface
+                    // the parsed reset time so watch.ts can pause until the window
+                    // reopens. The locks are released by the outer finally exactly as on
+                    // any other exit path.
+                    if (cls.class === "capacity_exhausted") {
+                        const resumeMs = extractResetTime(`${attempt.stderr}\n${attempt.stdout}`);
+                        const resumeAt = resumeMs !== null ? new Date(resumeMs).toISOString() : null;
+                        await appendEvent(deps.supabase, taskId, "capacity_paused", {
+                            phase: "claude_exit",
+                            exit_code: attempt.exitCode,
+                            exit_class: cls.class,
+                            resume_at: resumeAt,
+                            detail: cls.detail,
+                            runner_id: deps.session.runnerId,
+                        });
+                        return {
+                            taskId,
+                            status: "capacity_paused",
+                            phase: "claude_exit",
+                            exitCode: attempt.exitCode,
+                            error: cls.detail,
+                            capacity_exhausted: true,
+                            resume_at: resumeAt,
+                        };
+                    }
+                    // v0.63 (T-63-1): a bare silent instant-empty exit. Silence alone
+                    // cannot tell "broken host" from "claude momentarily unavailable", so
+                    // run an authoritative `claude --version` health probe as the
+                    // tiebreaker. Probe OK → the host is fine → treat exactly like
+                    // capacity_exhausted (pause + EXPONENTIAL backoff + retry; leave the
+                    // task 'running' for the stale-running sweep; NEVER quarantine, NEVER
+                    // burn the task/review retry caps). Probe FAILS → the binary/host is
+                    // genuinely broken → upgrade to env_broken and fall through to the
+                    // quarantine path (the pd36 contract, now gated by a real signal
+                    // rather than silence). A human INFRA alert fires only after a
+                    // SUSTAINED streak inside a rolling window, never on 2.
+                    if (cls.class === "claude_unavailable") {
+                        const probe = await healthProbe();
+                        await appendEvent(deps.supabase, taskId, "log", {
+                            phase: "claude_exit",
+                            stream: "stderr",
+                            event: "claude_health_probe",
+                            ok: probe.ok,
+                            detail: probe.detail,
+                            consecutive_instant_empty: instantEmptyStreak,
+                        });
+                        if (probe.ok) {
+                            const decision = recordClaudeUnavailable(Date.now());
+                            await appendEvent(deps.supabase, taskId, "claude_unavailable", {
+                                phase: "claude_exit",
+                                exit_code: attempt.exitCode,
+                                exit_class: cls.class,
+                                consecutive_instant_empty: instantEmptyStreak,
+                                duration_ms: attempt.durationMs,
+                                resume_at: decision.resumeAt,
+                                streak: decision.streak,
+                                window_count: decision.windowCount,
+                                detail: cls.detail,
+                                runner_id: deps.session.runnerId,
+                            });
+                            // Best-effort audit trail on the runner timeline.
+                            try {
+                                await deps.supabase.rpc("log_activity", {
+                                    p_verb: "runner.claude_unavailable",
+                                    p_target_id: deps.session.runnerId,
+                                    p_target_type: "runner",
+                                    p_payload: {
+                                        task_id: taskId,
+                                        streak: decision.streak,
+                                        window_count: decision.windowCount,
+                                        backoff_ms: decision.backoffMs,
+                                        resume_at: decision.resumeAt,
+                                        detail: cls.detail,
+                                    },
+                                });
+                            }
+                            catch { /* best-effort */ }
+                            // Escalate to a human only on a SUSTAINED streak — not a single
+                            // pause, not 2. Still NOT a quarantine: the runner stays online,
+                            // paused + retrying, while an operator investigates infra.
+                            if (decision.alert) {
+                                try {
+                                    await deps.supabase.rpc("log_activity", {
+                                        p_verb: "runner.claude_unavailable_infra",
+                                        p_target_id: deps.session.runnerId,
+                                        p_target_type: "runner",
+                                        p_payload: {
+                                            task_id: taskId,
+                                            window_count: decision.windowCount,
+                                            streak: decision.streak,
+                                            detail: cls.detail,
+                                        },
+                                    });
+                                }
+                                catch { /* best-effort */ }
+                                await postState_("blocked", "error_context", {
+                                    task_id: taskId,
+                                    phase: "code",
+                                    error_class: "claude_unavailable_sustained",
+                                    infra_alert: true,
+                                    window_count: decision.windowCount,
+                                    stderr_tail: cls.detail,
+                                });
+                            }
+                            return {
+                                taskId,
+                                status: "capacity_paused",
+                                phase: "claude_exit",
+                                exitCode: attempt.exitCode,
+                                error: `claude silently unavailable (instant-empty exit) — pausing ` +
+                                    `${Math.round(decision.backoffMs / 1000)}s then retrying, not quarantining`,
+                                claude_unavailable: true,
+                                resume_at: decision.resumeAt,
+                            };
+                        }
+                        // Probe failed — the host/binary really is broken. This IS an
+                        // env_broken; fall through to the quarantine path below.
+                        cls = {
+                            exitCode: attempt.exitCode,
+                            class: "env_broken",
+                            detail: `${cls.detail}; claude health probe failed: ${probe.detail}`,
+                        };
+                    }
+                    // v0.48 / v0.63: by this point claude_unavailable has either returned
+                    // (probe OK) or been rewritten to env_broken (probe failed), so any
+                    // non-task_error class is a genuine quarantine cause. consecutive
+                    // mirrors the instant-empty streak for an env_broken upgraded from the
+                    // heuristic (≥1), so quarantine.json still records 1 (definitive) vs 2
+                    // (heuristic-confirmed).
+                    const quarantineCause = cls.class !== "task_error" ? cls.class : undefined;
                     const quarantineConsecutive = quarantineCause === "env_broken" ? Math.max(instantEmptyStreak, 1) : undefined;
                     await appendEvent(deps.supabase, taskId, "error", {
                         phase: "claude_exit",
@@ -1020,6 +1331,10 @@ export function runTask(taskId, deps) {
                 // Success: expose the final attempt's stdout under the name the
                 // downstream report/PR path expects.
                 const capturedStdout = attempt.stdout;
+                // v0.63 (T-63-1): a clean claude run proves the binary recovered — reset
+                // the silent-unavailability streak so the next blip restarts the backoff
+                // ladder from the base and the rolling INFRA-alert window starts fresh.
+                clearClaudeUnavailableStreak();
                 // v0.14-MESSAGING-RUNTIME-WIRE: Claude finished successfully.
                 // testing covers the post-exit window where the runner inspects
                 // the captured stdout (parses the report, builds the cost event)
@@ -1036,6 +1351,10 @@ export function runTask(taskId, deps) {
                 // push failure here: a rework push that didn't land means the
                 // reviewer feedback was NOT addressed, so the task must fail.
                 if (reworkMeta) {
+                    // v0.57 T-57-2: gate conflict-resolution reworks before the push.
+                    const reworkIntegrity = await runResolutionIntegrityGate();
+                    if (reworkIntegrity)
+                        return reworkIntegrity;
                     try {
                         await git.push(workdir, branch);
                     }
@@ -1117,6 +1436,11 @@ export function runTask(taskId, deps) {
                 // Emit the first-commit SLO signal once, before push, so the funnel
                 // sees "coding started" independent of whether the push/PR succeeds.
                 await emitFirstCommit(deps.supabase, git, taskId, workdir, integrationBranch, branch);
+                // v0.57 T-57-2: gate conflict-resolution tasks before the push so a
+                // broken merge never reaches a PR. No-op for every non-conflict task.
+                const integrityGate = await runResolutionIntegrityGate();
+                if (integrityGate)
+                    return integrityGate;
                 // 5. Push + open PR. Both run from the worktree so the operator's
                 // shared clone never has the task branch checked out.
                 let prUrl = "";