@haiyangbg/buildbeat 2.0.0-beta.3 → 2.0.0-beta.5

package/src/v2/runtime/orchestrator.js

@@ -11,7 +11,7 @@
 // anything changed goes APPROVAL_STALE and back to WAITING_HUMAN.
 
 import { createHash } from "node:crypto";
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 
 import { nextStep } from "../engine/workflow.js";
@@ -26,7 +26,10 @@ import {
   releaseLock,
 } from "../workspace/workspace-manager.js";
 import { writeRunRecord } from "./run-record.js";
+import { computeWorkCost } from "./work-cost.js";
 import { assertRequires } from "./env-contract.js";
+import { materialisePrompt } from "./envelope.js";
+import { cacheKey, findReusableEvidence, lastReviewedCandidate, treeHash } from "./cache.js";
 import {
   buildAnchor,
   fingerprintFinding,
@@ -140,11 +143,25 @@ function makeContext(options, ledger, workspace) {
     ledger,
     workspace,
   };
+  // Effective cap per step: run config `budgets:` beats the preset, the
+  // preset beats the global default, and every BUDGET_EXTENDED a human
+  // granted on this ledger adds to it. Real incident: the preset's two
+  // review rounds could not be raised from the run config, and approving
+  // resume-review re-asked the same question forever.
+  context.runBudgets = options.budgets ?? {};
   context.maxAttemptsFor = (step) =>
-    workflow.budgets?.maxAttempts?.[step] ?? maxAttemptsPerStep;
+    (context.runBudgets.maxAttempts?.[step] ??
+      workflow.budgets?.maxAttempts?.[step] ??
+      maxAttemptsPerStep) +
+    (ledger.state.budgetExtensions?.[step] ?? 0) +
+    (ledger.state.steps[step]?.infraAttempts ?? 0);
   context.policies = options.policies ?? [];
   context.allowedPaths = options.allowedPaths ?? null;
   context.reviewTriage = options.reviewTriage ?? null;
+  context.envelope = options.envelope ?? null;
+  context.cache = options.cache ?? {};
+  context.redact = options.redact ?? [];
+  context.adapterConfigs = options.adapterConfigs ?? {};
   context.policyCtx = () => ({
     state: ledger.state,
     candidate: ledger.state.workspaces[workspace.workspaceId]?.candidate ?? null,
@@ -234,6 +251,7 @@ function settleOutcome(context, step, outcome, tree, exec) {
     if ((ledger.state.steps[step]?.attempts ?? 0) >= context.maxAttemptsFor(step)) {
       context.waitHuman(`resume-${step}`, [
         `budget exhausted: ${step} failed its final attempt (maxAttempts=${context.maxAttemptsFor(step)}); not routing to fix`,
+        `approving resume-${step} grants one more attempt; rejecting ends the run`,
       ]);
       return null;
     }
@@ -260,13 +278,13 @@ function settleOutcome(context, step, outcome, tree, exec) {
     },
   });
   if (!to) {
-    ledger.append({
-      type: "RUN_TERMINAL",
-      actor: KERNEL,
-      ts: now(),
-      data: { status: "FAILED", reason: `no transition for (${step}, ${outcome})` },
-    });
-    writeRunRecord({ repoRoot: context.repoRoot, ledger, ts: now() });
+    // A workflow without an edge for this outcome is not a verdict on the
+    // candidate; the person decides whether to rerun the step or end the
+    // run. Terminal FAILED here used to kill runs whose reviewer had merely
+    // errored out.
+    context.waitHuman(`resume-${step}`, [
+      `no transition for (${step}, ${outcome}); approve resume-${step} to rerun the step, reject to end the run`,
+    ]);
     return null;
   }
   ledger.append({
@@ -314,6 +332,30 @@ function drive(context, startStep, { skipBoundaryOnce = false } = {}) {
       return;
     }
 
+    // Work-level review cap (iteration 09): review rounds are counted across
+    // every run of the work, superseded ones included, so "one run per
+    // round" cannot slip past the per-run budget. Reaching the cap is a
+    // human decision (review once more, or merge/close as-is), not a stop.
+    const workCap = context.runBudgets.reviewRoundsPerWork;
+    if (workCap !== undefined && (stepDef.worker === "reviewer" || step === "review")) {
+      const prior = computeWorkCost(context.repoRoot, ledger.state.run.work, {
+        excludeRun: ledger.state.run.id,
+      });
+      const rounds = prior.reviewRounds + (ledger.state.steps[step]?.attempts ?? 0);
+      const allowed = workCap + (ledger.state.workReviewGrants ?? 0);
+      if (rounds >= allowed) {
+        context.waitHuman(
+          `enter-${step}`,
+          [
+            `work review cap reached: ${rounds} review round(s) across ${prior.runs + 1} run(s) of ${ledger.state.run.work} (budgets.reviewRoundsPerWork=${workCap})`,
+            `approve enter-${step} to review once more, or reject and merge/close the work on the evidence you have`,
+          ],
+          "work-review-cap",
+        );
+        return;
+      }
+    }
+
     const preGate = runPolicyGate(context, "pre", step);
     if (preGate.action === "block") {
       ledger.append({
@@ -335,6 +377,7 @@ function drive(context, startStep, { skipBoundaryOnce = false } = {}) {
     if (attempt > maxAttempts) {
       context.waitHuman(`resume-${step}`, [
         `budget exhausted: ${step} would exceed maxAttempts=${maxAttempts}`,
+        `approving resume-${step} grants one more attempt; rejecting ends the run`,
       ]);
       return;
     }
@@ -379,14 +422,76 @@ function drive(context, startStep, { skipBoundaryOnce = false } = {}) {
         }));
       }
     }
-    const exec = adapter.execute({
-      step,
+    // Envelope (C6): the worker's prompt, materialised into the run
+    // directory and handed over as BUILDBEAT_PROMPT / input.envelope.
+    const prompt = materialisePrompt({
+      envelope: context.envelope,
       worker: stepDef.worker,
-      workspacePath: workspace.worktreePath,
-      input,
-      timeoutMs: context.stepTimeoutMs,
-      outputPath,
+      runtimeDir: context.runtimeDir,
+      runId: ledger.state.run.id,
+      step,
+      attempt,
+      repoRoot: context.repoRoot,
     });
+    if (prompt) {
+      input.envelope = { promptRef: prompt.ref, file: prompt.file, digest: context.envelope.digest, vars: context.envelope.vars };
+    }
+    // Incremental review (C7): tell a reviewer which candidate the last
+    // review saw when it is an ancestor of this one.
+    if (stepDef.readonly) {
+      const head = before.head;
+      const lastReviewed = lastReviewedCandidate(context.repoRoot, ledger.state.run.work, workspace.worktreePath, head);
+      if (lastReviewed) {
+        input.lastReviewed = lastReviewed;
+      }
+    }
+    // Verification reuse (C7): same tree + same worker + same envelope that
+    // already passed is referenced, not re-run. Failures always re-run.
+    let stepCacheKey = null;
+    let reused = null;
+    if (context.cache[step] === "tree") {
+      const current = readback(workspace.worktreePath);
+      if (!current.dirty) {
+        stepCacheKey = cacheKey({
+          tree: treeHash(workspace.worktreePath),
+          worker: stepDef.worker,
+          adapterSpec: context.adapterConfigs[stepDef.worker] ?? null,
+          adapterName: adapter.name,
+          envelopeDigest: context.envelope?.digest ?? null,
+        });
+        reused = findReusableEvidence(context.repoRoot, stepCacheKey);
+      }
+    }
+    let exec;
+    if (reused) {
+      const at = now();
+      exec = {
+        adapter: "cache",
+        command: `reuse ${reused.run} ${reused.evidenceRef}`,
+        exitCode: 0,
+        signal: null,
+        stdout: `REUSED: identical tree/worker/envelope already passed in ${reused.run} (${reused.evidenceRef}, ${reused.digest}); not re-run`,
+        stderr: "",
+        timedOut: false,
+        spawnError: null,
+        startedAt: at,
+        finishedAt: at,
+      };
+    } else {
+      exec = adapter.execute({
+        step,
+        worker: stepDef.worker,
+        workspacePath: workspace.worktreePath,
+        input,
+        timeoutMs: context.stepTimeoutMs,
+        outputPath,
+        // Live output streams + marker land in the run directory so `status`
+        // can answer "is it still doing something" while the step runs.
+        liveDir: join(context.runtimeDir, "runs", ledger.state.run.id),
+        promptPath: prompt?.path ?? null,
+        vars: context.envelope?.vars ?? null,
+      });
+    }
     const tree = readback(workspace.worktreePath);
     const evidence = collectCommandEvidence({
       runtimeDir: context.runtimeDir,
@@ -395,6 +500,8 @@ function drive(context, startStep, { skipBoundaryOnce = false } = {}) {
       attempt,
       execResult: exec,
       subject: tree.head,
+      grade: reused ? reused.grade : stepDef.grade ?? "L2",
+      redact: context.redact,
     });
     ledger.append({
       type: "EVIDENCE_RECORDED",
@@ -407,6 +514,8 @@ function drive(context, startStep, { skipBoundaryOnce = false } = {}) {
         digest: evidence.digest,
         status: evidence.status,
         grade: evidence.grade,
+        ...(stepCacheKey ? { cacheKey: stepCacheKey } : {}),
+        ...(reused ? { reused: { run: reused.run, evidenceRef: reused.evidenceRef, digest: reused.digest } } : {}),
       },
     });
 
@@ -459,18 +568,53 @@ function drive(context, startStep, { skipBoundaryOnce = false } = {}) {
     } else {
       stepStatus = "succeeded";
     }
+    // Infrastructure failure vs candidate failure. A timeout, a crash,
+    // garbage output or the worker's own "environment unavailable" signal
+    // (exit 75, EX_TEMPFAIL) says nothing about the candidate: no failure
+    // fingerprint, no fixer, the attempt is refunded, and a human decides
+    // when the backend is back. Real incidents: a worker backend outage
+    // (review exit 97) and non-JSON reviewer output killed five runs in two
+    // days as "no transition for (review, failed)"; PATH, port and host-load
+    // verify failures dispatched fixers five times.
+    const infra =
+      stepStatus === "timeout" ||
+      stepStatus === "crashed" ||
+      stepStatus === "invalid-output" ||
+      (stepStatus === "failed" && exec.exitCode === 75);
     ledger.append({
       type: "STEP_FINISHED",
       actor: KERNEL,
       ts: now(),
-      data: { step, attempt, status: stepStatus, exitCode: exec.exitCode },
+      data: { step, attempt, status: stepStatus, exitCode: exec.exitCode, ...(infra ? { infra: true } : {}) },
     });
     ledger.append({
       type: "BUDGET_CONSUMED",
       actor: KERNEL,
       ts: now(),
-      data: { kind: "attempts", amount: 1, remaining: maxAttempts - attempt },
+      data: {
+        kind: "attempts",
+        amount: infra ? 0 : 1,
+        remaining: context.maxAttemptsFor(step) - attempt,
+      },
     });
+    if (infra) {
+      const cause =
+        stepStatus === "failed"
+          ? "exit 75 (worker reports its environment unavailable)"
+          : stepStatus === "invalid-output"
+            ? "output is not a worker envelope"
+            : stepStatus;
+      context.waitHuman(
+        `resume-${step}`,
+        [
+          `worker infrastructure failure at ${step}: ${cause}; not a candidate defect, attempt not charged`,
+          ...(tree.dirty ? [`the failed worker left the worktree dirty; inspect before rerunning`] : []),
+          `approve resume-${step} to rerun once the backend/environment is back; reject to end the run`,
+        ],
+        "infra",
+      );
+      return;
+    }
 
     let blockingFindings = [];
     if (envelope?.findings) {
@@ -617,6 +761,57 @@ function drive(context, startStep, { skipBoundaryOnce = false } = {}) {
   }
 }
 
+// Supersede (iteration 08, C2): a new Run for the same Work makes any older
+// Run still waiting on a human moot — the human would be approving a
+// candidate nobody intends to merge. Real incident: two WAITING_HUMAN runs
+// sat in a pilot repo's inbox for a day after their successor had already
+// shipped. Only WAITING_HUMAN runs are touched; RUNNING ones are protected
+// by the active lock, terminal ones are already settled.
+function supersedeWaitingRuns(repoRoot, workId, newRunId, now) {
+  const runsDir = join(repoRoot, ".buildbeat", "runtime", "runs");
+  const superseded = [];
+  const skipped = [];
+  if (!existsSync(runsDir)) {
+    return { superseded, skipped };
+  }
+  for (const entry of readdirSync(runsDir).sort()) {
+    if (entry === newRunId) {
+      continue;
+    }
+    const ledgerPath = join(runsDir, entry, "events.jsonl");
+    if (!existsSync(ledgerPath)) {
+      continue;
+    }
+    const ledger = EventLedger.open(ledgerPath);
+    const state = ledger.state;
+    if (ledger.corruption || !state.run || state.run.work !== workId) {
+      continue;
+    }
+    if (state.terminal || state.run.status !== "WAITING_HUMAN") {
+      continue;
+    }
+    try {
+      acquireLock(repoRoot, entry);
+    } catch {
+      skipped.push({ run: entry, reason: "locked by another process" });
+      continue;
+    }
+    try {
+      ledger.append({
+        type: "RUN_TERMINAL",
+        actor: KERNEL,
+        ts: now(),
+        data: { status: "SUPERSEDED", reason: `superseded by ${newRunId} (same work ${workId})` },
+      });
+      writeRunRecord({ repoRoot, ledger, ts: now() });
+      superseded.push(entry);
+    } finally {
+      releaseLock(repoRoot, entry);
+    }
+  }
+  return { superseded, skipped };
+}
+
 function openLedgerFor(repoRoot, runId) {
   const ledgerPath = join(repoRoot, ".buildbeat", "runtime", "runs", runId, "events.jsonl");
   const ledger = EventLedger.open(ledgerPath);
@@ -664,6 +859,10 @@ export function startRun(options) {
     const workspace = createWorkspace({ repoRoot, runId, base });
     const context = makeContext(options, ledger, workspace);
     const now = context.now;
+    const supersession =
+      options.supersede === "off"
+        ? { superseded: [], skipped: [] }
+        : supersedeWaitingRuns(repoRoot, workId, runId, now);
     ledger.append({
       type: "RUN_CREATED",
       actor: KERNEL,
@@ -678,6 +877,8 @@ export function startRun(options) {
         entry,
         planDigest: planDigest ?? "UNVERIFIED",
         intentDigest: intentDigest ?? "UNVERIFIED",
+        ...(supersession.superseded.length > 0 ? { supersedes: supersession.superseded } : {}),
+        ...(options.envelope ? { envelopeDigest: options.envelope.digest, envelopeSource: options.envelope.source } : {}),
       },
     });
     ledger.append({ type: "RUN_STARTED", actor: KERNEL, ts: now(), data: {} });
@@ -694,7 +895,15 @@ export function startRun(options) {
       },
     });
     drive(context, entry);
-    return { runId, workId, ledgerPath, state: ledger.state, workspace };
+    return {
+      runId,
+      workId,
+      ledgerPath,
+      state: ledger.state,
+      workspace,
+      superseded: supersession.superseded,
+      supersedeSkipped: supersession.skipped,
+    };
   });
 }
 
@@ -783,12 +992,51 @@ export function resumeRun(options) {
         });
         return { runId, ledgerPath, state: ledger.state, resumed: true, stale: true, reason: null };
       }
-      const step = resumeStepFromTransition(approval.transition);
+      // An adopted candidate names where to resume (verify, by convention):
+      // the fixer step the request was waiting on has nothing left to do.
+      const step = approval.resumeAt ?? resumeStepFromTransition(approval.transition);
       if (!step || !context.workflow.stepIds.has(step)) {
         throw new OrchestratorError(
           `cannot derive a resume step from approved transition ${approval.transition}`,
         );
       }
+      // An approved resume-<step> on an exhausted budget is the human saying
+      // "one more"; record the grant before driving or the same request
+      // comes straight back (the pilot's app-login runs ended CANCELLED
+      // with their candidates in production because of exactly that).
+      const requestKind = [...ledger.events]
+        .reverse()
+        .find((event) => event.type === "HUMAN_REQUESTED" && event.data.transition === approval.transition)
+        ?.data.kind;
+      if (requestKind === "work-review-cap") {
+        ledger.append({
+          type: "BUDGET_EXTENDED",
+          actor: KERNEL,
+          ts: now(),
+          data: {
+            step,
+            amount: 1,
+            scope: "work",
+            maxAttempts: (context.runBudgets.reviewRoundsPerWork ?? 0) + (state.workReviewGrants ?? 0) + 1,
+            approvalRef: approval.decisionRef,
+          },
+        });
+      } else if (
+        approval.transition.startsWith("resume-") &&
+        (state.steps[step]?.attempts ?? 0) >= context.maxAttemptsFor(step)
+      ) {
+        ledger.append({
+          type: "BUDGET_EXTENDED",
+          actor: KERNEL,
+          ts: now(),
+          data: {
+            step,
+            amount: 1,
+            maxAttempts: context.maxAttemptsFor(step) + 1,
+            approvalRef: approval.decisionRef,
+          },
+        });
+      }
       ledger.append({ type: "RUN_STARTED", actor: KERNEL, ts: now(), data: {} });
       drive(context, step, { skipBoundaryOnce: true });
       return { runId, ledgerPath, state: ledger.state, resumed: true, reason: null };