@haiyangbg/buildbeat 3.0.0 → 3.1.0

package/src/v2/runtime/orchestrator.js

@@ -59,9 +59,13 @@ const ACTIVE_LOCK = "active-run";
 function lockActive(repoRoot) {
   try {
     acquireLock(repoRoot, ACTIVE_LOCK);
-  } catch {
+  } catch (error) {
+    // A lock whose owner is gone was already reclaimed inside acquireLock;
+    // what reaches here is held (or unreadable), and the owner is the one
+    // fact a blocked caller needs.
+    const detail = error.lock?.detail;
     throw new OrchestratorError(
-      "another run is active in this repository (MVP allows a single active run)",
+      `another run is active in this repository (MVP allows a single active run)${detail ? `; ${detail}` : ""}`,
     );
   }
 }
@@ -149,12 +153,17 @@ function makeContext(options, ledger, workspace) {
   // review rounds could not be raised from the run config, and approving
   // resume-review re-asked the same question forever.
   context.runBudgets = options.budgets ?? {};
-  context.maxAttemptsFor = (step) =>
+  context.budgetLimitFor = (step) =>
     (context.runBudgets.maxAttempts?.[step] ??
       workflow.budgets?.maxAttempts?.[step] ??
       maxAttemptsPerStep) +
-    (ledger.state.budgetExtensions?.[step] ?? 0) +
-    (ledger.state.steps[step]?.infraAttempts ?? 0);
+    (ledger.state.budgetExtensions?.[step] ?? 0);
+  context.maxAttemptsFor = (step) => context.budgetLimitFor(step) +
+    (ledger.state.steps[step]?.infraAttempts ?? 0) +
+    (ledger.state.steps[step]?.freeAttempts ?? 0);
+  // Refunds must not move this ceiling: otherwise a success-only loop has
+  // an ever-growing limit. Human extensions deliberately raise it.
+  context.totalAttemptsFor = (step) => context.budgetLimitFor(step) * 3;
   context.policies = options.policies ?? [];
   context.allowedPaths = options.allowedPaths ?? null;
   context.reviewTriage = options.reviewTriage ?? null;
@@ -179,17 +188,64 @@ function makeContext(options, ledger, workspace) {
       evidenceDigest: lastEvidence?.digest ?? "UNVERIFIED",
     };
   };
-  context.waitHuman = (transition, reasons, kind = "boundary") => {
+  context.waitHuman = (transition, reasons, kind = "boundary", grants = []) => {
     ledger.append({
       type: "HUMAN_REQUESTED",
       actor: KERNEL,
       ts: context.now(),
-      data: { transition, subject: context.subjectNow(), reasons, kind },
+      data: { transition, subject: context.subjectNow(), reasons, kind, ...(grants.length ? { grants } : {}) },
     });
   };
   return context;
 }
 
+// Keep the charged budget distinct from the worker invocation number.
+function budgetUsage(context, step) {
+  const state = context.ledger.state.steps[step];
+  const attempts = state?.attempts ?? 0;
+  const used = attempts - (state?.infraAttempts ?? 0) - (state?.freeAttempts ?? 0);
+  const failures = context.ledger.events.filter((event) =>
+    event.type === "STEP_FINISHED" && event.data.step === step &&
+    event.data.status !== "succeeded" && event.data.infra !== true).length;
+  return { attempts, used, failures, limit: context.budgetLimitFor(step) };
+}
+
+function budgetReasons(context, step, safeguard = false) {
+  const { attempts, used, failures, limit } = budgetUsage(context, step);
+  const charging = context.workflow.steps.find((item) => item.id === step)?.readonly
+    ? "each round is charged" : "successful attempts are not charged";
+  return [
+    safeguard
+      ? `${step} budget exhausted (runaway safeguard): ${attempts}/${context.totalAttemptsFor(step)} total attempt(s), ${failures} real failure(s)`
+      : `${step} budget exhausted: ${used}/${limit} charged attempt(s) used, ${failures} real failure(s) (${charging})`,
+    `approve resume-${step} = ${safeguard ? "raise the safeguard and continue" : "one more attempt"}; reject = end this run and decide the merge on the evidence you have`,
+  ];
+}
+
+function workReviewBudget(context, step) {
+  const stepDef = context.workflow.steps.find((item) => item.id === step);
+  const cap = context.runBudgets.reviewRoundsPerWork;
+  if (cap === undefined || !(step === "review" || stepDef?.worker === "reviewer")) return null;
+  const prior = computeWorkCost(context.repoRoot, context.ledger.state.run.work, {
+    excludeRun: context.ledger.state.run.id,
+  });
+  const rounds = prior.reviewRounds + (context.ledger.state.steps[step]?.attempts ?? 0);
+  const allowed = cap + (context.ledger.state.workReviewGrants ?? 0);
+  return { rounds, allowed, exhausted: rounds >= allowed };
+}
+
+// Every cap the next attempt of `step` would hit. Recorded on the request
+// at stop time, so one approval lifts both layers (run and work) at once.
+function budgetGrants(context, step) {
+  const grants = [];
+  if ((context.ledger.state.steps[step]?.attempts ?? 0) >= context.maxAttemptsFor(step) ||
+      (context.ledger.state.steps[step]?.attempts ?? 0) >= context.totalAttemptsFor(step)) {
+    grants.push({ step, scope: "run" });
+  }
+  if (workReviewBudget(context, step)?.exhausted) grants.push({ step, scope: "work" });
+  return grants;
+}
+
 // Evaluates configured policies of `type` for `appliesTo`, records every
 // verdict as a POLICY_EVALUATED event, and reports what the kernel must do.
 // ADVISORY failures are recorded but never gate (doctor reports the gap).
@@ -249,10 +305,7 @@ function settleOutcome(context, step, outcome, tree, exec) {
     // A step that failed its final attempt can never run again, so routing
     // to fix would spend a worker on a candidate nothing can verify.
     if ((ledger.state.steps[step]?.attempts ?? 0) >= context.maxAttemptsFor(step)) {
-      context.waitHuman(`resume-${step}`, [
-        `budget exhausted: ${step} failed its final attempt (maxAttempts=${context.maxAttemptsFor(step)}); not routing to fix`,
-        `approving resume-${step} grants one more attempt; rejecting ends the run`,
-      ]);
+      context.waitHuman(`resume-${step}`, budgetReasons(context, step), "budget", budgetGrants(context, step));
       return null;
     }
   }
@@ -336,24 +389,19 @@ function drive(context, startStep, { skipBoundaryOnce = false } = {}) {
     // every run of the work, superseded ones included, so "one run per
     // round" cannot slip past the per-run budget. Reaching the cap is a
     // human decision (review once more, or merge/close as-is), not a stop.
-    const workCap = context.runBudgets.reviewRoundsPerWork;
-    if (workCap !== undefined && (stepDef.worker === "reviewer" || step === "review")) {
-      const prior = computeWorkCost(context.repoRoot, ledger.state.run.work, {
-        excludeRun: ledger.state.run.id,
-      });
-      const rounds = prior.reviewRounds + (ledger.state.steps[step]?.attempts ?? 0);
-      const allowed = workCap + (ledger.state.workReviewGrants ?? 0);
-      if (rounds >= allowed) {
-        context.waitHuman(
-          `enter-${step}`,
-          [
-            `work review cap reached: ${rounds} review round(s) across ${prior.runs + 1} run(s) of ${ledger.state.run.work} (budgets.reviewRoundsPerWork=${workCap})`,
-            `approve enter-${step} to review once more, or reject and merge/close the work on the evidence you have`,
-          ],
-          "work-review-cap",
-        );
-        return;
-      }
+    const workBudget = workReviewBudget(context, step);
+    if (workBudget?.exhausted) {
+      const { failures } = budgetUsage(context, step);
+      context.waitHuman(
+        `enter-${step}`,
+        [
+          `${step} work review budget exhausted: ${workBudget.rounds}/${workBudget.allowed} review round(s) across the work, ${failures} real failure(s) in this run`,
+          `approve enter-${step} = one more review round (also lifts this run's review cap if it is spent); reject = end this run and decide the merge on the evidence you have`,
+        ],
+        "work-review-cap",
+        budgetGrants(context, step),
+      );
+      return;
     }
 
     const preGate = runPolicyGate(context, "pre", step);
@@ -375,10 +423,12 @@ function drive(context, startStep, { skipBoundaryOnce = false } = {}) {
     const attempt = (ledger.state.steps[step]?.attempts ?? 0) + 1;
     const maxAttempts = context.maxAttemptsFor(step);
     if (attempt > maxAttempts) {
-      context.waitHuman(`resume-${step}`, [
-        `budget exhausted: ${step} would exceed maxAttempts=${maxAttempts}`,
-        `approving resume-${step} grants one more attempt; rejecting ends the run`,
-      ]);
+      context.waitHuman(`resume-${step}`, budgetReasons(context, step), "budget", budgetGrants(context, step));
+      return;
+    }
+
+    if (attempt > context.totalAttemptsFor(step)) {
+      context.waitHuman(`resume-${step}`, budgetReasons(context, step, true), "budget", budgetGrants(context, step));
       return;
     }
 
@@ -581,11 +631,13 @@ function drive(context, startStep, { skipBoundaryOnce = false } = {}) {
       stepStatus === "crashed" ||
       stepStatus === "invalid-output" ||
       (stepStatus === "failed" && exec.exitCode === 75);
+    const free = stepStatus === "succeeded" && stepDef.readonly !== true;
     ledger.append({
       type: "STEP_FINISHED",
       actor: KERNEL,
       ts: now(),
-      data: { step, attempt, status: stepStatus, exitCode: exec.exitCode, ...(infra ? { infra: true } : {}) },
+      data: { step, attempt, status: stepStatus, exitCode: exec.exitCode,
+        ...(infra ? { infra: true } : {}), ...(free ? { free: true } : {}) },
     });
     ledger.append({
       type: "BUDGET_CONSUMED",
@@ -593,7 +645,7 @@ function drive(context, startStep, { skipBoundaryOnce = false } = {}) {
       ts: now(),
       data: {
         kind: "attempts",
-        amount: infra ? 0 : 1,
+        amount: infra || free ? 0 : 1,
         remaining: context.maxAttemptsFor(step) - attempt,
       },
     });
@@ -736,26 +788,31 @@ function drive(context, startStep, { skipBoundaryOnce = false } = {}) {
       outcome = "succeeded";
     }
     const routed = settleOutcome(context, step, outcome, tree, exec);
-    // Finding triage gate (reviewTriage: required): blocking findings stop
-    // for a human verdict before any fixer runs. Findings are prescriptions,
-    // not facts — auto-routing them to a fixer burned four oscillation
-    // rounds in the deploy campaign before a human stopped the loop.
-    if (routed && outcome === "findings-blocking" && context.reviewTriage === "required") {
-      context.waitHuman(
-        `enter-${routed}`,
-        [
-          `review found ${blockingFindings.length} blocking finding(s); triage before ${routed} runs`,
-          ...blockingFindings
-            .slice(0, 5)
-            .map(
-              (finding) =>
-                `[${finding.severity} ${fingerprintFinding(finding)}] ${finding.summary.slice(0, 200)}`,
-            ),
-          `adjudicate fingerprints (findings adjudicate), then approve enter-${routed} or reject the run`,
-        ],
-        "finding-triage",
-      );
-      return;
+    // Ask before spending fix/verify workers: one approval covers the next
+    // round and both review caps, with the grant bound to this request.
+    if (routed && outcome === "findings-blocking") {
+      const isReview = step === "review" || stepDef.worker === "reviewer";
+      const grants = isReview ? budgetGrants(context, step) : [];
+      const triage = context.reviewTriage === "required";
+      if (triage || grants.length) {
+        const { used, limit, failures } = budgetUsage(context, step);
+        const workBudget = workReviewBudget(context, step);
+        context.waitHuman(
+          `enter-${routed}`,
+          [
+            ...(grants.length ? [
+              `${step} budget exhausted: ${used}/${limit} review round(s) used in this run${workBudget ? `, ${workBudget.rounds}/${workBudget.allowed} across the work` : ""}, ${failures} real failure(s); approve enter-${routed} = fix + re-verify + one more review round; reject = end this run and decide the merge on the evidence you have`,
+            ] : []),
+            `review found ${blockingFindings.length} blocking finding(s); ${triage ? "triage" : "approve another round"} before ${routed} runs`,
+            ...blockingFindings.slice(0, 5).map((finding) =>
+              `[${finding.severity} ${fingerprintFinding(finding)}] ${finding.summary.slice(0, 200)}`),
+            `adjudicate fingerprints (findings adjudicate), then approve enter-${routed} or reject the run`,
+          ],
+          triage ? "finding-triage" : "budget",
+          grants,
+        );
+        return;
+      }
     }
     step = routed;
   }
@@ -797,13 +854,26 @@ function supersedeWaitingRuns(repoRoot, workId, newRunId, now) {
       continue;
     }
     try {
-      ledger.append({
+      // Re-read under the lock: the run may have been approved, resumed or
+      // stopped since the scan, and the scan's read must not be written to.
+      const locked = EventLedger.open(ledgerPath);
+      const fresh = locked.state;
+      if (
+        locked.corruption ||
+        !fresh.run ||
+        fresh.run.work !== workId ||
+        fresh.terminal ||
+        fresh.run.status !== "WAITING_HUMAN"
+      ) {
+        continue;
+      }
+      locked.append({
         type: "RUN_TERMINAL",
         actor: KERNEL,
         ts: now(),
         data: { status: "SUPERSEDED", reason: `superseded by ${newRunId} (same work ${workId})` },
       });
-      writeRunRecord({ repoRoot, ledger, ts: now() });
+      writeRunRecord({ repoRoot, ledger: locked, ts: now() });
       superseded.push(entry);
     } finally {
       releaseLock(repoRoot, entry);
@@ -917,15 +987,8 @@ function resumeStepFromTransition(transition) {
   return null;
 }
 
-export function resumeRun(options) {
-  const { repoRoot, workflow, workflowDigest, runId, planDigest } = options;
-  if (!repoRoot || !runId) {
-    throw new OrchestratorError("repoRoot and runId are required");
-  }
-  if (options.requires?.length) {
-    assertRequires(options.requires);
-  }
-  const { ledger, ledgerPath } = openLedgerFor(repoRoot, runId);
+function resumeTarget(options, { ledger, ledgerPath }) {
+  const { repoRoot, workflowDigest, runId } = options;
   const state = ledger.state;
   if (!state.run) {
     throw new OrchestratorError(`no ledger for run ${runId}; use startRun`);
@@ -936,12 +999,11 @@ export function resumeRun(options) {
     );
   }
   if (state.terminal) {
-    return { runId, ledgerPath, state, resumed: false, reason: "run is terminal" };
+    return { early: { runId, ledgerPath, state, resumed: false, reason: "run is terminal" } };
   }
   if (state.run.status === "WAITING_HUMAN" && state.pendingHuman) {
-    return { runId, ledgerPath, state, resumed: false, reason: "waiting on a human decision" };
+    return { early: { runId, ledgerPath, state, resumed: false, reason: "waiting on a human decision" } };
   }
-
   const bound = state.workspaces[runId];
   if (!bound) {
     throw new OrchestratorError(`run ${runId} has no bound workspace; cannot resume`);
@@ -952,15 +1014,43 @@ export function resumeRun(options) {
       "worktree missing; recovery requires a human decision",
     );
   }
-  const workspace = {
-    workspaceId: runId,
-    repoRoot,
-    worktreePath,
-    branch: bound.branch,
-    base: bound.base,
+  return {
+    workspace: {
+      workspaceId: runId,
+      repoRoot,
+      worktreePath,
+      branch: bound.branch,
+      base: bound.base,
+    },
   };
+}
+
+export function resumeRun(options) {
+  const { repoRoot, runId, planDigest } = options;
+  if (!repoRoot || !runId) {
+    throw new OrchestratorError("repoRoot and runId are required");
+  }
+  if (options.requires?.length) {
+    assertRequires(options.requires);
+  }
+  // The read before the locks only answers early (terminal, waiting on a
+  // human) without taking the repository lock. Everything resume decides is
+  // decided again on a ledger read under the locks: another session may
+  // have approved, resumed or stopped the run in between, and writing
+  // through the earlier read would fork the hash chain.
+  const outside = resumeTarget(options, openLedgerFor(repoRoot, runId));
+  if (outside.early) {
+    return outside.early;
+  }
 
   return withRunLocks(repoRoot, runId, () => {
+    const { ledger, ledgerPath } = openLedgerFor(repoRoot, runId);
+    const target = resumeTarget(options, { ledger, ledgerPath });
+    if (target.early) {
+      return target.early;
+    }
+    const { workspace } = target;
+    const state = ledger.state;
     const context = makeContext(options, ledger, workspace);
     const now = context.now;
 
@@ -1004,36 +1094,73 @@ export function resumeRun(options) {
       // "one more"; record the grant before driving or the same request
       // comes straight back (the pilot's app-login runs ended CANCELLED
       // with their candidates in production because of exactly that).
-      const requestKind = [...ledger.events]
-        .reverse()
-        .find((event) => event.type === "HUMAN_REQUESTED" && event.data.transition === approval.transition)
-        ?.data.kind;
-      if (requestKind === "work-review-cap") {
-        ledger.append({
-          type: "BUDGET_EXTENDED",
-          actor: KERNEL,
-          ts: now(),
-          data: {
-            step,
-            amount: 1,
-            scope: "work",
-            maxAttempts: (context.runBudgets.reviewRoundsPerWork ?? 0) + (state.workReviewGrants ?? 0) + 1,
-            approvalRef: approval.decisionRef,
-          },
-        });
+      const decisionIndex = ledger.events.findIndex((event) =>
+        event.type === "DECISION_RECORDED" && event.data.decisionRef === approval.decisionRef);
+      // APPROVAL_STALE is itself a new request, with no inherited grants.
+      // Restrict lookup to this decision, rather than an older request for
+      // the same transition (or one whose subject was refreshed).
+      const request = ledger.events.slice(0, decisionIndex).reverse().find((event) =>
+        event.type === "HUMAN_REQUESTED" || event.type === "APPROVAL_STALE");
+      const requestData = request?.type === "HUMAN_REQUESTED" &&
+        request.data.transition === approval.transition ? request.data : null;
+      // The grant plan is fixed once, then pinned on the first
+      // BUDGET_EXTENDED it produces; a resume after a crash between two
+      // grants replays that plan instead of re-deriving it from a state the
+      // first grant already raised.
+      const applied = ledger.events.filter((event) =>
+        event.type === "BUDGET_EXTENDED" && event.data.approvalRef === approval.decisionRef);
+      let grants;
+      if (Array.isArray(applied[0]?.data.grants)) {
+        grants = applied[0].data.grants;
       } else if (
-        approval.transition.startsWith("resume-") &&
-        (state.steps[step]?.attempts ?? 0) >= context.maxAttemptsFor(step)
+        Array.isArray(requestData?.grants) &&
+        (canonicalJson(requestData.subject) === canonicalJson(approval.subject) ||
+          // resume --adopt answers this very request with a new candidate by
+          // design; the grant belongs to the round, not to a candidate. Real
+          // incident: the session's hand fix was adopted and the run stopped
+          // again at resume-review for the round the human had just granted.
+          (ledger.events[decisionIndex]?.data.adopted &&
+            approval.subject.planDigest === requestData.subject.planDigest))
       ) {
+        grants = requestData.grants;
+      } else {
+        // Requests without grants: ledgers written before grants existed,
+        // or a request refreshed by APPROVAL_STALE.
+        grants = [];
+        const attempts = state.steps[step]?.attempts ?? 0;
+        const runCapApproval = approval.transition.startsWith("resume-") &&
+          (attempts >= context.maxAttemptsFor(step) || attempts >= context.totalAttemptsFor(step));
+        if (requestData?.kind === "work-review-cap") grants.push({ step, scope: "work" });
+        if (runCapApproval) grants.push({ step, scope: "run" });
+        if (grants.length > 0) grants.push(...budgetGrants(context, step));
+      }
+      const plan = [];
+      const planned = new Set();
+      for (const grant of grants) {
+        const key = `${grant.step}:${grant.scope}`;
+        if (!planned.has(key)) {
+          planned.add(key);
+          plan.push({ step: grant.step, scope: grant.scope });
+        }
+      }
+      const extended = new Set(applied.map((event) => `${event.data.step}:${event.data.scope ?? "run"}`));
+      for (const grant of plan) {
+        const key = `${grant.step}:${grant.scope}`;
+        if (extended.has(key)) continue;
+        extended.add(key);
         ledger.append({
           type: "BUDGET_EXTENDED",
           actor: KERNEL,
           ts: now(),
           data: {
-            step,
+            step: grant.step,
             amount: 1,
-            maxAttempts: context.maxAttemptsFor(step) + 1,
+            ...(grant.scope === "work" ? { scope: "work" } : {}),
+            maxAttempts: grant.scope === "work"
+              ? (context.runBudgets.reviewRoundsPerWork ?? 0) + (ledger.state.workReviewGrants ?? 0) + 1
+              : context.maxAttemptsFor(grant.step) + 1,
             approvalRef: approval.decisionRef,
+            grants: plan,
           },
         });
       }

package/src/v2/storage/event-ledger.js

@@ -4,7 +4,7 @@
 // further appends — recovery is a human decision, never a silent repair.
 
 import { createHash } from "node:crypto";
-import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
+import { appendFileSync, existsSync, mkdirSync, readFileSync, statSync } from "node:fs";
 import { dirname } from "node:path";
 
 import { ENVELOPE_VERSION, GENESIS_DIGEST, validateEventInput } from "../domain/event-registry.js";
@@ -49,6 +49,9 @@ export class EventLedger {
   #run = null;
   #work = null;
   #reducer;
+  // Bytes of the file this instance has read or written. A writer whose
+  // count no longer matches the file is stale: someone else appended since.
+  #size = 0;
 
   constructor(filePath, reducer = RUN_REDUCER) {
     this.path = filePath;
@@ -72,7 +75,10 @@ export class EventLedger {
   }
 
   #load() {
-    const lines = readFileSync(this.path, "utf8")
+    const raw = readFileSync(this.path);
+    this.#size = raw.length;
+    const lines = raw
+      .toString("utf8")
       .split("\n")
       .filter((line) => line.length > 0);
     for (const [index, line] of lines.entries()) {
@@ -142,8 +148,21 @@ export class EventLedger {
     };
     event.digest = eventDigest(event);
     const nextState = this.#reducer.applyEvent(this.state, event);
+    // Guard against a stale writer: every event carries seq and prev from
+    // this instance's memory, so appending after another writer would fork
+    // the hash chain and leave the ledger corrupted for good. Refuse
+    // instead; re-reading and retrying is always safe. Writers re-read under
+    // the run lock, so this only fires on a writer that forgot to.
+    const onDisk = existsSync(this.path) ? statSync(this.path).size : 0;
+    if (onDisk !== this.#size) {
+      throw new LedgerError(
+        `ledger for ${runId} changed on disk since it was read (another writer); re-read it and retry`,
+      );
+    }
+    const line = `${JSON.stringify(event)}\n`;
     mkdirSync(dirname(this.path), { recursive: true });
-    appendFileSync(this.path, `${JSON.stringify(event)}\n`, "utf8");
+    appendFileSync(this.path, line, "utf8");
+    this.#size += Buffer.byteLength(line, "utf8");
     this.events.push(event);
     this.state = nextState;
     this.lastDigest = event.digest;