ai-foreman 1.7.28 → 1.7.29

package/dist/supervisor.js

@@ -1,9 +1,40 @@
 import { processStartIdentity } from "./processIdentity.js";
+import { stopCause } from "rafi-spec";
 import { WorkflowDb } from "./workflowDb.js";
+/**
+ * Retry only the short synchronous persistence operation. The caller owns
+ * the retry budget and remains responsible for deciding whether ownership is
+ * safe after the budget is exhausted.
+ */
+export async function persistWithBoundedRetry(persist, sleep, attempts = 3) {
+    let error;
+    let completedAttempts = 0;
+    for (let attempt = 1; attempt <= Math.max(1, attempts); attempt++) {
+        completedAttempts = attempt;
+        try {
+            persist();
+            return { ok: true, attempts: attempt };
+        }
+        catch (failure) {
+            error = failure;
+            if (attempt < attempts && retryablePersistenceError(failure))
+                await sleep(100 * attempt);
+            else
+                break;
+        }
+    }
+    return { ok: false, attempts: completedAttempts, error };
+}
+function retryablePersistenceError(error) {
+    const code = typeof error === "object" && error !== null && "code" in error ? String(error.code) : "";
+    const message = error instanceof Error ? error.message : String(error);
+    return code === "SQLITE_BUSY" || code === "SQLITE_LOCKED" || /database is (?:busy|locked)/i.test(message);
+}
 /** Durable worker restart loop. The supervisor never owns the project's mutation lease. */
 export class DurableSupervisor {
     options;
     stopRequested = false;
+    requestedStopCause;
     active;
     sleep;
     now;
@@ -12,8 +43,12 @@ export class DurableSupervisor {
         this.sleep = options.sleep ?? ((milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)));
         this.now = options.now ?? (() => new Date());
     }
-    requestStop() {
+    requestStop(cause = "unknown") {
         this.stopRequested = true;
+        const source = cause === "user-sigint" || cause === "parent-sigterm" ? "signal" : cause === "heartbeat-timeout" ? "heartbeat" : cause === "heartbeat-persistence-failure" ? "persistence" : cause === "human-decision" ? "human-decision" : "host";
+        const actor = cause === "user-sigint" ? "user" : cause === "parent-sigterm" ? "parent" : cause === "heartbeat-persistence-failure" ? "database" : cause === "unknown" ? "unknown" : "supervisor";
+        this.requestedStopCause = stopCause({ cause, source, actor, runId: this.options.runId, checkpoint: this.options.checkpoint(), at: this.now().toISOString(), requested: true });
+        this.options.onStopRequested?.(this.requestedStopCause);
         void this.active?.stop();
     }
     async run() {
@@ -64,24 +99,35 @@ export class DurableSupervisor {
             db.putSupervisorState(this.options.runId, state);
         });
         let lastCheckpoint = this.options.checkpoint();
-        const heartbeat = setInterval(() => {
-            state = { ...state, heartbeatAt: this.now().toISOString() };
+        let heartbeatInFlight = false;
+        const persistHeartbeat = async () => {
+            if (heartbeatInFlight || this.stopRequested)
+                return;
+            heartbeatInFlight = true;
             try {
-                saveState();
+                state = { ...state, heartbeatAt: this.now().toISOString() };
+                const persisted = await persistWithBoundedRetry(saveState, this.sleep);
+                if (persisted.ok)
+                    return;
+                this.requestStop("heartbeat-persistence-failure");
+                // Ownership failure remains fail-closed; the explicit cause is
+                // delivered to the worker-exit evidence by supervisedStart.
             }
-            catch {
-                this.requestStop();
+            finally {
+                heartbeatInFlight = false;
             }
-        }, 10_000);
+        };
+        let heartbeatTask;
+        const heartbeat = setInterval(() => { heartbeatTask = persistHeartbeat(); }, this.options.heartbeatIntervalMs ?? 10_000);
         heartbeat.unref();
         try {
             while (!this.stopRequested) {
                 const pending = db.pendingHumanDecisions(this.options.runId);
                 if (pending.some(decision => !this.options.policy.continueIndependentTickets || !decision.interruptionId.startsWith("ticket:"))) {
-                    state = { ...state, status: "waiting_for_human", workerPid: undefined, heartbeatAt: this.now().toISOString() };
+                    state = { ...state, status: "waiting_for_human", workerPid: undefined, heartbeatAt: this.now().toISOString(), stopCause: stopCause({ cause: "human-decision", source: "human-decision", actor: "supervisor", runId: this.options.runId, supervisorGeneration: state.generation, workerGeneration: state.workerGeneration, checkpoint: "waiting-for-human", at: this.now().toISOString(), requested: true }) };
                     saveState();
                     if (db.getRun(this.options.runId)?.status === "running")
-                        db.transitionSupervisor(this.options.runId, state.workerGeneration, { status: "paused", checkpoint: "waiting-for-human", event: "worker_waiting_for_human" });
+                        db.transitionSupervisor(this.options.runId, state.workerGeneration, { status: "paused", checkpoint: "waiting-for-human", event: "worker_waiting_for_human", payload: { cause: state.stopCause } });
                     return { kind: "waiting_for_human" };
                 }
                 const checkpoint = this.options.checkpoint();
@@ -92,22 +138,27 @@ export class DurableSupervisor {
                 this.active = await this.options.spawnWorker(state.workerGeneration);
                 state = { ...state, workerPid: this.active.pid };
                 saveState();
+                const workerPid = state.workerPid;
                 const outcome = await this.active.result;
                 this.active = undefined;
                 state = { ...state, workerPid: undefined, heartbeatAt: this.now().toISOString() };
+                const cause = this.requestedStopCause
+                    ? stopCause({ ...this.requestedStopCause, supervisorGeneration: state.generation, workerGeneration: state.workerGeneration, pid: workerPid, processStart: workerPid ? processStartIdentity(workerPid) : undefined, checkpoint: this.options.checkpoint(), at: this.now().toISOString() })
+                    : stopCause({ cause: "worker-exit", source: "worker", actor: "worker", runId: this.options.runId, supervisorGeneration: state.generation, workerGeneration: state.workerGeneration, pid: workerPid, checkpoint: this.options.checkpoint(), at: this.now().toISOString(), requested: false, detail: outcome.kind === "failed" ? outcome.detail : `worker outcome: ${outcome.kind}` });
+                state = { ...state, stopCause: cause };
                 if (outcome.kind !== "crashed") {
                     if (outcome.kind === "failed" && db.getRun(this.options.runId)?.status === "running") {
                         const eligibility = db.preparationEligibility(this.options.runId);
                         db.transitionSupervisor(this.options.runId, state.workerGeneration, {
                             status: eligibility.eligible ? "failed" : "paused",
                             checkpoint: eligibility.eligible ? "worker-failed" : "worker-reconciliation-required",
-                            event: "worker_exit_recorded", payload: { detail: outcome.detail, reasons: eligibility.reasons },
+                            event: "worker_exit_recorded", payload: { detail: outcome.detail, reasons: eligibility.reasons, cause },
                         });
                     }
                     else if (db.getRun(this.options.runId)?.status === "running")
                         db.transitionSupervisor(this.options.runId, state.workerGeneration, {
                             status: "paused", checkpoint: outcome.kind === "waiting_for_human" && db.pendingHumanDecisions(this.options.runId).length ? "waiting-for-human" : "worker-exited-before-checkpoint",
-                            event: "worker_exit_recorded", payload: { outcome: outcome.kind },
+                            event: "worker_exit_recorded", payload: { outcome: outcome.kind, cause },
                         });
                     const status = outcome.kind === "completed" ? "stopped" : outcome.kind === "waiting_for_human" ? "waiting_for_human" : outcome.kind === "failed" ? "failed" : "stopped";
                     state = { ...state, status };
@@ -118,7 +169,10 @@ export class DurableSupervisor {
                 if (!eligibility.eligible) {
                     state = { ...state, status: "waiting_for_human" };
                     saveState();
-                    db.transitionSupervisor(this.options.runId, state.workerGeneration, { status: "paused", checkpoint: "worker-reconciliation-required", event: "worker_restart_withheld", payload: { reasons: eligibility.reasons } });
+                    const cause = stopCause({ ...state.stopCause, cause: "worker-reconciliation-required", source: "reconciliation", actor: "supervisor", checkpoint: "worker-reconciliation-required", at: this.now().toISOString(), requested: false, detail: eligibility.reasons.join("; ") });
+                    state = { ...state, stopCause: cause };
+                    saveState();
+                    db.transitionSupervisor(this.options.runId, state.workerGeneration, { status: "paused", checkpoint: "worker-reconciliation-required", event: "worker_restart_withheld", payload: { reasons: eligibility.reasons, cause } });
                     return { kind: "waiting_for_human" };
                 }
                 const checkpointLimit = this.options.policy.limits.workerRestartsPerCheckpoint;
@@ -128,7 +182,7 @@ export class DurableSupervisor {
                     saveState();
                     if (db.getRun(this.options.runId)?.status === "running")
                         db.transitionSupervisor(this.options.runId, state.workerGeneration, {
-                            status: "failed", checkpoint: "worker-restart-budget-exhausted", event: "worker_restart_budget_exhausted", payload: { checkpoint, detail: outcome.detail },
+                            status: "failed", checkpoint: "worker-restart-budget-exhausted", event: "worker_restart_budget_exhausted", payload: { checkpoint, detail: outcome.detail, cause: state.stopCause },
                         });
                     return { kind: "failed", detail: `worker restart budget exhausted at ${checkpoint}: ${outcome.detail}` };
                 }
@@ -136,26 +190,29 @@ export class DurableSupervisor {
                 saveState();
                 await this.sleep(Math.min(10_000, 250 * (2 ** Math.min(5, state.checkpointRestarts - 1))));
             }
-            state = { ...state, status: "stopped", stopRequestedAt: this.now().toISOString(), workerPid: undefined };
+            state = { ...state, status: "stopped", stopRequestedAt: this.now().toISOString(), workerPid: undefined, stopCause: state.stopCause ?? this.requestedStopCause ?? stopCause({ cause: "unknown", source: "host", actor: "unknown", runId: this.options.runId, supervisorGeneration: state.generation, workerGeneration: state.workerGeneration, checkpoint: "worker-stopped", at: this.now().toISOString(), requested: true, detail: "Supervisor stopped without a more specific durable cause" }) };
             if (db.getRun(this.options.runId)?.status === "running")
-                db.transitionSupervisor(this.options.runId, state.workerGeneration, { status: "paused", checkpoint: "worker-stopped", event: "supervisor_stop_recorded" });
+                db.transitionSupervisor(this.options.runId, state.workerGeneration, { status: "paused", checkpoint: "worker-stopped", event: "supervisor_stop_recorded", payload: { cause: state.stopCause } });
             saveState();
             return { kind: "stopped" };
         }
         catch (error) {
-            state = { ...state, status: "failed", workerPid: undefined, heartbeatAt: this.now().toISOString() };
+            state = { ...state, status: "failed", workerPid: undefined, heartbeatAt: this.now().toISOString(), stopCause: stopCause({ cause: "launch-failure", source: "launch", actor: "supervisor", runId: this.options.runId, supervisorGeneration: state.generation, workerGeneration: state.workerGeneration, checkpoint: "worker-launch-failed", at: this.now().toISOString(), requested: false, detail: String(error) }) };
             saveState();
             if (db.getRun(this.options.runId)?.status === "running") {
                 const eligibility = db.preparationEligibility(this.options.runId);
+                const cause = stopCause({ cause: "launch-failure", source: "launch", actor: "supervisor", runId: this.options.runId, supervisorGeneration: state.generation, workerGeneration: state.workerGeneration, checkpoint: eligibility.eligible ? "worker-launch-failed" : "worker-reconciliation-required", at: new Date().toISOString(), requested: false, detail: String(error) });
                 db.transitionSupervisor(this.options.runId, state.workerGeneration, { status: eligibility.eligible ? "failed" : "paused",
                     checkpoint: eligibility.eligible ? "worker-launch-failed" : "worker-reconciliation-required", event: "worker_launch_failed",
-                    payload: { detail: String(error), reasons: eligibility.reasons },
+                    payload: { detail: String(error), reasons: eligibility.reasons, cause },
                 });
             }
             throw error;
         }
         finally {
             clearInterval(heartbeat);
+            if (heartbeatTask)
+                await heartbeatTask;
             try {
                 if (ownedAdmission && db.buildAdmission()?.token === ownedAdmission.token)
                     db.releaseBuildAdmission(ownedAdmission);

package/dist/workflowDb.js

@@ -3,7 +3,9 @@ import { canonicalJson } from "./qaProtocolV2.js";
 import { resolveEffectiveQaConfiguration } from "./qaEffectiveConfig.js";
 import { registerGraphDerived, graphDerivedAllowed, currentGraphDerivedAccess } from "./graph/derived.js";
 import { bindCoverageHostFields, parseContractCoverage, validateContractCoverage } from "./qaContractCoverage.js";
-import { applyHostQaComparison, findHostQaCheck, hostComparisonDigestFromCoverage, hostQaComparisonV1Schema } from "./hostQaComparison.js";
+import { findHostQaCheck, hostComparisonDigestFromCoverage, hostQaComparisonV1Schema } from "./hostQaComparison.js";
+import { findGraphQaChecks, hostGraphApplicabilityDigestFromCoverage, hostGraphApplicabilityV1Schema } from "./hostGraphApplicability.js";
+import { applyHostQaAugmentations } from "./hostQaAugmentation.js";
 import { QaPreparationStore, migrateQaPreparation, migrateQaPreparationGuards } from "./qaPreparationStore.js";
 import { GraphStore, migrateGraphStore } from "./graph/storage.js";
 import { reconcileWork } from "./buildWorkReconciliation.js";
@@ -54,6 +56,7 @@ export function heartbeatCurrentWorkflowLease(projectDir, lease, now = new Date(
         throw new Error("workflow recovery database not found");
     const db = new Database(path, { fileMustExist: true });
     db.function("rafi_protocol_v3", () => 1);
+    db.pragma("busy_timeout = 5000");
     registerHandbackWriter(db);
     try {
         const at = now.toISOString();
@@ -144,6 +147,10 @@ export class WorkflowDb {
         registerQaGraphJournalWriter(this.db);
         if (!readinessAccess)
             this.db.pragma("journal_mode = WAL");
+        // Short-lived coordinator/worker writes can overlap with a heartbeat or
+        // another durable transition. Let SQLite wait for the owning transaction
+        // instead of immediately surfacing SQLITE_BUSY to the supervisor.
+        this.db.pragma("busy_timeout = 5000");
         this.db.pragma("synchronous = FULL");
         this.db.pragma("foreign_keys = ON");
         try {
@@ -2086,9 +2093,6 @@ export class WorkflowDb {
             for (const method of check.verification)
                 if (method.equivalentAuthorityId && !store.equivalentResolver()(method.equivalentAuthorityId, contract, check, method, input.sourceStateDigest))
                     throw new Error("Equivalent verification authority is stale or foreign");
-        const errors = validateContractCoverage(contract, coverage, { phase: "qa", sourceDigest: input.sourceStateDigest, inputBasisDigest: input.reviewBasisDigest, attemptId: binding.attemptId, sessionId: binding.sessionId });
-        if (errors.length)
-            throw new Error(errors.join("; "));
         const initialTurns = this.db.prepare("SELECT intent_json FROM qa_turns WHERE run_id=? AND ticket_id=? AND review_number=? AND source_state_digest=? AND review_basis_digest=?").all(input.runId, input.ticketId, attempt.reviewNumber, input.sourceStateDigest, input.reviewBasisDigest);
         const initial = initialTurns.map(row => parseJson(row.intent_json)).find(turn => turn.slot === "initial");
         if (!initial || initial.providerSession.sessionId !== binding.sessionId)
@@ -2110,16 +2114,34 @@ export class WorkflowDb {
             })
             : undefined;
         let retainedCoverage = retainedProviderCoverage;
+        let retainedHostComparison;
         const hostCheck = findHostQaCheck(contract);
         if (hostCheck) {
             const hostDigest = hostComparisonDigestFromCoverage(coverage, hostCheck.id);
             if (!hostDigest)
                 throw new Error("Coverage is missing the host-owned comparison artifact reference");
             const hostResult = hostQaComparisonV1Schema.parse(store.artifact(hostDigest, "host-qa-comparison"));
-            retainedCoverage = retainedProviderCoverage
-                ? applyHostQaComparison(retainedProviderCoverage, contract, hostResult, hostDigest, this.projectDir)
+            retainedHostComparison = { result: hostResult, resultDigest: hostDigest };
+        }
+        const graphCheck = findGraphQaChecks(contract)[0];
+        if (graphCheck) {
+            const graphDigest = hostGraphApplicabilityDigestFromCoverage(coverage, graphCheck.id);
+            if (!graphDigest)
+                throw new Error("Coverage is missing the host-owned Graphify applicability artifact reference");
+            const graphResult = hostGraphApplicabilityV1Schema.parse(store.artifact(graphDigest, "host-graph-applicability"));
+            const retainedHostGraph = { result: graphResult, resultDigest: graphDigest };
+            retainedCoverage = retainedCoverage
+                ? applyHostQaAugmentations({ coverage: retainedCoverage, contract, projectDir: this.projectDir, hostComparison: retainedHostComparison, hostGraph: retainedHostGraph })
                 : undefined;
         }
+        else if (retainedCoverage && retainedHostComparison) {
+            retainedCoverage = applyHostQaAugmentations({ coverage: retainedCoverage, contract, projectDir: this.projectDir, hostComparison: retainedHostComparison });
+        }
+        if (!retainedCoverage)
+            throw new Error("Coverage lacks the retained terminal review receipt and authorized host augmentation");
+        const errors = validateContractCoverage(contract, coverage, { phase: "qa", sourceDigest: input.sourceStateDigest, inputBasisDigest: input.reviewBasisDigest, attemptId: binding.attemptId, sessionId: binding.sessionId });
+        if (errors.length)
+            throw new Error(errors.join("; "));
         if (!receipt || !retainedCoverage || canonicalJson(retainedCoverage) !== canonicalJson(coverage))
             throw new Error("Coverage is not bound to the retained terminal review receipt and authorized host augmentation");
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ai-foreman",
-  "version": "1.7.28",
+  "version": "1.7.29",
   "description": "An orchestrator that keeps Codex CLI / Claude Code builders moving through their step list unattended. Part of Rafi.",
   "license": "Apache-2.0",
   "type": "module",
@@ -311,8 +311,8 @@
     "luxon": "^3.7.2",
     "parse5": "7.2.1",
     "pdfjs-dist": "4.10.38",
-    "rafi-spec": "0.8.17",
-    "special-agents": "0.8.17",
+    "rafi-spec": "0.8.18",
+    "special-agents": "0.8.18",
     "yaml": "^2.5.1",
     "zod": "4.4.3"
   },