brainclaw 1.16.0 → 1.18.0

package/dist/core/agentrun-reconciler.js

@@ -38,7 +38,8 @@ import { loadClaim, releaseClaim } from './claims.js';
 import { loadAssignment } from './assignments.js';
 import { createRuntimeEvent } from './events.js';
 import { nowISO } from './ids.js';
-import { readHeartbeat, readLogTail, signalExists, latestActivityMs } from './runtime-signals.js';
+import { readHeartbeat, readLogTail, signalExists, latestActivityMs, readCompletionSignals } from './runtime-signals.js';
+import { findReservationByRunId, evidenceMatchesAttempt, launchGrant, revokeLaunchGrant } from './loops/attempt-reservation.js';
 // ── Constants ──────────────────────────────────────────────────────────────
 /**
  * Minimum age before a run is eligible for reconciliation. Below this, the
@@ -188,9 +189,51 @@ export function collectEvidence(run, cwd, options) {
             fs_activity_age_ms = now - lastFs;
     }
     catch { /* defensive */ }
+    // pln#630 PR2b-c (§13 R3) — read-strict gate. A run owned by a turn-attempt
+    // reservation is completed ONLY on turn-keyed evidence: a `completed` sentinel
+    // body whose turn_id/run_id/nonce match the attempt's CURRENT launch
+    // generation. A bare assignment-keyed presence sentinel — or a stale prior
+    // generation's body — never counts (closes phantom-completion). Legacy runs
+    // (no owning reservation) keep presence-based acceptance.
+    let turn_owned = false;
+    let turn_keyed_completed = false;
+    try {
+        const reservation = findReservationByRunId(run.id, cwd);
+        if (reservation) {
+            turn_owned = true;
+            const bodies = readCompletionSignals(signalRoot, run.assignment_id);
+            // Evidence must be turn-keyed AND carry the RIGHT status (a `.completed`
+            // file whose body says status:'failed' is not completion evidence —
+            // read-strict trusts the body, not the filename, review PR2b-c #C2).
+            const matchedCompleted = bodies.completed !== undefined
+                && bodies.completed.status === 'completed'
+                && evidenceMatchesAttempt(reservation, bodies.completed);
+            const matchedFailed = bodies.failed !== undefined
+                && bodies.failed.status === 'failed'
+                && evidenceMatchesAttempt(reservation, bodies.failed);
+            if (matchedCompleted && matchedFailed) {
+                // §13 R4 — a completed+failed contradiction WITHHOLDS both (never a
+                // silent accept). Conflict-event journaling is deferred to
+                // reconcileTurn (PR3); collectEvidence stays side-effect-free.
+                turn_keyed_completed = false;
+                failed_signal = false;
+            }
+            else {
+                turn_keyed_completed = matchedCompleted;
+                // Read-strict governs FAILURE too for a turn-owned run: override the
+                // presence-based failed_signal so a stale/legacy bare `.failed` marker
+                // cannot phantom-FAIL a healthy generation (symmetric to completed,
+                // review PR2b-c #B). A genuine death with no turn-keyed evidence still
+                // fails via the process-dead + stale path below.
+                failed_signal = matchedFailed;
+            }
+        }
+    }
+    catch { /* defensive — fall back to legacy (non-turn-owned) behavior */ }
     return {
         age_ms, has_post_start_commit, claim_released, assignment_completed, process_alive,
         completed_signal, failed_signal, heartbeat_exists, heartbeat_age_ms, fs_activity_age_ms,
+        turn_owned, turn_keyed_completed,
     };
 }
 /**
@@ -209,7 +252,7 @@ function fsActiveWithin(evidence, windowMs) {
  * Inference only fires after the stale window with no life evidence, so this is
  * conservative. (Loop auto-close on failure is a follow-up.)
  */
-function cascadeReleaseOnFailure(run, actor, cwd) {
+function cascadeReleaseOnFailure(run, actor, cwd, terminalStatus = 'failed') {
     if (!run.claim_id)
         return;
     try {
@@ -219,8 +262,8 @@ function cascadeReleaseOnFailure(run, actor, cwd) {
             createRuntimeEvent({
                 agent: actor,
                 session_id: run.session_id,
-                event_type: 'run_failed',
-                text: `Auto-released claim ${run.claim_id} after run ${run.id} was reconciled to failed (trp#433 GC cascade)`,
+                event_type: terminalStatus === 'cancelled' ? 'run_cancelled' : 'run_failed',
+                text: `Auto-released claim ${run.claim_id} after run ${run.id} was reconciled to ${terminalStatus} (trp#433 GC cascade)`,
                 tags: ['reconciler', 'gc', 'claim-release'],
                 assignment_id: run.assignment_id,
                 run_id: run.id,
@@ -232,6 +275,13 @@ function cascadeReleaseOnFailure(run, actor, cwd) {
     catch { /* best-effort — never let GC break reconciliation */ }
 }
 function anyCompletionEvidence(evidence) {
+    // pln#630 PR2b-c (§13 R3): a turn-owned run is completed ONLY on turn-keyed
+    // evidence — never a bare presence sentinel or an assignment-keyed proxy
+    // (claim_released / assignment_completed / post_start_commit). This is the
+    // read-strict split that closes the stale-sentinel + presence-only
+    // phantom-completion holes. Legacy (non-turn-owned) runs are unchanged.
+    if (evidence.turn_owned)
+        return evidence.turn_keyed_completed;
     return evidence.completed_signal
         || evidence.has_post_start_commit
         || evidence.claim_released
@@ -345,7 +395,7 @@ export function reconcileAgentRun(runId, cwd, options = {}) {
         const evidence = {
             age_ms: 0, has_post_start_commit: false, claim_released: false,
             assignment_completed: false, process_alive: undefined,
-            completed_signal: false, failed_signal: false, heartbeat_exists: false,
+            completed_signal: false, failed_signal: false, heartbeat_exists: false, turn_owned: false, turn_keyed_completed: false,
         };
         return {
             run_id: runId, action: 'no_op', reason: 'run not found', evidence,
@@ -361,6 +411,18 @@ export function reconcileAgentRun(runId, cwd, options = {}) {
             evidence, previous_status, current_status: run.status,
         };
     }
+    // pln#630 PR2c-lease (§4 + R5): a turn-owned run preallocated `created`/
+    // `launching` converges on its DISPATCH/LAUNCH LEASE, not the pid/heartbeat
+    // heuristics below (which assume a worker already spawned and can emit
+    // life-signs). Delegate before the generic evidence path so these runs never
+    // orphan `created`/`launching` forever. Inert for non-turn-owned runs
+    // (evidence.turn_owned=false → fall through to the unchanged generic path).
+    if ((run.status === 'created' || run.status === 'launching') && evidence.turn_owned) {
+        const reservation = findReservationByRunId(run.id, cwd);
+        if (reservation) {
+            return reconcileTurnOwnedPreRunLease(run, reservation, evidence, cwd, options);
+        }
+    }
     const grace = options.healthCheckGraceMs ?? DEFAULT_HEALTH_CHECK_GRACE_MS;
     const stale = options.staleAfterMs ?? DEFAULT_STALE_AFTER_MS;
     const actor = options.actor ?? 'reconciler';
@@ -450,6 +512,84 @@ export function reconcileAgentRun(runId, cwd, options = {}) {
         evidence, previous_status, current_status: run.status,
     };
 }
+/**
+ * pln#630 PR2c-lease (§4 + R5) — converge a TURN-OWNED run still in `created`/
+ * `launching` against its dispatch/launch LEASE, not the pid/heartbeat heuristics
+ * (which presuppose a spawned, life-sign-emitting worker). Reached only from
+ * `reconcileAgentRun`'s turn-owned delegate branch, so it always has the owning
+ * reservation + pre-collected evidence.
+ *
+ * Outcomes (never `completed` — no phantom success from an unspawned/unproven run):
+ *   - turn-keyed completion present → NO-OP (finalization is reconcileTurn's job, PR3);
+ *   - within lease → NO-OP (the worker may yet cross the launch fence / start);
+ *   - past lease + launch grant CROSSED → `failed` / `launch_attempted_unknown`
+ *     (the worker WAS invoked; its outcome is unknowable, so it must not complete);
+ *   - past lease + grant not crossed → `cancelled` / `reserved_never_launched`
+ *     (no launch receipt — nothing ran; matches attemptStatus(revoked)='cancelled').
+ */
+function reconcileTurnOwnedPreRunLease(run, reservation, evidence, cwd, options) {
+    const now = options.nowMs ?? Date.now();
+    const actor = options.actor ?? 'reconciler';
+    const previous_status = run.status;
+    // A genuine turn-keyed completion is NOT an expiry: never fail/cancel a run
+    // that produced accepted evidence. Its finalization (assignment/claim/artifacts)
+    // is reconcileTurn's responsibility (PR3); here we simply decline to expire it.
+    if (evidence.turn_keyed_completed) {
+        return {
+            run_id: run.id, action: 'no_op',
+            reason: 'turn-keyed completion present — defer finalization to reconcileTurn',
+            evidence, previous_status, current_status: run.status,
+        };
+    }
+    // Effective deadline: once armed, the launch lease is the tighter authoritative
+    // bound; before arm, the dispatch lease bounds how long a committed reservation
+    // may wait to spawn. A non-parseable lease cannot expire a run (fail-safe: leave
+    // it for explicit reconcile rather than converge on garbage).
+    const leaseISO = reservation.launch?.lease_deadline ?? reservation.lease_deadline;
+    const leaseMs = Date.parse(leaseISO);
+    if (!Number.isFinite(leaseMs) || now < leaseMs) {
+        return {
+            run_id: run.id, action: 'no_op',
+            reason: Number.isFinite(leaseMs)
+                ? `within lease (deadline ${leaseISO})`
+                : `lease deadline unparseable (${leaseISO}) — cannot expire`,
+            evidence, previous_status, current_status: run.status,
+        };
+    }
+    // Past lease, no accepted completion — the launch-grant status (authoritative,
+    // decision-file reconciled) decides the terminal.
+    const grant = launchGrant(reservation.turn_id, cwd);
+    const crossed = grant?.status === 'crossed';
+    const targetStatus = crossed ? 'failed' : 'cancelled';
+    const action = crossed ? 'inferred_failed' : 'inferred_cancelled';
+    const reason = crossed
+        ? `launch_attempted_unknown: launch grant crossed but run never reached running by lease ${leaseISO} — outcome unknown, never completed`
+        : `reserved_never_launched: no launch receipt by lease ${leaseISO} (grant=${grant?.status ?? 'none'})`;
+    // pln#630 dec#149 R1 (review Finding 2) — make the reserved_never_launched strand REACHABLE
+    // through the WIRED lazy reconciler: revoke the still-armed grant so its authoritative status
+    // becomes `revoked`. That is what lets reconcileTurn's fix-cycle strand detector see the strand
+    // and re-emit, and lets the re-dispatch re-arm a fresh generation (dec#149 R1). Idempotent +
+    // best-effort: a race to crossed/revoked is fine (the decision file governs) and must never
+    // block the run's terminal transition below.
+    if (!crossed && grant?.status === 'armed') {
+        try {
+            revokeLaunchGrant(reservation.turn_id, grant.epoch, 'reserved_never_launched', cwd, actor);
+        }
+        catch { /* raced to crossed/revoked — authoritative status governs */ }
+    }
+    try {
+        transitionAgentRun(run.id, targetStatus, { actor, status_reason: reason }, cwd);
+        cascadeReleaseOnFailure(run, actor, cwd, targetStatus);
+        return { run_id: run.id, action, reason, evidence, previous_status, current_status: targetStatus };
+    }
+    catch (err) {
+        return {
+            run_id: run.id, action: 'no_op',
+            reason: `lease-expiry transition rejected: ${err instanceof Error ? err.message : String(err)}`,
+            evidence, previous_status, current_status: run.status,
+        };
+    }
+}
 /**
  * Read-path reconciliation for a `running` run whose tracked PID reads dead.
  *
@@ -478,7 +618,7 @@ export function reconcileDeadPidRunningAgentRunAtRead(runId, cwd, options = {})
         const evidence = {
             age_ms: 0, has_post_start_commit: false, claim_released: false,
             assignment_completed: false, process_alive: undefined,
-            completed_signal: false, failed_signal: false, heartbeat_exists: false,
+            completed_signal: false, failed_signal: false, heartbeat_exists: false, turn_owned: false, turn_keyed_completed: false,
         };
         return {
             run_id: runId, action: 'no_op', reason: 'run not found', evidence,
@@ -610,6 +750,30 @@ export function sweepDeadPidRunningAgentRunsAtRead(cwd, options = {}) {
         .slice(0, limit);
     return candidates.map((run) => reconcileDeadPidRunningAgentRunAtRead(run.id, cwd, options));
 }
+/**
+ * pln#630 PR2c-lease — lazy read-path sweep for TURN-OWNED runs still in
+ * `created`/`launching`, converging them on their dispatch/launch lease via
+ * `reconcileAgentRun`'s turn-owned delegate. Strictly turn-owned: a legacy
+ * (non-reservation-owned) `created`/`launching` run is SKIPPED here, so this
+ * sweep changes nothing for existing runs and is fully inert until live
+ * turn-owned dispatch (PR2c-b) mints such a run. Errors are isolated per run.
+ */
+export function sweepTurnOwnedPreRunLeaseAtRead(cwd, options = {}) {
+    const results = [];
+    for (const status of ['created', 'launching']) {
+        for (const run of listAgentRuns(cwd, { status })) {
+            // Only turn-owned runs converge on the lease here — legacy runs keep their
+            // existing (non-read-path) reconciliation route untouched.
+            if (!findReservationByRunId(run.id, cwd))
+                continue;
+            try {
+                results.push(reconcileAgentRun(run.id, cwd, options));
+            }
+            catch { /* best-effort — never block reads on reconciliation errors */ }
+        }
+    }
+    return results;
+}
 /**
  * Reconcile every non-terminal agent_run matching `filter`. Useful for
  * batch sweeps from `bclaw_assignment_events` or `brainclaw doctor --dispatch`.
@@ -628,7 +792,7 @@ export function reconcileAllOpenRuns(cwd, filter = {}, options = {}) {
             catch {
                 results.push({
                     run_id: run.id, action: 'no_op', reason: 'reconcile threw — skipped',
-                    evidence: { age_ms: 0, has_post_start_commit: false, claim_released: false, assignment_completed: false, process_alive: undefined, completed_signal: false, failed_signal: false, heartbeat_exists: false },
+                    evidence: { age_ms: 0, has_post_start_commit: false, claim_released: false, assignment_completed: false, process_alive: undefined, completed_signal: false, failed_signal: false, heartbeat_exists: false, turn_owned: false, turn_keyed_completed: false },
                     previous_status: run.status, current_status: run.status,
                 });
             }

package/dist/core/agentruns.js

@@ -101,7 +101,12 @@ export function findLatestAgentRunForAssignment(assignmentId, cwd) {
     })[0];
 }
 const VALID_TRANSITIONS = new Map([
-    ['created', new Set(['launching', 'waiting_input', 'running', 'cancelled', 'interrupted'])],
+    // `failed` added (pln#630 PR2c-lease): a turn-owned run preallocated `created`
+    // whose launch grant CROSSED but crashed before the created→launching
+    // transition is `launch_attempted_unknown` — a failure, reconciled straight
+    // from `created`. (reserved_never_launched, where the grant never crossed,
+    // goes to `cancelled`, already permitted.)
+    ['created', new Set(['launching', 'waiting_input', 'running', 'failed', 'cancelled', 'interrupted'])],
     ['launching', new Set(['waiting_input', 'running', 'failed', 'cancelled', 'timed_out', 'interrupted'])],
     ['waiting_input', new Set(['launching', 'running', 'blocked', 'cancelled', 'timed_out', 'interrupted'])],
     ['running', new Set(['blocked', 'completed', 'failed', 'cancelled', 'timed_out', 'interrupted'])],