@ai-dossier/sched 0.28.0 → 0.30.0

package/dist/engine.js

@@ -74,6 +74,7 @@ const core_1 = require("@ai-dossier/core");
 const batch_dispatch_1 = require("./batch-dispatch");
 const dispatch_1 = require("./dispatch");
 const dispatch_health_1 = require("./dispatch-health");
+const fence_1 = require("./fence");
 const groundtruth_1 = require("./groundtruth");
 const journal_1 = require("./journal");
 const labels_1 = require("./labels");
@@ -185,11 +186,21 @@ function pollUnits(deps, state) {
         if (issue === null || out.has(slot.unit))
             continue;
         const milestone = deps.groundTruth.latestMilestone(issue);
+        // #684: a slot whose branch no milestone ever named — every redispatch,
+        // since `CLEARED_SLOT_FIELDS` nulls `branch` on release and only the
+        // setup milestone carries `branch=`, which is never the LATEST milestone
+        // again — recovers the branch from the issue's setup milestone here, so
+        // the pushed-commit stall signal has a ref to watch. Consulted only while
+        // the slot's own is unknown (one successful read persists it below), and
+        // only for running slots — exited/verifying slots never consume a head.
+        const branch = slot.branch ??
+            (slot.status === 'running' ? (deps.groundTruth.setupInfo(issue)?.branch ?? null) : null);
         out.set(slot.unit, {
             reachable: milestone !== undefined,
             milestone: milestone ?? null,
             closed: deps.groundTruth.issueClosed(issue),
-            head: slot.branch !== null ? deps.groundTruth.branchHead(slot.branch) : null,
+            head: branch !== null ? deps.groundTruth.branchHead(branch) : null,
+            branch,
         });
     }
     return out;
@@ -458,9 +469,15 @@ function spawnAndRecord(ctx, state, unit, slot, opts) {
         return failUnit(ctx, state, unit, `spawn-error: ${err.message}`, opts.failOpts);
     }
     const now = ctx.deps.now();
+    const pidStart = ctx.deps.spawnDeps.processStart(pid);
+    // #683 AC2: the takeover owns a fence from the moment it was fenced in — bind the
+    // fence to this process so later reads can verify the owner is real. Runs BEFORE the
+    // patch/transition below (which don't touch run_id/fence_phase), against the
+    // PRE-spawn slot snapshot that already carries this dispatch's fence coordinates.
+    bindFenceToSpawn(ctx, unit, (0, journal_1.issueOfUnit)(unit) ?? 0, slot, pid, pidStart);
     const patch = {
         pid,
-        pid_start: ctx.deps.spawnDeps.processStart(pid),
+        pid_start: pidStart,
         phase: opts.phase,
         last_progress_at: now.toISOString(),
         // #524: distinct from last_progress_at, which later progress signals
@@ -515,7 +532,12 @@ function spawnUnit(ctx, state, unit) {
         // replaced is refused. A first dispatch is generation 0 and reads as it always did.
         // The tier's own resolved prompt (#527) — falls back to the global
         // dispatch.prompt when the tier has no override.
-        prompt: (0, dispatch_1.buildPrompt)(ctx.dispatch.tiers[entry.tier].prompt, issue, slot.gen),
+        prompt: (0, dispatch_1.buildPrompt)(ctx.dispatch.tiers[entry.tier].prompt, issue, slot.gen, 
+        // #683 AC6: the takeover is told its slot identity alongside the generation —
+        // the same label the fence announced and the bind names (one spelling,
+        // `takeoverLabelFor`). Descriptive only: ownership is decided by run id +
+        // generation, never by matching this label (AC7).
+        slot.gen > 0 ? (0, fence_1.takeoverLabelFor)(slot.id, slot.recoveries) : undefined),
         phase: 'gate',
         ...(slot.gen > 0 ? { journalExtra: { detail: `takeover gen=${slot.gen}` } } : {}),
     });
@@ -547,7 +569,9 @@ function spawnReportAgent(ctx, state, unit) {
         // (#504): a report slot is fenced by the same ladder, and a report agent that did
         // not know its generation would have its `report done` milestone refused by the
         // CLI — recovering forever on a PR that already merged.
-        prompt: (0, dispatch_1.buildReportPrompt)(ctx.dispatch.reportPrompt, issue, entry.pr, entry.cleanup, slot.gen),
+        prompt: (0, dispatch_1.buildReportPrompt)(ctx.dispatch.reportPrompt, issue, entry.pr, entry.cleanup, slot.gen, 
+        // #683 AC6, same rule as the cycle-agent prompt above.
+        slot.gen > 0 ? (0, fence_1.takeoverLabelFor)(slot.id, slot.recoveries) : undefined),
         phase: 'report',
         // Merged-aware: the PR is merged — a report spawn failure never blocks
         // dependents (gating already released at `shipped`).
@@ -660,7 +684,7 @@ const RUN_ID_FOR_ISSUE_RE = /^r-(\d+)-[0-9a-f]{4,}$/;
 const PHASE_TOKEN_RE = /^[a-z][a-z0-9-]{0,31}$/;
 /**
  * Post the takeover record for a unit about to be redispatched (#504 AC1), returning the
- * generation the fence installed — or null when no fence could be written.
+ * fence it installed — or null when no fence could be written.
  *
  * Every null is journaled as `fence-failed` WITH its cause, and every one is DEGRADED
  * rather than fatal:
@@ -699,7 +723,11 @@ function writeFence(ctx, unit, issue, slot, truth) {
     // is preferred over spending the attempt on it.
     const claimed = truth.milestone?.phase ?? '';
     const phase = PHASE_TOKEN_RE.test(claimed) ? claimed : (slot.phase ?? 'gate');
-    const takeover = `slot-${slot.id}-r${slot.recoveries + 1}`;
+    // The label of the dispatch ABOUT to spawn: `recoveries + 1` here, because the
+    // slot's counter is bumped by this recovery's transition right after. Every later
+    // reader of the same dispatch (the spawn-path bind, the exit-path release) derives
+    // the label from the BUMPED counter via `takeoverLabelFor`, so both spellings agree.
+    const takeover = (0, fence_1.takeoverLabelFor)(slot.id, slot.recoveries + 1);
     const outcome = ctx.deps.fencer(issue, run, phase, takeover);
     if (!outcome.ok) {
         return failed(`${outcome.reason} — redispatching at gen=${slot.gen}`);
@@ -708,7 +736,60 @@ function writeFence(ctx, unit, issue, slot, truth) {
         slot: slot.id,
         detail: `${run} gen=${outcome.gen} takeover=${takeover}`,
     });
-    return outcome.gen;
+    return { gen: outcome.gen, run, phase, takeover };
+}
+/**
+ * Bind the slot's fence to the process just spawned for its takeover (#683 AC2).
+ *
+ * Called from `spawnAndRecord` right after the spawn resolved, while pid and
+ * `/proc` start-time are in hand: the bind is what lets every later READER of the trail
+ * (`runstate check`, `post`'s guard) tell a live owner from a ghost whose release was
+ * missed. Best-effort by design — no binder configured (an older engine), no fence on
+ * this slot (`gen === 0`), or a failed post all degrade to an UNBOUND fence, which
+ * reads fail-closed (still fences) exactly as fences did before #683; a failure
+ * journals `fence-bind-failed` so the gap is visible.
+ */
+function bindFenceToSpawn(ctx, unit, issue, slot, pid, pidStart) {
+    const binder = ctx.deps.fenceBinder;
+    if (binder === undefined || slot.gen <= 0 || slot.run_id === null)
+        return;
+    const phase = slot.fence_phase ?? slot.phase ?? 'gate';
+    const takeover = (0, fence_1.takeoverLabelFor)(slot.id, slot.recoveries);
+    const outcome = binder(issue, slot.run_id, phase, takeover, slot.gen, pid, pidStart);
+    journal(ctx, outcome.ok ? 'fence-bound' : 'fence-bind-failed', unit, {
+        slot: slot.id,
+        pid,
+        fence_gen: slot.gen,
+        detail: outcome.ok
+            ? `${slot.run_id} gen=${slot.gen} owner pid=${pid}${pidStart !== null ? ` pid_start=${pidStart}` : ''}`
+            : outcome.reason,
+    });
+}
+/**
+ * Release the slot's fence because its owning dispatch ENDED (#683 AC1) — the
+ * `exit-detected` hook, called only from `reconcileRunning`'s dead-pid branch, the one
+ * place the engine already learns "this run's owner just ended", abnormally or not.
+ *
+ * Best-effort like the bind: no releaser configured, no fence on the slot, or a failed
+ * post all leave the fence on the trail, and successors then fall back to the
+ * bind-based stale-on-read check — the belt-and-suspenders the issue asks for. A
+ * failure journals `fence-release-failed`; a success journals the release so the trail
+ * reads "owner ended, fence lifted" without diffing comments.
+ */
+function releaseFenceOnExit(ctx, unit, issue, slot) {
+    const releaser = ctx.deps.fenceReleaser;
+    if (releaser === undefined || slot.gen <= 0 || slot.run_id === null)
+        return;
+    const phase = slot.fence_phase ?? slot.phase ?? 'gate';
+    const takeover = (0, fence_1.takeoverLabelFor)(slot.id, slot.recoveries);
+    const outcome = releaser(issue, slot.run_id, phase, takeover, slot.gen);
+    journal(ctx, outcome.ok ? 'fence-released' : 'fence-release-failed', unit, {
+        slot: slot.id,
+        fence_gen: slot.gen,
+        detail: outcome.ok
+            ? `${slot.run_id} gen=${slot.gen} released — owner exit detected`
+            : outcome.reason,
+    });
 }
 /**
  * The end of the ladder: the unit has no stronger tier left, so it fails —
@@ -792,6 +873,16 @@ causeEvent, cause, evidence) {
     const extra = {
         ...(typeof evidence.last_tool === 'string' ? { last_tool: evidence.last_tool } : {}),
         ...(prCheck !== 'skipped' ? { pr_check: prCheck } : {}),
+        // #684 AC3: the terminal stall verdict keeps what it checked — last
+        // milestone time, last observed head, head at decision time — so a false
+        // `stall-at-strongest-tier` (the issue:4156 kill, `last_head` never
+        // populated) is diagnosable from the `unit-failed` entry alone. The keys
+        // are present only when the caller supplied them: a stall passes all
+        // three (nulls included — `last_head: null` IS the diagnostic), while a
+        // `verify-incomplete` exit passes none.
+        ...('last_milestone_at' in evidence ? { last_milestone_at: evidence.last_milestone_at } : {}),
+        ...('last_head' in evidence ? { last_head: evidence.last_head } : {}),
+        ...('decision_head' in evidence ? { decision_head: evidence.decision_head } : {}),
     };
     return failUnit(ctx, state, unit, reason, {
         merged: report,
@@ -822,6 +913,9 @@ options = {}) {
     const slot = slotOf(state, unit);
     if (!entry || !slot)
         return state;
+    // #683 AC3: a `deferred-to-owner` cause arrives AFTER the agent's exit was
+    // already detected and recorded by the dead-pid branch of `reconcileRunning`
+    // — same contract as `verify-incomplete` (#524), never re-recorded here.
     killUnitAgent(ctx, state, unit);
     // #524: only the STALL path kills a still-live, not-yet-recorded agent —
     // `causeEvent === 'verify-incomplete'`/`'dispatch-failure'` arrive from
@@ -840,10 +934,11 @@ options = {}) {
     if (escalate) {
         const escalated = report ? (0, dispatch_1.reportTierFor)(slot.recoveries + 1) : (0, dispatch_1.escalateTier)(entry.tier);
         if (slot.recoveries >= types_1.ESCALATION_CAP || escalated === null) {
-            // #629: unreachable with `causeEvent === 'dispatch-failure'` — that
-            // cause always passes `escalate: false` above, so this branch (and the
-            // narrower `causeEvent` type `failOrAdoptOpenPr` declares) is never
-            // actually asked to terminally fail a unit over a provider wall.
+            // #629: unreachable with `causeEvent === 'dispatch-failure'` or
+            // `'deferred-to-owner'` (#683) — both causes always pass
+            // `escalate: false` above, so this branch (and the narrower
+            // `causeEvent` type `failOrAdoptOpenPr` declares) is never actually
+            // asked to terminally fail a unit over a provider wall or a defer.
             return failOrAdoptOpenPr(ctx, state, unit, slot, report, causeEvent, cause, evidence);
         }
         resolvedTier = escalated;
@@ -863,11 +958,17 @@ options = {}) {
     let next = (0, state_1.transitionSlot)(state, slot.id, 'recovering', {
         pid: null,
         recoveries: escalate ? slot.recoveries + 1 : slot.recoveries,
-        gen: fenced ?? slot.gen,
+        gen: fenced !== null ? fenced.gen : slot.gen,
         // Only a fence that actually landed starts the short takeover watch: an
         // unfenced redispatch is already degraded, and cutting its allowance down
         // would compound one failure with another.
         fenced_at: fenced === null ? null : now.toISOString(),
+        // #683: record the fence's trail coordinates so the spawn-path bind and the
+        // exit-path release can name the same record without a trail read. A degraded
+        // fence keeps whatever the slot already recorded (the same run's earlier fence,
+        // if any — the coordinates are per-dispatch, and the spawn that follows an
+        // unfenced recovery is not bound to anything).
+        ...(fenced !== null ? { run_id: fenced.run, fence_phase: fenced.phase } : {}),
     }, now);
     if (!report && escalate) {
         next = {
@@ -982,6 +1083,16 @@ function applyProgressSignals(ctx, state, slot, truth, unit) {
     let next = state;
     let progressed = false;
     let milestoneAdvanced = false;
+    // #684: persist the branch recovered at poll time (the issue's setup
+    // milestone), so the recovery runs once per slot-life instead of every
+    // tick and `last_head` starts flowing for a slot no milestone ever named —
+    // INCLUDING a tick with no milestone at all, which is the long-review
+    // window the pushed-commit signal exists to cover. With the branch known,
+    // the head comparison below is the unchanged pushed-commit signal — a slot
+    // whose branch advanced on the remote within the window is not stalled.
+    if (slot.branch === null && truth.branch !== null) {
+        next = (0, state_1.patchSlot)(next, slot.id, { branch: truth.branch }, now);
+    }
     if (truth.milestone !== null) {
         // Live phase per unit (AC6).
         if (truth.milestone.phase !== slot.phase) {
@@ -1269,10 +1380,15 @@ function journalStaleMilestoneIfIgnored(ctx, state, unit, slot, truth, verifiedC
     });
     return (0, state_1.patchSlot)(state, slot.id, { stale_milestone_ignored_for: slot.spawned_at }, now);
 }
-const NO_DISPATCH_SIGNALS = { lastTool: null, apiError: null };
+const NO_DISPATCH_SIGNALS = {
+    lastTool: null,
+    apiError: null,
+    fenceAbort: null,
+};
 /**
- * Read THIS dispatch's last tool call and API-error classification (#591,
- * #620, #629) from its log slice — a pure, side-effect-free parse, safe to
+ * Read THIS dispatch's last tool call, API-error classification (#591,
+ * #620, #629), and fence-abort evidence (#683 AC3) from its log slice — a
+ * pure, side-effect-free parse, safe to
  * call from any slot status and any number of times: unlike
  * `recordDispatchRunLog` below, it writes nothing to `runs.jsonl`, so it
  * carries no exactly-once constraint. Exists because a `verify-incomplete`/
@@ -1281,12 +1397,23 @@ const NO_DISPATCH_SIGNALS = { lastTool: null, apiError: null };
  * then the slot has moved past `running` and `recordDispatchRunLog`'s guard
  * refuses to re-read, but the dispatch's log file is static once the agent
  * has exited, so re-parsing it here yields the same answer every time.
+ *
+ * `run` is the trail run id the fence check must name to count as THIS
+ * dispatch's checkpoint: the slot's own `run_id` when it holds one, else the
+ * trail's current run (a gen-0 dispatch fenced out by a recovery's fence —
+ * the #4153 shape). Doc text quoting HISTORICAL run ids never matches, which
+ * is what keeps the defer classification from being faked by an agent that
+ * merely read `docs/agent-traps.md`.
  */
-function readDispatchSignalsForSlot(ctx, slot, unit) {
+function readDispatchSignalsForSlot(ctx, slot, unit, run) {
     if (slot.spawned_at === null)
         return NO_DISPATCH_SIGNALS;
     const { content } = dispatchLogSlice(ctx, slot, unit);
-    return { lastTool: (0, core_1.parseLastToolUse)(content), apiError: (0, core_1.parseDispatchApiError)(content) };
+    return {
+        lastTool: (0, core_1.parseLastToolUse)(content),
+        apiError: (0, core_1.parseDispatchApiError)(content),
+        fenceAbort: (0, run_log_1.parseFenceAbort)(content, run),
+    };
 }
 /**
  * THIS dispatch's slice of the per-unit log: path, start offset, contents.
@@ -1318,7 +1445,12 @@ function dispatchLogSlice(ctx, slot, unit) {
  * `running` and actually spawned), not by the call sites' ordering — two of
  * the four reach slots in any non-idle status.
  */
-function recordDispatchRunLog(ctx, state, slot, unit) {
+function recordDispatchRunLog(ctx, state, slot, unit, 
+// #683 AC3: the trail run id a fence-abort marker in THIS dispatch's log slice must
+// name to count as evidence (callers with a polled truth pass its run; a kill whose
+// signals are discarded passes ''). Doc text quotes historical run ids, so without
+// this match an agent that merely READ docs could be classified as having deferred.
+run = '') {
     // Enforce the once-per-dispatch invariant HERE rather than restating it in
     // prose at four call sites (#524 review). `blockTransitiveDependents` and
     // `enterRecovery` reach slots in any non-idle status: a slot already moved
@@ -1375,8 +1507,15 @@ function recordDispatchRunLog(ctx, state, slot, unit) {
     // #591: the last tool this dispatch called, so an unverified exit attributes to a
     // concrete cause (e.g. `Monitor`) without opening the transcript — see `enterRecovery`.
     // #629: whether this SAME dispatch was a confirmed provider API error — see
-    // `completeUnitOrRecover`.
-    return { lastTool: (0, core_1.parseLastToolUse)(logContent), apiError: (0, core_1.parseDispatchApiError)(logContent) };
+    // `completeUnitOrRecover`. #683 AC3: whether it ended because a fence told it to —
+    // `run` names the trail run id the fence check must have answered to; with no run
+    // known, no fence-abort evidence can be this dispatch's own and the parse is skipped.
+    const fenceRun = run !== '' ? run : (slot.run_id ?? '');
+    return {
+        lastTool: (0, core_1.parseLastToolUse)(logContent),
+        apiError: (0, core_1.parseDispatchApiError)(logContent),
+        fenceAbort: (0, run_log_1.parseFenceAbort)(logContent, fenceRun),
+    };
 }
 /** Reconcile one running slot against its polled ground truth. */
 function reconcileRunning(ctx, state, slot, truth, unit) {
@@ -1385,7 +1524,19 @@ function reconcileRunning(ctx, state, slot, truth, unit) {
     // is DETECTED, never trusted as completion (AC2/AC3).
     if (slot.pid !== null && !ctx.deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined)) {
         journal(ctx, 'exit-detected', unit, { pid: slot.pid, slot: slot.id });
-        const signals = recordDispatchRunLog(ctx, state, slot, unit);
+        // #683 AC1: the owning run just ended — release its fence here, BEFORE the
+        // classification below decides between completion and a (re-fencing) recovery, so
+        // no path can leave a dead run's fence governing the trail. A recovery re-fences
+        // at gen+1 right after, which dominates the released generation as before.
+        releaseFenceOnExit(ctx, unit, (0, journal_1.issueOfUnit)(unit) ?? 0, slot);
+        // #683 AC3: the defer classification must recognize THIS dispatch's own
+        // checkpoint output. A gen>0 takeover's slot carries its fence's run_id; a
+        // gen-0 dispatch (re-enqueued fresh, fenced out by an earlier recovery —
+        // the #4153 shape) carries none, so the trail's current run — the run id
+        // the checkpoint instruction tells every dispatch to check under — is the
+        // match key. Wrong-run markers never match (docs quote historical ids),
+        // which is what keeps the classification unfakeable by read text alone.
+        const signals = recordDispatchRunLog(ctx, state, slot, unit, truth.milestone?.run ?? '');
         const exited = (0, state_1.transitionSlot)(state, slot.id, 'exited', {}, now);
         return completeUnitOrRecover(ctx, exited, unit, truth, 'verify-complete', signals);
     }
@@ -1454,6 +1605,14 @@ function reconcileRunning(ctx, state, slot, truth, unit) {
         return enterRecovery(ctx, progress.state, unit, 'stalled', 'stall', truth, {
             active_phase: activePhase,
             stall_timeout_ms: stallTimeoutMs,
+            // #684 AC3: record what the verdict checked, so a false stall is
+            // diagnosable from the journal alone — `last_head: null` here is the
+            // signature of a slot whose pushed-commit signal never had a head to
+            // compare (the issue:4156 failure), distinguishable from a head that
+            // was observed and simply had not moved.
+            last_milestone_at: truth.milestone?.at ?? null,
+            last_head: slot.last_head,
+            decision_head: truth.head,
             ...(slot.fenced_at !== null ? { fenced_at: slot.fenced_at } : {}),
         });
     }
@@ -1608,6 +1767,27 @@ dispatchSignals = NO_DISPATCH_SIGNALS) {
         return enterRecovery(ctx, recorded, unit, 'dispatch-failure', (0, dispatch_health_1.dispatchApiErrorDetail)(resolved.apiError), truth, (0, dispatch_health_1.dispatchApiErrorFields)(resolved.apiError), { escalate: false });
     }
     const suspect = msSinceLastProgress(slot, now) < types_1.SUSPECT_DISPATCH_WINDOW_MS;
+    // #683 AC3: the dispatch's own log carries its supersession-checkpoint
+    // verdict for THIS trail run — the agent checked, found an owner, and
+    // stepped aside. That is a deliberate, CORRECT no-op: never an
+    // `unverified-exit`, never an escalation rung. Classified deterministically
+    // off the CLI's exact output (the #629 lesson — never a heuristic on exit
+    // codes), recorded as a HEALTHY dispatch (no `suspect-dispatch`), and
+    // redispatched at the same tier via the #629 unescalated rail — the fence
+    // `enterRecovery` writes lands at the next generation and its takeover gets
+    // the pid bind, so the #4153 livelock self-heals in one step instead of
+    // climbing to `unverified-exit-at-strongest-tier`. Checked AFTER
+    // `recordDispatchApiError`'s branch: a confirmed provider wall (num_turns 1,
+    // no check output) cannot also be a defer, and its rail owns that shape.
+    if (resolved.fenceAbort !== null) {
+        const recorded = recordDispatchOutcome(ctx, next, unit, slot, false);
+        return enterRecovery(ctx, recorded, unit, 'deferred-to-owner', 'deferred to the fence owner — deliberate supersession-checkpoint abort, no escalation rung consumed', truth, {
+            fence_gen: resolved.fenceAbort.gen,
+            ...(resolved.fenceAbort.takeover !== null
+                ? { fence_takeover: resolved.fenceAbort.takeover }
+                : {}),
+        }, { escalate: false });
+    }
     next = recordDispatchOutcome(ctx, next, unit, slot, suspect);
     return enterRecovery(ctx, next, unit, 'verify-incomplete', 'unverified-exit', truth, {
         observed: truth.milestone
@@ -1664,6 +1844,7 @@ function reconcileSlots(ctx, state, polled) {
             milestone: null,
             closed: false,
             head: null,
+            branch: null,
         };
         switch (slot.status) {
             case 'assigned':
@@ -1683,7 +1864,7 @@ function reconcileSlots(ctx, state, polled) {
                 // whatever `reconcileRunning` saw. Passed as a thunk so the read
                 // happens only on the tick that actually reaches the unverified-exit
                 // decision, not on every tick an outage holds the slot here.
-                next = completeUnitOrRecover(ctx, next, unit, truth, 'verify-complete', () => readDispatchSignalsForSlot(ctx, slot, unit));
+                next = completeUnitOrRecover(ctx, next, unit, truth, 'verify-complete', () => readDispatchSignalsForSlot(ctx, slot, unit, truth.milestone?.run ?? ''));
                 break;
             case 'recovering':
                 next = reconcileRecovering(ctx, next, unit);