@ai-dossier/sched 0.22.2 → 0.23.0

package/dist/engine.js

@@ -121,17 +121,6 @@ function slotOf(state, unit) {
 function msSinceLastProgress(slot, now) {
     return now.getTime() - (slot.last_progress_at ? Date.parse(slot.last_progress_at) : 0);
 }
-/**
- * Metadata patch on a slot WITHOUT a status transition (pid/phase/branch/
- * last_head/last_progress are data, not machine states — RFC-0001 §D.3 keeps
- * them alongside the status, and the transition tables stay pure).
- */
-function patchSlot(state, slotId, patch, now) {
-    return {
-        ...state,
-        slots: state.slots.map((s) => s.id === slotId ? { ...s, ...patch, updated_at: now.toISOString() } : s),
-    };
-}
 /**
  * Walk a slot to `idle` one declared edge per iteration; `step` picks the
  * next status. Returns the released slot id — non-null only when the walk
@@ -479,7 +468,7 @@ function spawnAndRecord(ctx, state, unit, slot, opts) {
     };
     const next = slot.status === 'assigned' || slot.status === 'recovering'
         ? (0, state_1.transitionSlot)(state, slot.id, 'running', patch, now)
-        : patchSlot(state, slot.id, patch, now);
+        : (0, state_1.patchSlot)(state, slot.id, patch, now);
     journal(ctx, 'spawned', unit, {
         pid,
         tier: opts.tier,
@@ -718,6 +707,80 @@ function writeFence(ctx, unit, issue, slot, truth) {
     });
     return outcome.gen;
 }
+/**
+ * The end of the ladder: the unit has no stronger tier left, so it fails —
+ * UNLESS ground truth says its branch already carries an open PR the
+ * milestone trail never recorded (#596), in which case it parks and the
+ * watcher owns it. Extracted from `enterRecovery` so the terminal decision,
+ * which now has four distinct outcomes, is readable on its own.
+ *
+ * Fails closed (AC5): a report slot (no branch of its own — AC7), a stall
+ * (a hung agent never had the chance to open anything), an unknown branch,
+ * or an unreachable lookup all take the terminal path. Only a confirmed open
+ * PR number parks (AC4). Which of those four it was is journaled as
+ * `pr_check`, because "we checked and there was none" and "`gh` was down and
+ * we failed a unit whose PR may have been mergeable" are the same
+ * `unit-failed` line otherwise — and that indistinguishability is exactly
+ * what cost imboard-monorepo#3999 (docs/agent-traps.md).
+ */
+function failOrAdoptOpenPr(ctx, state, unit, slot, report, causeEvent, cause, evidence) {
+    let prCheck = 'skipped';
+    if (!report && causeEvent === 'verify-incomplete') {
+        if (slot.branch === null) {
+            prCheck = 'no-branch';
+        }
+        else {
+            const openPr = ctx.deps.groundTruth.openPrForBranch(slot.branch);
+            if (typeof openPr === 'number') {
+                // A unit that demonstrably opened a PR is proof this dispatch was
+                // healthy, whatever its exit looked like — reset the suspect streak
+                // exactly like the milestone-verified park in `completeUnitOrRecover`
+                // (#505), or a rescued unit still counts toward a false-positive
+                // `dispatch-unhealthy` pause. `completeUnitOrRecover` has already
+                // journaled this tick's `suspect-dispatch`; the reset following it is
+                // the correct record of a classification made on incomplete evidence
+                // and then retracted.
+                return parkUnit(ctx, recordDispatchOutcome(ctx, state, unit, slot, false), unit, openPr, {
+                    detail: 'unverified-exit-recovered-open-pr',
+                    branch: slot.branch,
+                });
+            }
+            prCheck = openPr === undefined ? 'unreachable' : 'none';
+            if (prCheck === 'unreachable') {
+                journal(ctx, 'ground-truth-unreachable', unit, {
+                    slot: slot.id,
+                    detail: `open-PR check for branch ${slot.branch} unreachable — failing closed; re-check with \`gh pr list --head ${slot.branch} --state open\` before writing this unit off`,
+                });
+            }
+        }
+    }
+    // Cap reached (2 escalations) or already at the strongest tier — the
+    // designed signal that a human, not a stronger model, is next.
+    const reason = report
+        ? 'report-escalation-cap'
+        : slot.recoveries >= types_1.ESCALATION_CAP
+            ? 'escalation-cap'
+            : `${cause}-at-strongest-tier`;
+    // #591/#620: surface `last_tool` on the terminal `unit-failed` journal
+    // entry too — the `evidence` object already carries it for the non-terminal
+    // `verify-incomplete` journal event, because BOTH rails that reach here
+    // with `causeEvent === 'verify-incomplete'` thread it through
+    // `completeUnitOrRecover`: `reconcileRunning`'s dead-pid rail, from
+    // `recordDispatchRunLog`'s own read, and (since #620) `reconcileSlots`'
+    // `exited`/`verifying` rail, from `readLastToolForSlot`. The two agree —
+    // the dispatch log is static once the agent has exited — so the tool name
+    // survives regardless of which tick reaches this decision. It stays
+    // OPTIONAL: a slice with no parseable `tool_use` yields null, hence the
+    // guard below.
+    const extra = {
+        ...(typeof evidence.last_tool === 'string' ? { last_tool: evidence.last_tool } : {}),
+        ...(prCheck !== 'skipped' ? { pr_check: prCheck } : {}),
+    };
+    return failUnit(ctx, state, unit, reason, {
+        merged: report,
+        extra: Object.keys(extra).length > 0 ? extra : undefined,
+    });
+}
 /**
  * The recovery decision for a unit that must be redispatched one tier
  * stronger (stall or unverified exit, AC4). At the escalation cap or the
@@ -751,18 +814,7 @@ function enterRecovery(ctx, state, unit, causeEvent, cause, truth, evidence = {}
     const report = (0, state_1.isReportSlot)(slot);
     const nextTier = report ? (0, dispatch_1.reportTierFor)(slot.recoveries + 1) : (0, dispatch_1.escalateTier)(entry.tier);
     if (slot.recoveries >= types_1.ESCALATION_CAP || nextTier === null) {
-        // Cap reached (2 escalations) or already at the strongest tier — the
-        // designed signal that a human, not a stronger model, is next.
-        const reason = report
-            ? 'report-escalation-cap'
-            : slot.recoveries >= types_1.ESCALATION_CAP
-                ? 'escalation-cap'
-                : `${cause}-at-strongest-tier`;
-        // #591: surface `last_tool` on the terminal `unit-failed` journal entry too — the
-        // `evidence` object already carries it (added above by `completeUnitOrRecover`)
-        // for the non-terminal `verify-incomplete` journal event.
-        const extra = typeof evidence.last_tool === 'string' ? { last_tool: evidence.last_tool } : undefined;
-        return failUnit(ctx, state, unit, reason, { merged: report, extra });
+        return failOrAdoptOpenPr(ctx, state, unit, slot, report, causeEvent, cause, evidence);
     }
     // Fence BEFORE the respawn (#504 AC1/AC4): `killUnitAgent` above only reaches a pid
     // this process can see and signal, and #472 proved that is not the same as a dead
@@ -843,23 +895,31 @@ function completeUnit(ctx, state, unit, via) {
     return withEntry;
 }
 /**
- * Park a unit whose agent exited after parking its PR on auto-merge (#468):
- * the exit is VERIFIED (the ship phase's `awaiting-merge` milestone with
- * `pr=`), entry → parked (pr recorded), slot released — a waiting unit
- * consumes zero slots (AC5) and the watcher owns it from here.
+ * Park a unit whose agent exited having already produced an open PR — the
+ * ship phase's `awaiting-merge` milestone with `pr=` (#468), or (#596) a
+ * terminal unverified exit whose branch ground truth found one the milestone
+ * trail never recorded: entry → parked (pr recorded), slot released — a
+ * waiting unit consumes zero slots (AC5) and the watcher owns it from here.
+ * `extra.detail` names WHICH path adopted the PR, so an operator reading
+ * `pr-parked` alone can tell a milestone-verified park from a
+ * recovery-adopted one (#596 AC3); `extra.branch` names the ref the PR was
+ * found on, without which "why is this unit parked on PR N?" has nothing to
+ * correlate against — `slot.branch` is captured from whichever milestone
+ * first carried one and is not re-derived between recovery redispatches.
+ *
+ * Typed rather than a `Record<string, unknown>` bag: both keys are declared
+ * `JournalEvent` fields, and routing them through an open record would
+ * discard the excess-property check they exist to get.
  */
-function parkUnit(ctx, state, unit, milestone) {
+function parkUnit(ctx, state, unit, pr, extra = {}) {
     const issue = (0, journal_1.issueOfUnit)(unit);
     if (issue === null)
         return state;
     const now = ctx.deps.now();
-    const pr = (0, groundtruth_1.prOfMilestone)(milestone);
-    if (pr === null)
-        return state; // isParkedMilestone guarantees this
     const walked = walkSlotToIdle(state, unit, now, stepVerifiedExitToIdle);
     let next = walked.state;
     next = (0, state_1.transitionIssue)(next, issue, 'parked', { pr }, now);
-    journal(ctx, 'pr-parked', unit, { pr });
+    journal(ctx, 'pr-parked', unit, { pr, ...extra });
     ctx.result.parked.push(unit);
     journalSlotReleased(ctx, unit, walked.releasedSlotId, 'parked');
     return next;
@@ -873,13 +933,13 @@ function applyProgressSignals(ctx, state, slot, truth, unit) {
     if (truth.milestone !== null) {
         // Live phase per unit (AC6).
         if (truth.milestone.phase !== slot.phase) {
-            next = patchSlot(next, slot.id, { phase: truth.milestone.phase }, now);
+            next = (0, state_1.patchSlot)(next, slot.id, { phase: truth.milestone.phase }, now);
             journal(ctx, 'phase-updated', unit, { phase: truth.milestone.phase, slot: slot.id });
         }
         // The setup milestone carries the branch name — capture it once so the
         // pushed-commit stall signal can watch the remote head.
         if (slot.branch === null && typeof truth.milestone.keys.branch === 'string') {
-            next = patchSlot(next, slot.id, { branch: truth.milestone.keys.branch }, now);
+            next = (0, state_1.patchSlot)(next, slot.id, { branch: truth.milestone.keys.branch }, now);
         }
         if (Date.parse(truth.milestone.at) > Date.parse(slot.last_progress_at ?? '')) {
             progressed = true;
@@ -887,12 +947,12 @@ function applyProgressSignals(ctx, state, slot, truth, unit) {
     }
     if (truth.head !== null && truth.head !== slot.last_head) {
         progressed = true;
-        next = patchSlot(next, slot.id, { last_head: truth.head }, now);
+        next = (0, state_1.patchSlot)(next, slot.id, { last_head: truth.head }, now);
     }
     if (progressed) {
         // The takeover is demonstrably alive, so the short fence watch has done its job and
         // the phase's ordinary stall allowance takes over from here (#504 AC4).
-        next = patchSlot(next, slot.id, {
+        next = (0, state_1.patchSlot)(next, slot.id, {
             last_progress_at: now.toISOString(),
             ...(slot.fenced_at !== null ? { fenced_at: null } : {}),
         }, now);
@@ -927,21 +987,77 @@ function effectiveClosedSignal(slot, truth) {
  * when the raw milestone WOULD have completed the unit under the old,
  * unfenced rule (`isVerifiedComplete(milestone, false)`, `dispatchedAt`
  * omitted) but the fenced check just rejected it — i.e. a `report/done`
- * milestone that predates this dispatch's `spawned_at`. Journaled per-tick
- * while the condition holds, same convention as `ground-truth-unreachable`.
+ * milestone that predates this dispatch's `spawned_at`.
+ *
+ * #610 fixed this event on the batch member rail (`reconcileMemberSlot`,
+ * `batch-dispatch.ts`) and the same two defects were live here, on the rail
+ * that fires far more often:
+ *
+ *  - CADENCE. It was journaled per-tick while the condition held, so a unit
+ *    carrying one stale milestone emitted an identical line every reconcile
+ *    interval for its whole run — roughly twenty for a 40-minute unit, which
+ *    `tick.sh` then forwards to Telegram one by one. It is now gated on the
+ *    same `SlotEntry.stale_milestone_ignored_for` marker: at most once per
+ *    DISPATCH. The marker holds the `spawned_at` it was decided for, so a
+ *    redispatch's new `spawned_at` re-arms it with no reset site.
+ *  - PAYLOAD. `at` was the MILESTONE's timestamp here and the engine's
+ *    decision time on the batch rail — one event name, two meanings, and
+ *    nothing in the line to say which. Both rails now agree: `at` is the
+ *    decision time, `milestone_at` is how old the ignored milestone is.
+ *
+ * Returns the patched state (the marker is persisted) — callers must thread
+ * it, or the event re-fires next tick as if nothing had been recorded.
  */
-function journalStaleMilestoneIfIgnored(ctx, unit, slot, truth, verifiedComplete, closedSignal) {
+function journalStaleMilestoneIfIgnored(ctx, state, unit, slot, truth, verifiedComplete, closedSignal) {
     const { milestone } = truth;
     if (verifiedComplete || closedSignal || milestone === null)
-        return;
+        return state;
     if (!(0, groundtruth_1.isVerifiedComplete)(milestone, false))
-        return;
+        return state;
+    if (slot.stale_milestone_ignored_for === slot.spawned_at)
+        return state;
+    const now = ctx.deps.now();
     journal(ctx, 'stale-milestone-ignored', unit, {
         slot: slot.id,
         run: milestone.run,
-        at: milestone.at,
+        at: now.toISOString(),
+        milestone_at: milestone.at,
         detail: `predates dispatch spawned_at=${slot.spawned_at}`,
     });
+    return (0, state_1.patchSlot)(state, slot.id, { stale_milestone_ignored_for: slot.spawned_at }, now);
+}
+/**
+ * Read THIS dispatch's last tool call from its log slice (#591, #620) — a
+ * pure, side-effect-free parse, safe to call from any slot status and any
+ * number of times: unlike `recordDispatchRunLog` below, it writes nothing to
+ * `runs.jsonl`, so it carries no exactly-once constraint. Exists because a
+ * `verify-incomplete`/`unit-failed` decision can land on a LATER tick than
+ * the one that detected the dead pid (e.g. ground truth was unreachable in
+ * between) — by then the slot has moved past `running` and
+ * `recordDispatchRunLog`'s guard refuses to re-read, but the dispatch's log
+ * file is static once the agent has exited, so re-parsing it here yields the
+ * same answer every time.
+ */
+function readLastToolForSlot(ctx, slot, unit) {
+    if (slot.spawned_at === null)
+        return null;
+    return (0, core_1.parseLastToolUse)(dispatchLogSlice(ctx, slot, unit).content);
+}
+/**
+ * THIS dispatch's slice of the per-unit log: path, start offset, contents.
+ *
+ * #524: the log is per-unit and append-mode, so reading from byte 0 would
+ * include every PRIOR dispatch's output too (claude: unparseable JSON
+ * concatenation; opencode: summed tokens double-counted). One reader so the
+ * `?? 0` legacy default (a pre-1.7.0 slot has no recorded offset) cannot
+ * drift between the two callers — `readLastToolForSlot`'s "always agrees
+ * with whatever `reconcileRunning` saw" guarantee is exactly the claim that
+ * a second copy of this derivation would quietly break.
+ */
+function dispatchLogSlice(ctx, slot, unit) {
+    const logFile = (0, dispatch_1.dispatchLogPath)(ctx.deps.store.runsDir, unit);
+    const offset = slot.log_offset_at_spawn ?? 0;
+    return { logFile, offset, content: (0, run_log_1.readDispatchLog)(logFile, offset) };
 }
 /**
  * Append this dispatch's `runs.jsonl` entry (#524) — one per completed
@@ -996,13 +1112,8 @@ function recordDispatchRunLog(ctx, state, slot, unit) {
     // dispatch.command/tierModels — so a mixed agent-CLI ladder's runs.jsonl
     // entry (AC3) matches what was actually spawned for this dispatch.
     const { cmd, model } = (0, dispatch_1.resolveTierSpawn)(ctx.dispatch, tier, issue);
-    const logFile = (0, dispatch_1.dispatchLogPath)(ctx.deps.store.runsDir, unit);
-    // #524: read only THIS dispatch's slice — the log is per-unit and
-    // append-mode, so the whole file would include every prior dispatch's
-    // output too (claude: unparseable JSON concatenation; opencode: summed
-    // tokens double-counted).
-    const offset = slot.log_offset_at_spawn ?? 0;
-    const logContent = (0, run_log_1.readDispatchLog)(logFile, offset);
+    // #524: read only THIS dispatch's slice — see `dispatchLogSlice`.
+    const { logFile, offset, content: logContent } = dispatchLogSlice(ctx, slot, unit);
     const runEntry = (0, run_log_1.buildSchedRunLogEntry)({
         unit,
         role: slot.role,
@@ -1052,7 +1163,9 @@ function reconcileRunning(ctx, state, slot, truth, unit) {
     // — a re-enqueued issue's PREVIOUS run's report milestone must not read as
     // "complete" the moment the fresh agent's first tick polls it.
     const verifiedComplete = (0, groundtruth_1.isVerifiedComplete)(truth.milestone, closedSignal, slot.spawned_at);
-    journalStaleMilestoneIfIgnored(ctx, unit, slot, truth, verifiedComplete, closedSignal);
+    // Threaded, not discarded: the once-per-dispatch marker lives in the state
+    // this returns.
+    const marked = journalStaleMilestoneIfIgnored(ctx, state, unit, slot, truth, verifiedComplete, closedSignal);
     if (verifiedComplete && !(0, groundtruth_1.isParkedMilestone)(truth.milestone)) {
         journal(ctx, 'external-advance', unit, {
             pid: slot.pid,
@@ -1062,15 +1175,15 @@ function reconcileRunning(ctx, state, slot, truth, unit) {
             // at completion regardless — the actual signal was the report milestone.
             detail: closedSignal ? 'issue closed' : `report done (role=${slot.role})`,
         });
-        killUnitAgent(ctx, state, unit);
+        killUnitAgent(ctx, marked, unit);
         // The agent was still alive (that's what "externally-advanced" means) —
         // log it here too, or an external-advance dispatch would never get a
         // runs.jsonl entry at all (it never takes the dead-pid branch above).
-        recordDispatchRunLog(ctx, state, slot, unit);
-        const exited = (0, state_1.transitionSlot)(state, slot.id, 'exited', {}, now);
+        recordDispatchRunLog(ctx, marked, slot, unit);
+        const exited = (0, state_1.transitionSlot)(marked, slot.id, 'exited', {}, now);
         return completeUnitOrRecover(ctx, exited, unit, truth, 'external-advance');
     }
-    const progress = applyProgressSignals(ctx, state, slot, truth, unit);
+    const progress = applyProgressSignals(ctx, marked, slot, truth, unit);
     if (progress.progressed)
         return progress.state;
     // No progress: the stall timer (AC4). The phase now IN FLIGHT is the last
@@ -1154,6 +1267,14 @@ function completeUnitOrRecover(ctx, state, unit, truth, via,
 // `external-advance` caller passes none — it DOES record a run log (`reconcileRunning`,
 // #524), but ground truth already confirmed completion, so this path never reaches the
 // `unverified-exit` branch and the tool name has nothing to attribute.
+//
+// #620: a THUNK, not a value, for the deferred `exited`/`verifying` rail — that caller
+// has no read of its own to hand over and would otherwise have to read the dispatch log
+// eagerly on every tick, including the majority that return early (ground truth
+// unreachable) or complete the unit, discarding it. The log is bounded at
+// MAX_DISPATCH_LOG_BYTES (32 MiB) and its size is set by the spawned agent, so an
+// eager read is up to 32 MiB per slot per tick for as long as an outage holds slots in
+// `verifying`. Resolved once, only on the branch that consumes it.
 lastTool = null) {
     const now = ctx.deps.now();
     let next = state;
@@ -1185,7 +1306,22 @@ lastTool = null) {
         // healthy park sandwiched between two unrelated units' suspect exits
         // would be invisible to the cross-unit correlation and could still tip
         // it into a false-positive pause.
-        return parkUnit(ctx, recordDispatchOutcome(ctx, next, unit, slot, false), unit, truth.milestone);
+        const parked = recordDispatchOutcome(ctx, next, unit, slot, false);
+        const pr = (0, groundtruth_1.prOfMilestone)(truth.milestone); // non-null: isParkedMilestone guarantees it
+        if (pr === null) {
+            // Structurally unreachable, and deliberately not silent if it ever is:
+            // the slot is already in `verifying`, so an early return with nothing
+            // journaled leaves it re-deciding the same way every tick, forever,
+            // against an empty trail. Return the recorded state (the pre-#596 form
+            // did — `parkUnit`'s internal guard returned the state it was handed),
+            // not the un-recorded `next`.
+            journal(ctx, 'ground-truth-unreachable', unit, {
+                slot: slot.id,
+                detail: `parked milestone (run=${truth.milestone?.run ?? 'unknown'}) carries no parseable pr= key — holding in verifying`,
+            });
+            return parked;
+        }
+        return parkUnit(ctx, parked, unit, pr);
     }
     // #575: fence to THIS dispatch's `spawned_at` — an agent that exited having
     // posted nothing new must not read as complete against the issue's
@@ -1193,19 +1329,20 @@ lastTool = null) {
     // below (`unverified-exit`) instead.
     const closedSignal = effectiveClosedSignal(slot, truth);
     const verifiedComplete = (0, groundtruth_1.isVerifiedComplete)(truth.milestone, closedSignal, slot.spawned_at);
-    journalStaleMilestoneIfIgnored(ctx, unit, slot, truth, verifiedComplete, closedSignal);
+    next = journalStaleMilestoneIfIgnored(ctx, next, unit, slot, truth, verifiedComplete, closedSignal);
     if (verifiedComplete) {
         return completeUnit(ctx, recordDispatchOutcome(ctx, next, unit, slot, false), unit, via);
     }
     const suspect = msSinceLastProgress(slot, now) < types_1.SUSPECT_DISPATCH_WINDOW_MS;
     next = recordDispatchOutcome(ctx, next, unit, slot, suspect);
+    const resolvedLastTool = typeof lastTool === 'function' ? lastTool() : lastTool;
     return enterRecovery(ctx, next, unit, 'verify-incomplete', 'unverified-exit', truth, {
         observed: truth.milestone
             ? `milestone ${truth.milestone.phase}/${truth.milestone.status}; closed=${truth.closed}`
             : `no milestone; closed=${truth.closed}`,
         // #591: attributes the unverified exit to a concrete cause (e.g. `Monitor`)
         // without opening the transcript. Omitted when the log yielded no tool_use.
-        ...(lastTool !== null ? { last_tool: lastTool } : {}),
+        ...(resolvedLastTool !== null ? { last_tool: resolvedLastTool } : {}),
     });
 }
 /** Re-attach or spawn a slot left `assigned` by a crash between assign and spawn. */
@@ -1256,7 +1393,16 @@ function reconcileSlots(ctx, state, polled) {
                 break;
             case 'exited':
             case 'verifying':
-                next = completeUnitOrRecover(ctx, next, unit, truth, 'verify-complete');
+                // #620: this branch is reached on a LATER tick than the one that
+                // detected the dead pid whenever the first attempt's ground truth
+                // was unreachable (`completeUnitOrRecover` returns early in that
+                // case, before consuming the lastTool `reconcileRunning` already
+                // read). Re-read it here rather than losing it — the log itself is
+                // static once the agent has exited, so this always agrees with
+                // whatever `reconcileRunning` saw. Passed as a thunk so the read
+                // happens only on the tick that actually reaches the unverified-exit
+                // decision, not on every tick an outage holds the slot here.
+                next = completeUnitOrRecover(ctx, next, unit, truth, 'verify-complete', () => readLastToolForSlot(ctx, slot, unit));
                 break;
             case 'recovering':
                 next = reconcileRecovering(ctx, next, unit);