agent-coord-mcp 0.26.6 → 0.26.8

package/src/tools/stall.ts

@@ -21,6 +21,7 @@ import path from "node:path";
 import { z } from "zod";
 import { parseWorkDoc, workstreamsV1RowsOf } from "@davidbalzan/groundwork-seam";
 import { ROOT, AGENTS_FILE, readJson } from "../store.js";
+import { loadLiveTransports } from "./registry.js";
 
 const BOARD_DOC = "docs/WORKSTREAMS.md";
 const STALL_MS = 30 * 60 * 1000;
@@ -71,9 +72,32 @@ export function haltState(): { halted: boolean; reason?: string; by?: string; at
 // ---------- the run mark ----------
 
 /** Every run leaves this, HIT or MISS. It is what makes a dead clock visible. */
+/**
+ * Record a run that FAILED.
+ *
+ * Acceptance from the queue item, and the clause most easily skipped: "a
+ * scheduled check reports its FETCH FAILURES, or a broken check is
+ * indistinguishable from a quiet registry". Without this, a clock that fires
+ * every 30 minutes and throws every time leaves NO marks at all — identical on
+ * disk to a clock that was never installed.
+ */
+export function markRunFailure(reason: string): void {
+  mkdirSync(ROOT, { recursive: true });
+  let history: RunMark[] = [];
+  try {
+    history = JSON.parse(readFileSync(runFile(), "utf8")).history ?? [];
+  } catch {
+    /* first run */
+  }
+  history.push({ at: Date.now(), hits: 0, checked: 0, failed: reason });
+  writeFileSync(runFile(), JSON.stringify({ history: history.slice(-200) }, null, 2));
+}
+
+type RunMark = { at: number; hits: number; checked: number; failed?: string };
+
 function markRun(result: { hits: StallHit[]; checked: number }) {
   mkdirSync(ROOT, { recursive: true });
-  let history: { at: number; hits: number; checked: number }[] = [];
+  let history: RunMark[] = [];
   try {
     history = JSON.parse(readFileSync(runFile(), "utf8")).history ?? [];
   } catch {
@@ -94,7 +118,7 @@ export const lastRanSchema = { maxAgeMinutes: z.number().optional() };
  */
 export async function stallClockStatusTool(args: { maxAgeMinutes?: number }) {
   const maxAge = (args.maxAgeMinutes ?? 60) * 60 * 1000;
-  let history: { at: number; hits: number; checked: number }[] = [];
+  let history: RunMark[] = [];
   try {
     history = JSON.parse(readFileSync(runFile(), "utf8")).history ?? [];
   } catch {
@@ -105,6 +129,12 @@ export async function stallClockStatusTool(args: { maxAgeMinutes?: number }) {
     };
   }
   const last = history[history.length - 1];
+  // A RUN THAT FAILED IS NOT A RUN THAT PASSED. Without this the clock reads
+  // "fresh" off a mark it wrote while erroring — the age is honest and the
+  // health is not, which is the same shape as a fresh heartbeat from a stuck
+  // agent.
+  const failures = history.filter((h) => h.failed);
+  const lastFailed = last?.failed;
   const age = Date.now() - (last?.at ?? 0);
   // `>=`, NOT `>`, AND THE DIFFERENCE IS A REAL RACE RATHER THAN PEDANTRY.
   //
@@ -118,18 +148,27 @@ export async function stallClockStatusTool(args: { maxAgeMinutes?: number }) {
   // 0-minute window means nothing is ever fresh, which is what a caller asking
   // for one means.
   const stale = age >= maxAge;
-  const misses = history.filter((h) => h.hits === 0).length;
+  const misses = history.filter((h) => !h.failed && h.hits === 0).length;
+  const hits = history.filter((h) => !h.failed && h.hits > 0).length;
   return {
-    ok: !stale,
+    // A FAILING CLOCK IS NOT A HEALTHY ONE. It writes marks on schedule, so the
+    // age looks fresh while nothing is being measured — a fresh heartbeat from
+    // a stuck agent, one level up.
+    ok: !stale && !lastFailed,
     ...(stale
       ? {
           error: `stall_check last ran ${Math.round(age / 60000)}m ago, past the ${args.maxAgeMinutes ?? 60}m window — THE CLOCK IS STOPPED. No alerts is not the same as no stalls.`,
         }
-      : {}),
+      : lastFailed
+        ? {
+            error: `the clock is RUNNING but its last run FAILED: ${lastFailed}. It is firing on schedule and measuring nothing, which reads as fresh and is not.`,
+          }
+        : {}),
     lastRanMinutesAgo: Math.round(age / 60000),
     runs: history.length,
     misses,
-    hits: history.length - misses,
+    hits,
+    failures: failures.length,
   };
 }
 
@@ -143,17 +182,56 @@ export async function stallCheckTool(args: { repo?: string; stallMinutes?: numbe
   const board = path.join(repo, BOARD_DOC);
   if (!existsSync(board)) return { ok: false as const, error: `no ${BOARD_DOC} under '${repo}'` };
 
+  const liveTransports = await loadLiveTransports();
   const rows = workstreamsV1RowsOf(parseWorkDoc(readFileSync(board, "utf8")));
   const inFlight = rows.filter((r) => /🚧/.test(r.status));
   const reg = await readJson<Record<string, { lastHeartbeat: number }>>(AGENTS_FILE, {});
   const now = Date.now();
   const hits: StallHit[] = [];
+  /** Rows whose VCS activity could not be measured. NOT stall claims. */
+  const unmeasurable: { agentId: string; value: string; why: string }[] = [];
 
   for (const row of inFlight) {
     const agentId = row.owner.replace(/[`*]/g, "").trim();
     const entry = reg[agentId];
     if (entry) {
       const age = now - entry.lastHeartbeat;
+      // A HEARTBEAT IS ONLY EVIDENCE WHERE SOMETHING WRITES ONE.
+      //
+      // `heartbeat` is called by the PUSHER, never by the agent. Only
+      // `coord-pusher.mjs` (the REMOTE pusher) calls it, and it must: a remote
+      // marker cannot be pid-probed across machines, so its liveness IS the
+      // heartbeat. `hooks/tmux-pusher.mjs` — what this fleet actually runs —
+      // never calls it, because a LOCAL marker's liveness is `isPidAlive`.
+      //
+      // So for a local transport there is no heartbeat SOURCE at all. Before
+      // kit#137, `list_agents` stamped these agents and that fabrication was
+      // the only thing keeping the field moving; removing it left the field
+      // honest and empty. Measured: three attached, working agents at an
+      // IDENTICAL 44.7m — the uniform signature of one shared cause, not three
+      // stalls.
+      //
+      // WHY NOT JUST MAKE tmux-pusher HEARTBEAT: because the signal would mean
+      // "the pusher process is alive", which `isPidAlive` already answers for
+      // local markers. During the 17-hour stall every transport was live the
+      // whole time, so a pusher heartbeat would have read FRESH for all 17
+      // hours. It would restore a field without restoring a detector — and the
+      // case this verb exists for is exactly the one it would miss. The vcs
+      // half is what catches "alive and not progressing"; saying so is more
+      // honest than a green field.
+      //
+      // Unknown is not stalled (kit#138). This does NOT narrow Task 3.5: an
+      // agent with no transport, or a REMOTE one where the heartbeat genuinely
+      // is the liveness mechanism, still HITs on a dead heartbeat.
+      const marker = liveTransports.get(agentId);
+      if (age > limit && marker && marker.transport === "tmux-push") {
+        unmeasurable.push({
+          agentId,
+          value: marker.transport,
+          why: `heartbeat is ${Math.round(age / 60000)}m old, but nothing writes heartbeats for a local 'tmux-push' transport — hooks/tmux-pusher.mjs does not call heartbeat, and this marker's liveness is its pid. There is no heartbeat SOURCE here, so the age measures nothing about this agent`,
+        });
+        continue;
+      }
       if (age > limit) {
         hits.push({ kind: "no-heartbeat", agentId, stream: row.stream.slice(0, 60), minutes: Math.round(age / 60000) });
         continue;
@@ -162,22 +240,61 @@ export async function stallCheckTool(args: { repo?: string; stallMinutes?: numbe
     // A FRESH HEARTBEAT IS NOT PROGRESS. An agent can be alive and stuck, which is
     // the case "notice the room" never catches: the pane is responsive, so nobody
     // looks. Ask the branch instead.
-    const branch = (row.branchWorktree.match(/`([^`]+)`/)?.[1] ?? "").trim();
-    if (!branch || !/\//.test(branch)) continue;
+    //
+    // RESOLVE A REF, AND SAY SO WHEN THE VALUE IS NOT ONE.
+    //
+    // `git log -1 <value>` accepts a PATHSPEC exactly as readily as a ref, and
+    // the old guard only required a `/` — which every path has. So the board's
+    // `Branch · Worktree` cells, which hold PATHS, were fed to git and silently
+    // measured as paths: the aide was reported at 2,271 minutes, the age of
+    // `docs/phases/phase5`'s last commit, not of any activity by that agent.
+    // Verified: `docs/phases/phase5` does not resolve as a ref, yet
+    // `git log -1 --format=%cI docs/phases/phase5` returns a date.
+    //
+    // Other rows read plausibly only by coincidence — a path that happens to be
+    // committed often looks like an active branch.
+    //
+    // UNKNOWN IS NOT STALLED, which this verb already gets right for
+    // heartbeats. An unresolvable value is REPORTED as unmeasurable rather than
+    // skipped in silence: a silent `continue` and a healthy agent produce the
+    // same output, which is the failure this whole verb exists to avoid.
+    const raw = (row.branchWorktree.match(/`([^`]+)`/)?.[1] ?? "").trim();
+    if (!raw) {
+      unmeasurable.push({ agentId, value: "", why: "no value in the Branch · Worktree cell" });
+      continue;
+    }
+    let sha = "";
+    try {
+      sha = execFileSync("git", ["rev-parse", "--verify", "--quiet", `${raw}^{commit}`], {
+        cwd: repo, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"],
+      }).trim();
+    } catch {
+      sha = "";
+    }
+    if (!sha) {
+      unmeasurable.push({
+        agentId,
+        value: raw,
+        why: `'${raw}' does not resolve as a git ref — it is a path or glob, and \`git log <path>\` would silently report that PATH's last commit as this agent's activity`,
+      });
+      continue;
+    }
     try {
-      const iso = execFileSync("git", ["log", "-1", "--format=%cI", branch], {
+      // The resolved SHA, and `--`: a ref can then never be re-read as a
+      // pathspec, which is the ambiguity that produced the wrong number.
+      const iso = execFileSync("git", ["log", "-1", "--format=%cI", sha, "--"], {
         cwd: repo, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"],
       }).trim();
       const age = now - Date.parse(iso);
       if (age > limit) {
-        hits.push({ kind: "no-vcs-activity", agentId, branch, minutes: Math.round(age / 60000) });
+        hits.push({ kind: "no-vcs-activity", agentId, branch: raw, minutes: Math.round(age / 60000) });
       }
     } catch {
-      /* unknown branch: not a stall claim we can make */
+      unmeasurable.push({ agentId, value: raw, why: "resolved as a ref but its log could not be read" });
     }
   }
 
-  const result = { hits, checked: inFlight.length };
+  const result = { hits, checked: inFlight.length, unmeasurable };
   markRun(result);
   return {
     ok: true as const,
@@ -188,7 +305,7 @@ export async function stallCheckTool(args: { repo?: string; stallMinutes?: numbe
     dm: hits.length > 0,
     note:
       hits.length === 0
-        ? `MISS — ${inFlight.length} in-flight row(s), none stalled. No DM. The run IS recorded: read it with stall_clock_status, because no alert and nothing running look identical from here.`
+        ? `MISS — ${inFlight.length} in-flight row(s), none stalled${unmeasurable.length ? `; ${unmeasurable.length} row(s) UNMEASURABLE for VCS activity (${unmeasurable.map((u) => u.agentId).join(", ")}) — reported, not counted as healthy` : ""}. No DM. The run IS recorded: read it with stall_clock_status, because no alert and nothing running look identical from here.`
         : undefined,
   };
 }

package/src/tools/transport.ts

@@ -1338,7 +1338,6 @@ export async function doctorTool(args: { fix?: boolean; maxFileBytes?: number })
   //      newest hooks file). Local tmux-push only, same as 1b.
   {
     const onDisk = onDiskBuildMtime();
-    const outdated: string[] = [];
     const unverifiable: string[] = [];
     for (const fname of await listTransportFiles()) {
       const file = path.join(TRANSPORT_DIR, fname);
@@ -1361,18 +1360,45 @@ export async function doctorTool(args: { fix?: boolean; maxFileBytes?: number })
       }
       if (onDisk === undefined) continue;
       if (marker.serverBuildMtime < onDisk - 1) {
+        // A COMPONENT CAN REPORT ITS OWN FRESHNESS; A SIBLING'S RECORD OF IT
+        // GOES STALE SILENTLY. That asymmetry is the whole correction here.
+        //
+        // `serverBuildMtime` is stamped inside `attach_agent` — by the PUSHER's
+        // attach, once — so it records what the server was AT THE LAST ATTACH.
+        // A server-only restart (David's `/mcp`) reloads the server and touches
+        // no marker, so this stamp cannot be refreshed by the thing it
+        // describes and CANNOT DISTINGUISH a restarted server from an
+        // un-restarted one.
+        //
+        // Measured on three agents: each answered `list_subscriptions` — a tool
+        // that does not exist before 0.26.7 — while its marker still stamped a
+        // pre-0.26.7 build. The stamp was 2.2 hours behind the installed dist
+        // on a server demonstrably running the new code.
+        //
+        // So this is reported as what it IS: the marker predates the build, and
+        // the server's actual state is UNKNOWN from here. Calling it an
+        // outdated server is a claim this record cannot support, and it priced
+        // a restart decision on a count where at least three entries had
+        // already restarted.
+        //
+        // The pusher half is untouched and remains valid: `scriptMtime` is
+        // self-reported by the process it describes.
         const stamped = new Date(marker.serverBuildMtime).toISOString();
         const current = new Date(onDisk).toISOString();
-        outdated.push(`${marker.agentId} (stamped by server build ${stamped}, on-disk build ${current})`);
+        unverifiable.push(
+          `${marker.agentId} (marker stamped at the last ATTACH from server build ${stamped}, on-disk build ${current} — this says the MARKER is behind, NOT that the server is: a server-only restart refreshes no marker. Exercise the server to settle it, or detach_agent + attach_agent to refresh the stamp)`,
+        );
       }
     }
-    const bad = [...outdated, ...unverifiable];
+    // `outdated` is gone, not emptied: nothing can populate it any more, and a
+    // bucket that always reports 0 is a claim the reader still has to discount.
+    const bad = unverifiable;
     findings.push({
       check: "marker-server-provenance",
       level: bad.length ? "warn" : "ok",
       detail: bad.length
-        ? `${outdated.length} transport marker(s) stamped by a server build older than dist/ and ${unverifiable.length} with no provenance stamp at all — the stamping/spawn logic (freshness basis, pusher argv) predates or cannot be tied to the current code. Restart that agent's session, then detach_agent + attach_agent.`
-        : "all local transport markers were stamped by the current server build",
+        ? `${bad.length} transport marker(s) cannot report their server's build: the stamp is written by attach_agent, so it records the server AT THE LAST ATTACH and a server-only restart refreshes nothing. This says the MARKER is behind or absent — NOT that the server is stale. Exercise the server to settle it (a tool that exists only in the newer build), or detach_agent + attach_agent to refresh the stamp. Restart the session too if the server itself is old.`
+        : "every local transport marker was stamped at an attach against the current build — which says the markers are current, not that every server is",
       fixable: false,
       items: bad.length ? bad : undefined,
     });

package/src/tools/worktrees.ts

@@ -173,6 +173,28 @@ async function create(a: {
     // made: a verb that silently returns a tree on a different branch than asked
     // for is the adjacent-answer shape.
     const head = git(existing.path, ["rev-parse", "HEAD"]);
+    // IS THE REUSED TREE ACTUALLY AT THE BASE IT CLAIMS?
+    //
+    // The verb warned when an existing tree was on a different BRANCH and said
+    // nothing about it being behind the BASE — so a tree left on last week's
+    // main was handed back as ready. Starting a new task there produces the
+    // "stale main -> confidently-wrong inventories" failure the worker card
+    // already warns about, and nothing about the result looks stale.
+    //
+    // Reported here, not refused: an existing tree legitimately holds
+    // in-progress work mid-slice (1.3). The refusal belongs to `claim`, which
+    // is the verb that means "start something new".
+    let atBase = false;
+    let behindBy: number | null = null;
+    try {
+      const baseSha = git(repo, ["rev-parse", "--verify", `${ref}^{commit}`]);
+      atBase = head === baseSha;
+      if (!atBase) behindBy = Number(git(existing.path, ["rev-list", "--count", `HEAD..${ref}`])) || 0;
+    } catch {
+      // NOT MEASURED IS NOT AT-BASE. Leaving `atBase` false is the safe
+      // direction: it makes `claim` ask rather than assume.
+      atBase = false;
+    }
     return {
       ok: true as const,
       path: existing.path,
@@ -180,6 +202,8 @@ async function create(a: {
       branch: existing.branch,
       created: false,
       base: ref,
+      atBase,
+      behindBy: behindBy ?? 0,
       ...(existing.branch !== branch
         ? { warning: `existing tree is on '${existing.branch}', not the '${branch}' this call would have created — reusing it, NOT re-pointing it` }
         : {}),
@@ -198,6 +222,10 @@ async function create(a: {
     branch,
     created: true,
     base: ref,
+    // Cut from `sha`, which IS origin/<base> — true by construction here, and
+    // stated so callers need not special-case "created" to know it.
+    atBase: true,
+    behindBy: 0,
     ...(a.ephemeral
       ? { ephemeral: true, removeWith: `git -C ${repo} worktree remove --force ${target} && git -C ${repo} branch -D ${branch}` }
       : {}),