@cohortapp/agent-sdk 2.18.14 → 2.18.16

package/lib/telemetry/collect.mjs

@@ -84,8 +84,14 @@
  *       // Also on `machine` (open record), so hq's fleet view can show it next
  *       // to sdkVersion without a schema change. `ok` is the attempt's verdict:
  *       // true = healthy on `to`; false = install failed or rolled back to `from`.
+ *       // `failStreak` is the one field here that describes a PATTERN: N
+ *       // consecutive failed attempts to reach @latest, since `stuckSince`,
+ *       // against `streakTarget`. Absent below 1. A seat that has simply not
+ *       // been asked (nothing newer published, or held after its last failure)
+ *       // never accrues one — see autoupdate.sh's write_last.
  *       upgrade?: { at: ISO8601, from: string, to: string, ok: boolean,
- *                   reason?: string, healthy?: boolean },  // reason: WHY a failure failed
+ *                   reason?: string, healthy?: boolean,     // reason: WHY a failure failed
+ *                   failStreak?: number, stuckSince?: ISO8601, streakTarget?: string },
  *       // THE BEATING DAEMON — which process is emitting this beat, what code
  *       // it is actually running, and the last health-gate verdict on it.
  *       // Also on `machine` (open record). `sdkVersion` ABOVE is what is
@@ -166,6 +172,7 @@ import { snapshot as countersSnapshot } from "../diagnostics/counters.mjs";
 import { replyDebtFromCounters } from "../daemon/reply-debt.mjs";
 import { summarisePinnedDrift, countPins } from "../upgrade/pinned-drift.mjs";
 import { IGNORED_DRIFT_REL } from "../upgrade/ignored-drift.mjs";
+import { listHandoffs, DEFAULT_HANDOFF_DEADLINE_MS } from "../session/handoffs.mjs";
 
 /** Default temperature probe ceiling — used only as a guard in alerts; here we just report. */
 const VALID_STATES = new Set(["active", "idle", "busy", "error", "offline"]);
@@ -1129,6 +1136,44 @@ async function collectIdentity(o, osImpl) {
 /** A main-session heartbeat older than this is not live. */
 export const SESSION_STALE_MS = 90_000;
 
+/**
+ * A FAILED REVIVE, written by the on-host actors (the daemon's front-door
+ * revive and `autoupdate.sh#revive_daemon`) when a kickstart returned but no
+ * live pid could be confirmed — a failure to escalate, which has to reach the
+ * fleet, not just the seat's log. Relative to the agent root; fresh window
+ * bounds how long it rides the beat after the last failed attempt.
+ */
+export const REVIVE_NOTE_REL = "state/telemetry/revive-note.json";
+export const REVIVE_NOTE_FRESH_MS = 90 * 60 * 1000;
+
+/**
+ * Pure: a revive-note that NAMES budget exhaustion, or null.
+ *
+ * A separate legibility defect from a failed revive: when the revive ladder
+ * spends its whole restart budget and STOPS (`shouldReviveFrontDoor` →
+ * `budget-spent`), nothing is written — the seat goes quiet and a quiet system
+ * looks healthy. A ladder that gave up is a failure that does not name itself.
+ * This turns that silence into a beat note carried through the SAME channel as
+ * `machine.reviveNote`, so the beat says "stopped because budget spent" rather
+ * than nothing. Only the budget-spent verdict qualifies — any other verdict
+ * (still confirming, backing off, answering) is not the ladder giving up.
+ *
+ * @param {{verdict?:string, target?:string, attempts?:number, at?:string}} a
+ * @returns {{reason:string, target?:string, detail:string, at:string}|null}
+ */
+export function budgetSpentNote(a = {}) {
+  const x = a && typeof a === "object" ? a : {};
+  if (x.verdict !== "budget-spent") return null;
+  const attempts = Number.isFinite(x.attempts) ? x.attempts : 0;
+  const at = typeof x.at === "string" && x.at ? x.at : new Date().toISOString();
+  return {
+    reason: "revive-budget-spent",
+    ...(typeof x.target === "string" && x.target ? { target: x.target } : {}),
+    detail: `revive budget spent after ${attempts} attempts`,
+    at,
+  };
+}
+
 /**
  * Pure: derive `{frontDoor, sessionLive}` from the heartbeat the main session's
  * feed writes every 15 s (`state/session/heartbeat.json` `{pid, ppid, sessionId,
@@ -1389,13 +1434,145 @@ export function sessionNote(a = {}) {
   };
 }
 
+/**
+ * Front-door-routed inbox lanes. `sweep`, `held` and `deferred` are the
+ * consumer's OWN maintenance lanes — items it parked on purpose, not items
+ * waiting on a person — and a `cohort-held-*` id is the sibling hold-queue
+ * shape (see meeting-capture's `kind==HOLD`). Neither is evidence the front
+ * door is failing to read; counting them would wedge a seat that is doing
+ * exactly what it was told to do.
+ */
+const WEDGE_EXCLUDED_LANES = new Set(["sweep", "held", "deferred"]);
+const WEDGE_HELD_ID_RE = /^cohort-held-/;
+
+/** An inbox item this file's `sessionWedge` treats as front-door-routed. */
+function isFrontDoorInboxItem(item) {
+  if (!item || typeof item !== "object") return false;
+  const id = typeof item.id === "string" ? item.id : "";
+  if (WEDGE_HELD_ID_RE.test(id)) return false;
+  const lane = typeof item.lane === "string" ? item.lane : "";
+  if (WEDGE_EXCLUDED_LANES.has(lane)) return false;
+  return true;
+}
+
+/**
+ * Pure: the wedge verdict — "beating but not reading". A seat can have a
+ * perfectly live heartbeat while the process behind it has stopped actually
+ * answering inbound (wedged on a dialog with no watchdog for THIS failure
+ * mode, a deadlocked event loop, anything short of the process dying). Two
+ * independent signals, checked in order, because they catch different
+ * failures and the first is the harder evidence:
+ *
+ *  1. PRIMARY — an un-acked handoff (`lib/session/handoffs.mjs`) past its
+ *     deadline. The consumer handed a cadence tick to the front door and
+ *     nobody acked it in time; this is true regardless of which front door is
+ *     nominally live, because a handoff is only ever written TO whichever
+ *     door claimed the tick. Deadline math mirrors `expireHandoffs` exactly
+ *     (`deadlineAt`, else `enqueuedAt + deadlineMs`) so the two never
+ *     disagree about which handoff is overdue.
+ *  2. SECONDARY — inbox-unconsumed. DEFERRED and HELD OFF in this build
+ *     (`SECONDARY_WEDGE_ENABLED === false`), because it cannot yet fire
+ *     correctly end-to-end (Hannah, PR #70 review #2). Three things must land
+ *     together first, NONE of them in this PR:
+ *       (i)   a consume-stamp on EVERY front-door path — today only
+ *             `ackHandoff` stamps `lastConsumedAt`, so it advances only on
+ *             handoff acks (~every 30 min), never on the inbound
+ *             claim/reply/done path in session-runtime;
+ *       (ii)  a real front-door inbox reader feeding `frontDoorInbox` into
+ *             `collectStatus` — nothing feeds it, so `inbox` is always `[]`;
+ *       (iii) lane/held keying matched to the real item shape — on this fleet
+ *             inbox items carry `channel`/`kind`/`claimed_by`, not `lane`, and
+ *             no id begins `cohort-held-`, so `isFrontDoorInboxItem`'s
+ *             exclusion is keyed on fields the items do not carry.
+ *     Shipped live at the 90s bound it would read most seats wedged, and
+ *     `rollout`'s `done` (which requires zero wedged) would never clear. The
+ *     branch, `isFrontDoorInboxItem` and `WEDGE_EXCLUDED_LANES` are kept
+ *     intact behind the flag so the follow-up wires them rather than rebuilds
+ *     them; the PRIMARY handoff-overdue tell is the real signal and is fully
+ *     live.
+ *
+ * Defensive by construction: every read is behind a type check, nothing here
+ * touches a clock or the filesystem, and no input shape can make it throw.
+ *
+ * @param {object} [a]
+ * @param {object[]} [a.handoffs]      `listHandoffs()` records (or equivalent)
+ * @param {object[]} [a.inbox]         raw inbox items, each `{id?, lane?}`
+ * @param {number|null} [a.lastConsumedAt] epoch ms, or null/undefined = never
+ * @param {"session"|"daemon"} [a.frontDoor]
+ * @param {boolean} [a.sessionLive]
+ * @param {number} [a.now]             epoch ms
+ * @param {number} [a.deadlineMs]      handoff deadline (default {@link DEFAULT_HANDOFF_DEADLINE_MS})
+ * @param {number} [a.staleMs]         inbox staleness bound (default {@link SESSION_STALE_MS})
+ * @returns {{wedged:false}|{wedged:true, reason:"handoff-overdue"|"inbox-unconsumed", since?:string}}
+ */
+// SECONDARY (inbox-unconsumed) is deferred — see the header above. A named
+// const, not a deleted branch: the follow-up flips this once the consume-stamp,
+// the frontDoorInbox feed and the real lane/held keying are all wired.
+export const SECONDARY_WEDGE_ENABLED = false;
+
+export function sessionWedge(a = {}) {
+  const o = a && typeof a === "object" ? a : {};
+  const now = Number(o.now);
+  if (!Number.isFinite(now)) return { wedged: false };
+  const deadlineMs = Number.isFinite(o.deadlineMs) && o.deadlineMs > 0 ? o.deadlineMs : DEFAULT_HANDOFF_DEADLINE_MS;
+  const staleMs = Number.isFinite(o.staleMs) ? o.staleMs : SESSION_STALE_MS;
+
+  // 1. PRIMARY — mirrors expireHandoffs' deadline math exactly.
+  const handoffs = Array.isArray(o.handoffs) ? o.handoffs : [];
+  for (const rec of handoffs) {
+    if (!rec || typeof rec !== "object" || rec.status !== "open") continue;
+    let deadline = Date.parse(rec.deadlineAt || "");
+    if (!Number.isFinite(deadline)) {
+      const enq = Date.parse(rec.enqueuedAt || "");
+      deadline = Number.isFinite(enq) ? enq + deadlineMs : now; // undated → already due
+    }
+    if (now >= deadline) {
+      return { wedged: true, reason: "handoff-overdue", since: new Date(deadline).toISOString() };
+    }
+  }
+
+  // 2. SECONDARY — DEFERRED (see header): held off until the consume-stamp,
+  // the frontDoorInbox feed and the real lane/held keying are wired end-to-end.
+  // Kept behind the flag so it is not shipped reading seats wedged spuriously.
+  if (SECONDARY_WEDGE_ENABLED && o.frontDoor === "session" && o.sessionLive === true) {
+    const inbox = Array.isArray(o.inbox) ? o.inbox : [];
+    const routed = inbox.filter(isFrontDoorInboxItem);
+    if (routed.length > 0) {
+      const lastConsumedAt = Number.isFinite(o.lastConsumedAt) ? o.lastConsumedAt : null;
+      const staleConsume = lastConsumedAt === null || now - lastConsumedAt > staleMs;
+      if (staleConsume) {
+        const since = lastConsumedAt !== null ? lastConsumedAt : oldestInboxAt(routed);
+        return {
+          wedged: true,
+          reason: "inbox-unconsumed",
+          ...(Number.isFinite(since) ? { since: new Date(since).toISOString() } : {}),
+        };
+      }
+    }
+  }
+
+  return { wedged: false };
+}
+
+/** Oldest `enqueuedAt`/`ts` among inbox items, epoch ms, else NaN. */
+function oldestInboxAt(items) {
+  let oldest = NaN;
+  for (const it of items) {
+    const ms = Date.parse((it && (it.enqueuedAt || it.ts)) || "");
+    if (Number.isFinite(ms) && (Number.isNaN(oldest) || ms < oldest)) oldest = ms;
+  }
+  return oldest;
+}
+
 /**
  * Pure: the beat's `machine.upgrade` from state/autoupdate/last.json
  * (`{from,to,at,ok,healthy,reason}` as autoupdate.sh writes it). Only the four
  * fields the fleet view reads; anything malformed → null, and the field drops
  * out rather than lying.
  * @param {object|null} last
- * @returns {{at:string, from:string, to:string, ok:boolean}|null}
+ * @returns {{at:string, from:string, to:string, ok:boolean, reason?:string,
+ *            healthy?:boolean, failStreak?:number, stuckSince?:string,
+ *            streakTarget?:string}|null}
  */
 export function upgradeSummary(last) {
   if (!last || typeof last !== "object") return null;
@@ -1413,6 +1590,19 @@ export function upgradeSummary(last) {
   // Bounded on the way out: a reason is a short token plus at most a trimmed
   // error line, and the beat is not a log shipper.
   const reason = typeof last.reason === "string" ? last.reason.trim().slice(0, 300) : "";
+  // `failStreak` — how many CONSECUTIVE upgrade attempts have failed, with the
+  // instant the run of failures began and the version it is failing against.
+  // Written by autoupdate.sh's write_last; see the long block above it for what
+  // does and does not count as an attempt. This is the ONLY field here that is
+  // about a pattern rather than an event, and it is the reason three seats sat
+  // on 2.17.0 for days without anything saying so: every individual record was
+  // a truthful `ok:false` that told a reader nothing about the fortieth one.
+  //
+  // Absent when zero, so a healthy seat's beat carries nothing extra and a
+  // reader can treat presence as the finding.
+  const failStreak = Number.isInteger(last.failStreak) && last.failStreak > 0 ? last.failStreak : 0;
+  const stuckSince = typeof last.stuckSince === "string" && !Number.isNaN(Date.parse(last.stuckSince)) ? last.stuckSince : "";
+  const streakTarget = typeof last.streakTarget === "string" ? last.streakTarget.trim().slice(0, 40) : "";
   return {
     at,
     from: typeof last.from === "string" ? last.from : "",
@@ -1420,6 +1610,9 @@ export function upgradeSummary(last) {
     ok: last.ok === true,
     ...(reason ? { reason } : {}),
     ...(typeof last.healthy === "boolean" ? { healthy: last.healthy } : {}),
+    ...(failStreak ? { failStreak } : {}),
+    ...(failStreak && stuckSince ? { stuckSince } : {}),
+    ...(failStreak && streakTarget ? { streakTarget } : {}),
   };
 }
 
@@ -2045,6 +2238,60 @@ export async function collectStatus(o = {}) {
     } catch { /* no sessionNote — the beat still carries frontDoor/sessionLive */ }
   }
 
+  // 4b-iii. BEATING BUT NOT READING (DARK-SEAT) — `machine.wedge`, present
+  // ONLY when wedged, exactly like `sessionNote` above. The decision is the
+  // pure `sessionWedge`; this is the reads that feed it — the open handoff
+  // ledger — and it is fail-open in the same way every other probe here is: a
+  // throw anywhere (a corrupt handoff file) drops the field, never the beat.
+  // `opt.handoffs` overrides disk, mirroring `opt.attention` above, so a
+  // caller that already has the ledger in memory (the daemon, a test) is never
+  // made to round-trip it through the filesystem.
+  //
+  // The `frontDoorInbox`/`lastConsumedAt` reads that fed the SECONDARY
+  // (inbox-unconsumed) signal are gone from here: that signal is DEFERRED
+  // (`SECONDARY_WEDGE_ENABLED === false` in sessionWedge), so feeding it would
+  // be inert work. They return when the follow-up wires the secondary.
+  try {
+    const handoffs = opt.handoffs !== undefined ? opt.handoffs : listHandoffs(opt.agentRoot);
+    const wedge = sessionWedge({
+      handoffs,
+      frontDoor: machine.frontDoor,
+      sessionLive: machine.sessionLive,
+      now: nowMs,
+      deadlineMs: opt.handoffDeadlineMs,
+    });
+    if (wedge && wedge.wedged) machine.wedge = { reason: wedge.reason, ...(wedge.since ? { since: wedge.since } : {}) };
+  } catch { /* no wedge field — the beat still carries frontDoor/sessionLive/sessionNote */ }
+
+  // 4b-iv. A FAILED REVIVE — `machine.reviveNote`, present only when a revive
+  // kickstarted a job and then could NOT confirm a live pid (Jacob's
+  // drain-then-restart-daemon.sh exited 0 with a dead daemon). The two on-host
+  // actors — the daemon's front-door revive (agent-daemon.mjs) and the hourly
+  // daemon backstop (autoupdate.sh#revive_daemon) — write it to
+  // state/telemetry/revive-note.json; a no-pid-after-kickstart is a FAILURE TO
+  // ESCALATE and has to reach the fleet, not just this seat's log — the seat
+  // needs a person. Surfaced only while fresh (REVIVE_NOTE_FRESH_MS) so a
+  // recovered seat ages out; the writers refresh it every attempt and clear it
+  // on a confirmed revive. `machine` is an OPEN record (like wedge/upgrade), so
+  // this lands without an hq schema change. Fail-open. `opt.reviveNote`
+  // overrides disk for tests.
+  try {
+    const rn = opt.reviveNote !== undefined
+      ? opt.reviveNote
+      : (opt.agentRoot ? safeReadJson(join(resolve(opt.agentRoot), REVIVE_NOTE_REL)) : null);
+    if (rn && typeof rn === "object" && typeof rn.reason === "string") {
+      const at = Date.parse(rn.at || "");
+      const freshMs = Number.isFinite(opt.reviveNoteFreshMs) ? opt.reviveNoteFreshMs : REVIVE_NOTE_FRESH_MS;
+      if (Number.isFinite(at) && nowMs - at <= freshMs) {
+        machine.reviveNote = {
+          reason: rn.reason,
+          at: new Date(at).toISOString(),
+          ...(typeof rn.target === "string" && rn.target ? { target: rn.target } : {}),
+        };
+      }
+    }
+  } catch { /* no reviveNote — the beat still carries everything else */ }
+
   // 4c. last upgrade outcome (WP-M6) — `machine.upgrade`, absent until the
   // first autoupdate attempt; a corrupt file drops the field, never the beat.
   try {
@@ -2145,6 +2392,7 @@ export const _internals = {
   readSdkVersion,
   sessionLiveness,
   sessionNote,
+  sessionWedge,
   sanitizeNoteDetail,
   sessionJobLabel,
   sessionJobInstalled,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cohortapp/agent-sdk",
-  "version": "2.18.14",
+  "version": "2.18.16",
   "description": "Cohort Agent SDK — autonomous AI colleague runtime. Deploy senior AI colleagues on dedicated Mac minis, wired to the Cohort operating surface.",
   "type": "module",
   "bin": {