@cohortapp/agent-sdk 2.18.14 → 2.18.16

package/scripts/local-triggers/autoupdate.sh

@@ -43,7 +43,17 @@
 #       STABLE_S apart AND every pid's `ps etime` is ≥ STABLE_S. STABLE_S is 90 s
 #       (MAESTRO_AUTOUPDATE_STABLE_S): 3× the observed ~25 s crash interval, so a
 #       loop cannot straddle both samples by luck, and well inside the hourly
-#       cadence. A 25 s loop fails on the second sample and on uptime.
+#       cadence. A 25 s loop fails on the second sample and on uptime. A pid
+#       CHANGE within the window is not, on its own, proof of that loop: this
+#       script's own restart_daemon (or an operator's `launchctl kickstart -k`)
+#       changes the pid deliberately and can land inside the same window —
+#       2.18.14 read that as a crash loop and rolled back a release that had
+#       done nothing wrong. daemon_last_exit() tells the two apart: a crash
+#       loop's reaped exit is a real non-zero every time; a deliberate
+#       kickstart's is 0 (or "-"/unknown, for a launchd too old to say). Only a
+#       real non-zero exit fails here on the pid change alone — a clean or
+#       unknown exit falls through to the uptime check, which still catches a
+#       genuinely unstable respawn (its pid is always younger than STABLE_S).
 #   (b) NO FATAL LINE in the daemon's own log since THIS daemon booted: the
 #       lines lib/util/unhandled.mjs and agent-daemon.mjs write on the way down
 #       ("[DAEMON] uncaughtException:", "[daemon] Fatal:"), the wrapper's
@@ -270,6 +280,18 @@ FATAL_RE='\[DAEMON\] uncaughtException:|\[daemon\] Fatal:|\[wrapper\] FATAL:|ERR
 HEALTH_REASON=""   # set by is_healthy on failure; a phrase, no quotes
 DAEMON_LABEL=""
 SESSION_LABEL=""
+# The pure daemon-revive decision (lib/session/revive.mjs) run by revive_daemon
+# below — the on-host backstop for a daemon that is DOWN, which a live daemon
+# can never observe about itself. Defined up here (not with SESSION_STALE_S,
+# which lives in the not-up-to-date path) because revive_daemon fires on the
+# UP-TO-DATE unhealthy branch too. DAEMON_STALE_S mirrors SESSION_STALE_S / the
+# 90s front-door observers.
+LIB_REVIVE="$AGENT_DIR/lib/session/revive.mjs"
+DAEMON_STALE_S="${MAESTRO_DAEMON_STALE_S:-90}"
+# Gap between the two settle-window reads that prove the kickstarted daemon is
+# one continuous process, not a come-up-then-crash loop. A one-shot read would
+# pass a phantom pid AND a crash loop whose local beat file it keeps re-stamping.
+DAEMON_CONFIRM_SETTLE_S="${MAESTRO_DAEMON_CONFIRM_SETTLE_S:-5}"
 
 daemon_pids(){ # every pid whose argv IS this agent's daemon entry (see DAEMON_ARGV_RE), sorted, space-joined ("" when none)
   pgrep -f "$DAEMON_ARGV_RE" 2>/dev/null | sort -n | tr '\n' ' ' | sed 's/ *$//'
@@ -355,14 +377,46 @@ sibling_job_audit(){ # log-only: every OTHER ai.maestro.<first>-* job whose LAST
   [ -n "$bad" ] && log "sibling jobs with a non-zero last exit (not a gate reason; see logs/launchd and logs/polling): $bad"
   return 0
 }
+daemon_last_exit(){ # the DAEMON_LABEL's last exit status (column 2 of `launchctl list`, the same column sibling_job_audit reads), or nothing when the label is unknown or not in the list
+  [ -n "$DAEMON_LABEL" ] || resolve_labels   # is_healthy on the up-to-date/not-stale path never called restart_daemon, so labels may still be unresolved here
+  [ -n "$DAEMON_LABEL" ] || return 0
+  launchctl list 2>/dev/null | awk -v l="$DAEMON_LABEL" 'NR>1 && $3==l { print $2; exit }'
+}
 is_healthy(){ # (a)+(b)+(c) above. Sets HEALTH_REASON on failure.
-  local p1 p2 pid up oldest oldest_pid dlog fatal why offset boot
+  local p1 p2 pid up oldest oldest_pid dlog fatal why offset boot exit_code
   HEALTH_REASON=""
   p1="$(daemon_pids)"
   [ -n "$p1" ] || { HEALTH_REASON="no daemon process for $DAEMON_PATTERN"; return 1; }
   [ "$STABLE_GAP_S" -gt 0 ] && sleep "$STABLE_GAP_S"
   p2="$(daemon_pids)"
-  [ "$p1" = "$p2" ] || { HEALTH_REASON="daemon pid changed within ${STABLE_GAP_S}s (${p1} -> ${p2:-none}) — crash loop"; return 1; }
+  if [ "$p1" != "$p2" ]; then
+    # A pid change alone is not proof of a crash loop: `launchctl kickstart -k`
+    # (this same script's own restart_daemon, or an operator's) changes the pid
+    # deliberately and can land inside this very window. 2.18.14 read that as a
+    # crash loop and rolled back a release that had done nothing wrong. The
+    # daemon's OWN last exit says which one happened: a crash-looping process
+    # exits non-zero every time launchd reaps it; a clean kickstart's reaped
+    # exit is 0 (or "-"/unknown, for a launchd too old to say). Only a REAL
+    # non-zero exit is a crash loop here; a clean or unknown exit is treated as
+    # an intended restart and falls through to the uptime check below, which
+    # still catches a genuinely unstable respawn (a crash-looping daemon's new
+    # pid is always younger than STABLE_S).
+    exit_code="$(daemon_last_exit)"
+    case "$exit_code" in
+    ""|"-"|"0")
+      # A clean/unknown exit with NO new pid at all is not a restart in
+      # progress, it is the daemon gone — the "no daemon process" case, not a
+      # crash loop and not healthy.
+      [ -n "$p2" ] || { HEALTH_REASON="daemon pid $p1 exited (last exit ${exit_code:-unknown}) and no replacement is running"; return 1; }
+      log "daemon pid changed within ${STABLE_GAP_S}s (${p1} -> ${p2}) but its last exit was ${exit_code:-unknown} — treating as an intended restart (deliberate kickstart), not a crash loop; checking the new pid's uptime"
+      p1="$p2"
+      ;;
+    *)
+      HEALTH_REASON="daemon pid changed within ${STABLE_GAP_S}s (${p1} -> ${p2:-none}), last exit ${exit_code} — crash loop"
+      return 1
+      ;;
+    esac
+  fi
   oldest=""
   for pid in $p1; do
     up="$(pid_uptime_s "$pid")"
@@ -450,19 +504,133 @@ revive_session(){ # a wedged front door cannot restart itself; restart it FOR it
   # Leave that one to session_reconciled, which already names it.
   label_loaded "$SESSION_LABEL" || return 0
   session_beating && return 0
-  log "front door $SESSION_LABEL is loaded but not beating (> ${SESSION_STALE_S}s) — kickstarting it rather than waiting for a session that cannot answer"
+  # Two-read hysteresis (see SESSION_REVIVE_RECHECK_S): the first stale read is
+  # not enough to drop a session, because a supervisor resume gaps the beat for
+  # a moment. Confirm the door is STILL silent on a second read 60s later
+  # before kickstarting — if it beat again in between, that was a resume gap,
+  # not a wedge, and the session keeps its context.
+  log "front door $SESSION_LABEL is loaded but not beating (> ${SESSION_STALE_S}s) — confirming over a second read ${SESSION_REVIVE_RECHECK_S}s apart before kickstarting"
+  sleep "$SESSION_REVIVE_RECHECK_S"
+  session_beating && { log "front door $SESSION_LABEL beat again on the second read — a resume gap, not a wedge; leaving it"; return 0; }
+  # Supervisor coordination via the resume loop's OWN lock (Jacob), not a soft
+  # note. The supervisor holds the `session` singleton (state/locks/process/
+  # session.pid) for its whole life and beats as a healthy resume progresses.
+  # We are past the FULL grace and two reads apart, so a resume gap is already
+  # excluded by time — a live holder here is a supervisor PARKED in its launch
+  # probe (the 3.5-day / 15-day case), for which the hard kickstart is right.
+  # Consulting the lock tells a parked supervisor (override, logged) from a dead
+  # one (clean restart); it never blocks the parked case a hard skip would.
+  local slock="$AGENT_DIR/state/locks/process/session.pid" spid=""
+  [ -f "$slock" ] && spid="$(head -n1 "$slock" 2>/dev/null | tr -dc '0-9')"
+  if [ -n "$spid" ] && kill -0 "$spid" 2>/dev/null; then
+    log "front door $SESSION_LABEL dark past grace while a supervisor (pid $spid) still holds the session lock — parked in its launch probe; hard-kickstarting over it"
+  fi
+  log "front door $SESSION_LABEL not beating on two reads ${SESSION_REVIVE_RECHECK_S}s apart — kickstarting it rather than waiting for a session that cannot answer"
   launchctl kickstart -k "gui/$(id -u)/$SESSION_LABEL" >> "$LOG" 2>&1 \
     && { log "front door $SESSION_LABEL restarted"; return 0; }
   log "WARN: could not kickstart $SESSION_LABEL (maestro session restart --force)"
   return 1
 }
 
+daemon_launchctl_state(){ # "pid" | "dash" | "absent" for DAEMON_LABEL, read from column 1 (PID) of `launchctl list` — a number is a live pid, "-" is loaded-with-no-pid, missing is absent
+  [ -n "$DAEMON_LABEL" ] || { printf 'absent'; return 0; }
+  launchctl list 2>/dev/null | awk -v l="$DAEMON_LABEL" '
+    NR>1 && $3==l { print ($1 ~ /^[0-9]+$/ ? "pid" : "dash"); f=1; exit }
+    END { if (!f) print "absent" }'
+}
+daemon_beat_age_ms(){ # age of the daemon dashboard beat file in ms; a large number when it is absent (never a small/fresh value)
+  node -e 'try { const s = require("fs").statSync(process.argv[1]); process.stdout.write(String(Math.max(0, Date.now() - s.mtimeMs))); } catch { process.stdout.write("999999999"); }' \
+    "$AGENT_DIR/state/dashboards/daemon-health.yaml" 2>/dev/null || printf '999999999'
+}
+REVIVE_NOTE_REL="state/telemetry/revive-note.json"
+write_revive_note(){ # $1=reason $2=target $3=detail — a failed revive must land ON THE BEAT (collect.mjs → machine.reviveNote), not only in this log; a person reading the fleet sees the failed escalation
+  local dir="$AGENT_DIR/state/telemetry" tmp
+  mkdir -p "$dir" 2>/dev/null || return 0
+  tmp="$(mktemp "$dir/.revive-note.XXXXXX" 2>/dev/null)" || return 0
+  printf '{\n  "reason": "%s",\n  "target": "%s",\n  "detail": "%s",\n  "at": "%s"\n}\n' \
+    "$(json_str "$1")" "$(json_str "$2")" "$(json_str "$3")" "$(date -u +%FT%TZ)" > "$tmp" \
+    && mv -f "$tmp" "$dir/$(basename "$REVIVE_NOTE_REL")"
+  return 0
+}
+daemon_launchctl_pid(){ # the numeric pid launchctl lists for DAEMON_LABEL (empty when it lists a dash or is absent)
+  [ -n "$DAEMON_LABEL" ] || return 0
+  launchctl list 2>/dev/null | awk -v l="$DAEMON_LABEL" 'NR>1 && $3==l { if ($1 ~ /^[0-9]+$/) print $1; exit }'
+}
+daemon_sample_json(){ # one settle-window observation: {"pid":N|null,"alive":bool,"startTime":"…"|null}. A LISTED pid is not a RUNNING one — kill -0 verifies; ps -o lstart fingerprints the instance so a relaunch (crash loop) shows a different start-time.
+  local pid start
+  pid="$(daemon_launchctl_pid)"
+  if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then
+    start="$(ps -o lstart= -p "$pid" 2>/dev/null | tr -d '\n"')"
+    printf '{"pid":%s,"alive":true,"startTime":"%s"}' "$pid" "$start"
+  elif [ -n "$pid" ]; then
+    printf '{"pid":%s,"alive":false,"startTime":null}' "$pid"   # phantom: launchctl lists it, ps says dead
+  else
+    printf '{"pid":null,"alive":false,"startTime":null}'
+  fi
+}
+revive_daemon(){ # the daemon-DOWN backstop. A daemon cannot restart ITSELF — a process alive to ask shows a live pid — so this hourly job is the on-host actor for shouldReviveDaemon (lib/session/revive.mjs). Jacob 2026-09-25: heartbeat.json ticking, no overdue handoff, daemon dead; visible on-host ONLY here.
+  # is_healthy bails at the empty-pid check before daemon_last_exit (its label
+  # resolver) runs, so resolve them here — the down case is exactly the one
+  # where nothing upstream did.
+  [ -n "$DAEMON_LABEL" ] || resolve_labels
+  [ -n "$DAEMON_LABEL" ] || return 0
+  [ -f "$LIB_REVIVE" ] || return 0
+  # ONLY the no-process case. A crash-looping, young, or beat-unhealthy daemon
+  # still has a pid — that is the health gate's business, and a kickstart would
+  # fight the very restart it is doing. pgrep-empty is Jacob's lane.
+  [ -z "$(daemon_pids)" ] || return 0
+  local st beat lpid palive verdict
+  st="$(daemon_launchctl_state)"
+  beat="$(daemon_beat_age_ms)"
+  # PHANTOM PID (Hannah): launchctl can list a pid whose process is dead.
+  # A listed pid is not a running daemon — verify it, and pass pidAlive so
+  # shouldReviveDaemon demotes a phantom to the dash gate instead of rubber-
+  # stamping it alive (which would deadlock: pgrep-empty says down, launchctl
+  # says pid, and nothing revives).
+  lpid="$(daemon_launchctl_pid)"
+  if [ -n "$lpid" ] && kill -0 "$lpid" 2>/dev/null; then palive="true"; else palive="false"; fi
+  verdict="$(node --input-type=module -e 'import { pathToFileURL } from "node:url"; const m = await import(pathToFileURL(process.argv[1]).href); const v = m.shouldReviveDaemon({ launchctl: process.argv[2], beatAgeMs: Number(process.argv[3]), staleMs: Number(process.argv[4]), pidAlive: process.argv[5] === "true" }); process.stdout.write(v.revive ? "revive" : ("no:" + v.reason));' "$LIB_REVIVE" "$st" "$beat" "$((DAEMON_STALE_S * 1000))" "$palive" 2>/dev/null)"
+  case "$verdict" in
+    revive) : ;;
+    *) log "daemon $DAEMON_LABEL down-check: launchctl=$st pidAlive=$palive beat=${beat}ms — not reviving (${verdict:-no})"; return 0 ;;
+  esac
+  log "daemon $DAEMON_LABEL is loaded but down (launchctl=$st pidAlive=$palive, daemon beat ${beat}ms stale) — kickstarting it; a daemon cannot restart itself, so the hourly backstop does"
+  launchctl kickstart -k "gui/$UID_/$DAEMON_LABEL" >> "$LOG" 2>&1 || { log "WARN: could not kickstart $DAEMON_LABEL"; write_revive_note "daemon-revive-failed" "$DAEMON_LABEL" "kickstart returned nonzero"; return 1; }
+  # ASSERT across a SETTLE WINDOW (Jacob, incident-validated): a one-shot read
+  # passes a phantom pid AND a come-up-then-crash whose local beat it keeps
+  # stamping. Two reads DAEMON_CONFIRM_SETTLE_S apart prove one continuous,
+  # live process (same pid, same start-time, alive on both) — the uptime
+  # continuity a crash loop cannot fake. serverBeatFresh is left unset here (an
+  # hourly backstop does not round-trip the org just to confirm); continuity
+  # carries it, and the daemon's own presence publish is the fleet-visible proof.
+  local s1 s2 samples ok
+  s1="$(daemon_sample_json)"
+  sleep "$DAEMON_CONFIRM_SETTLE_S"
+  s2="$(daemon_sample_json)"
+  samples="[$s1,$s2]"
+  ok="$(node --input-type=module -e 'import { pathToFileURL } from "node:url"; const m = await import(pathToFileURL(process.argv[1]).href); const v = m.confirmRevived({ samples: JSON.parse(process.argv[2]), staleMs: Number(process.argv[3]) }); process.stdout.write(v.ok ? "ok" : v.reason);' "$LIB_REVIVE" "$samples" "$((DAEMON_STALE_S * 1000))" 2>/dev/null)"
+  if [ "$ok" = "ok" ]; then
+    log "daemon $DAEMON_LABEL restarted and holding one continuous live pid across ${DAEMON_CONFIRM_SETTLE_S}s — the backstop revived it"
+    rm -f "$AGENT_DIR/$REVIVE_NOTE_REL" 2>/dev/null   # recovered — clear the beat's failed-revive note
+    return 0
+  fi
+  # ESCALATE WITH EVIDENCE, and STOP — no blind second kickstart (it would bury
+  # this failure's log under the loop). One attempt, then hand up with what it
+  # saw: the launchctl list line and the tail of the daemon's own log.
+  local lc_line dlog_tail evidence
+  lc_line="$(launchctl list 2>/dev/null | awk -v l="$DAEMON_LABEL" 'NR>1 && $3==l { print; exit }')"
+  dlog_tail="$(tail -n 3 "$DLOG_DIR/daemon-$(date +%Y-%m-%d).log" 2>/dev/null | tr '\n' '|')"
+  evidence="assert=${ok:-unknown} samples=${samples} launchctl=[${lc_line:-none}] dlog=[${dlog_tail:-none}]"
+  log "daemon $DAEMON_LABEL kickstart did NOT revive it (${ok:-unknown}) — a helper's exit code is not a revive; this seat needs a person (failure to escalate). ${evidence}"
+  write_revive_note "daemon-revive-failed" "$DAEMON_LABEL" "$evidence"
+  return 1
+}
 session_reconciled(){ # only asked when a -session plist was generated; never a rollback reason
   label_loaded "$SESSION_LABEL" || { log "reconcile-failed: session job $SESSION_LABEL is not loaded (maestro session start)"; return 1; }
   pgrep -f "$AGENT_DIR/scripts/session/supervisor" >/dev/null 2>&1 || { log "reconcile-failed: $SESSION_LABEL is loaded but no supervisor process is alive for $AGENT_DIR (maestro session status)"; return 1; }
   return 0
 }
-json_str(){ printf '%s' "$1" | tr -d '"\\' | tr '\n' ' '; } # a value safe inside "…" in the JSON files below
+json_str(){ printf '%s' "$1" | tr -d '"\\' | tr '\n\t\r' '   ' | tr -d '\000-\037'; } # a value safe inside "…" in the JSON files below — quotes/backslashes stripped, and EVERY control char (tabs from `launchctl list` columns, CR, others) folded to a space or removed, so an evidence string built from raw tool output never breaks JSON.parse
 write_notice(){ # $1 = from, $2 = to, [$3 = reason → healthy:false] — tell the front-door session; never restart it here
   local dir="$AGENT_DIR/state/session" tmp
   mkdir -p "$dir" 2>/dev/null || return 0
@@ -481,18 +649,133 @@ write_notice(){ # $1 = from, $2 = to, [$3 = reason → healthy:false] — tell t
 }
 write_upgrade_notice(){ write_notice "$1" "$2"; }
 LAST_JSON="$AGENT_DIR/state/autoupdate/last.json"
-write_last(){ # $1 from, $2 to, $3 ok, $4 healthy, $5 reason — state/autoupdate/last.json (→ beat machine.upgrade)
-  local dir="$AGENT_DIR/state/autoupdate" tmp
+# How many CONSECUTIVE failed attempts to reach @latest make a seat STUCK, and
+# at what point it stops being a warning. See the `failStreak` block below.
+STUCK_STREAK="${MAESTRO_AUTOUPDATE_STUCK_STREAK:-3}"
+# ── failStreak — "this seat cannot get to @latest" is a FACT, not a discovery ─
+# The fleet learned on 2026-09-25 that three seats had sat on 2.17.0 for days.
+# Nothing was broken in the sense anything reports: the launchd job fired every
+# hour, npm answered, the install ran, the health gate failed, the rollback
+# worked, last.json recorded it honestly, and the beat carried it. Each hour was
+# a correct, self-contained failure — and a correct failure repeated forty times
+# is a different fact from a correct failure once. Only the REPETITION says the
+# seat will never arrive on its own, and no single record can hold it, so the
+# count lives in last.json and rides the beat the rest of that record already
+# rides. There is no second channel here and there must not be one: the seats
+# this is about are, by construction, the ones running the oldest code.
+#
+# WHAT COUNTS AS AN ATTEMPT — the whole honesty of the number:
+#   · An ATTEMPT is a record whose `to` differs from its `from`: the upgrade
+#     path actually installed (or tried to install) a different version. Those
+#     are the only records that move the streak.
+#   · A FAILED attempt (`ok:false`) increments it and stamps `stuckSince` with
+#     the first failure of the run of failures, so a reader gets a DURATION and
+#     not merely a tally — "4 attempts since Monday" and "4 attempts in the last
+#     hour" are different seats.
+#   · A SUCCEEDED attempt (`ok:true`) clears all three fields. Arrival is the
+#     only thing that resets it; a fleet-wide publish does not, because a seat
+#     that fails 2.18.12, 2.18.13 and 2.18.14 in turn has failed three times to
+#     do the one thing asked of it, and restarting the count on every release
+#     would guarantee the number never reaches any threshold.
+#   · Everything else PRESERVES the fields untouched. A `to == from` record is
+#     a report on the installed version, not an attempt to leave it, so it must
+#     not move a count of ATTEMPTS in either direction. (Every such call site
+#     today sits on the up-to-date branch and therefore takes the arrival path
+#     below instead — this rule governs the writer, for the next caller that
+#     does not.) And a run that SKIPS writes no record at all, which is the
+#     same preservation by a shorter route. THIS IS THE POINT OF (d): a seat that has not been ASKED —
+#     nothing newer published, or held for the day after its last failure — is
+#     not a stuck seat, and must never accumulate a streak for sitting still.
+#     Only a real, completed, failed attempt does.
+#   · WHICH RUNS ACTUALLY SKIP — the exact list, because a wrong one was
+#     written here first. The paths that reach `exit 0` with NO write_last are:
+#     the failed-target hold, the overlap lock, the kill-switch, and a failed
+#     `npm view`. "SIMPLY BEING UP TO DATE" IS NOT AMONG THEM and never was —
+#     the up-to-date branch is the daemon health gate, and it calls write_last
+#     on every one of its outcomes (healthy, unhealthy-current,
+#     ahead-of-registry, recovered, and the stale-daemon kickstart). The
+#     original text claimed otherwise; nothing checked it, and the streak
+#     arithmetic on that branch is not what the sentence implied.
+#   · ARRIVAL IS BEING AT @latest, NOT MERELY INSTALLING IT. Every write on the
+#     up-to-date branch passes `at_latest=true`, which clears all three fields
+#     — because that branch is only entered when `CUR >= LATEST`, i.e. there is
+#     nothing left for this seat to reach. That covers the case the increment
+#     rule alone gets wrong: if a bad release is YANKED and @latest rolls back
+#     to the version a seat already runs, the seat stops attempting anything,
+#     so nothing would ever reset it and `upgrade_stuck` would fire forever
+#     (critical past 6) on a seat that is in fact perfectly current. It also
+#     makes the stale-daemon kickstart record honest rather than accidental:
+#     that hop really is an arrival.
+# The write is done in node rather than printf because the record now depends on
+# the record before it; a shell that reads a file it is about to overwrite gets
+# that wrong at exactly the moments it matters.
+write_last(){ # $1 from, $2 to, $3 ok, $4 healthy, $5 reason, [$6 at_latest] — state/autoupdate/last.json (→ beat machine.upgrade)
+  local dir="$AGENT_DIR/state/autoupdate" tmp streak at_latest="${6:-false}"
   mkdir -p "$dir" 2>/dev/null || return 0
   tmp="$(mktemp "$dir/.last.XXXXXX" 2>/dev/null)" || return 0
-  printf '{\n  "from": "%s",\n  "to": "%s",\n  "at": "%s",\n  "ok": %s,\n  "healthy": %s,\n  "reason": "%s"\n}\n' \
-    "$1" "$2" "$(date -u +%FT%TZ)" "$3" "$4" "$(json_str "$5")" > "$tmp" && mv -f "$tmp" "$LAST_JSON"
+  streak="$(node -e '
+    const fs = require("fs");
+    const [file, tmp, from, to, okS, healthyS, reason, stuckStreakS, atLatestS] = process.argv.slice(1);
+    const ok = okS === "true";
+    const at = new Date().toISOString().replace(/\.\d{3}Z$/, "Z");
+    let prev = null;
+    try { prev = JSON.parse(fs.readFileSync(file, "utf8")); } catch {}
+    if (!prev || typeof prev !== "object") prev = {};
+    const prevStreak = Number.isInteger(prev.failStreak) && prev.failStreak > 0 ? prev.failStreak : 0;
+    const attempt = from !== to;               // an upgrade was actually tried
+    const atLatest = atLatestS === "true";     // this seat is AT or past @latest — nothing left to reach
+    let failStreak, stuckSince, streakTarget;
+    if (atLatest || (attempt && ok)) {         // arrival, by install OR because @latest came back to us (a yank)
+      failStreak = 0; stuckSince = ""; streakTarget = "";
+    } else if (!attempt) {                     // a report on the installed version — carry the count, do not touch it
+      failStreak = prevStreak;
+      stuckSince = typeof prev.stuckSince === "string" ? prev.stuckSince : "";
+      streakTarget = typeof prev.streakTarget === "string" ? prev.streakTarget : "";
+    } else {
+      failStreak = prevStreak + 1;
+      stuckSince = typeof prev.stuckSince === "string" && prev.stuckSince ? prev.stuckSince : at;
+      streakTarget = to;
+    }
+    const rec = { from, to, at, ok, healthy: healthyS === "true", reason };
+    if (failStreak > 0) {
+      rec.failStreak = failStreak;
+      if (stuckSince) rec.stuckSince = stuckSince;
+      if (streakTarget) rec.streakTarget = streakTarget;
+    }
+    fs.writeFileSync(tmp, JSON.stringify(rec, null, 2) + "\n");
+    // stdout is the shell'"'"'s only view of what was decided: "<streak> <target> <since>"
+    process.stdout.write(failStreak >= Number(stuckStreakS) && attempt && !ok ? `STUCK ${failStreak} ${streakTarget} ${stuckSince}` : "");
+  ' "$LAST_JSON" "$tmp" "$1" "$2" "$3" "$4" "$5" "$STUCK_STREAK" "$at_latest" 2>/dev/null)" || streak=""
+  if [ -s "$tmp" ]; then
+    mv -f "$tmp" "$LAST_JSON"
+  else
+    # The node writer produced nothing. Before this branch existed the tmp was
+    # simply removed and last.json was LEFT AT ITS PREVIOUS VALUE — so a write
+    # failure did not lose a record, it published a STALE one, and the 24 h
+    # failed-target hold (which reads `to` and `reason` out of this file) then
+    # made its decision from a previous run's outcome while believing it was
+    # this run's. A missing record is fail-open by design here; a wrong one is
+    # not. So write the event itself with printf, which needs no interpreter,
+    # and say loudly that the count did not survive: the streak is the one
+    # field that cannot be reconstructed without reading the old file, and a
+    # silently reset counter is the failure this whole block exists to prevent.
+    log "WARN: write_last could not run node — recording the bare outcome without the failStreak. The count restarts from this run; see state/autoupdate/last.json."
+    printf '{\n  "from": "%s",\n  "to": "%s",\n  "at": "%s",\n  "ok": %s,\n  "healthy": %s,\n  "reason": "%s"\n}\n' \
+      "$1" "$2" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$3" "$4" "$(printf '%s' "$5" | sed 's/[\\"]/\\&/g')" > "$LAST_JSON" 2>/dev/null || true
+  fi
   rm -f "$tmp" 2>/dev/null
+  if [ -n "$streak" ]; then
+    set -- $streak
+    log "STUCK: $2 consecutive failed attempts to reach $3 (since $4). This seat will not arrive on its own — the org now carries machine.upgrade.failStreak=$2 and an \`upgrade_stuck\` alert on every beat. maestro doctor; then rm state/autoupdate/last.json to retry immediately."
+  fi
   return 0
 }
 last_reason(){ # the reason field of last.json, or nothing
   node -e 'try { process.stdout.write(String(JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")).reason || "")); } catch {}' "$LAST_JSON" 2>/dev/null
 }
+last_streak(){ # the failStreak of last.json as a positive integer, or nothing
+  node -e 'try { const n = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")).failStreak; if (Number.isInteger(n) && n > 0) process.stdout.write(String(n)); } catch {}' "$LAST_JSON" 2>/dev/null
+}
 # The version the RUNNING daemon started on. scripts/daemon/health.mjs resolves
 # sdk_version once at load and writes it to state/dashboards/daemon-health.yaml
 # with its pid, precisely so a reader can tell the installed package from the
@@ -595,13 +878,13 @@ if [ "$CUR" = "$LATEST" ] || [ "$(printf '%s\n%s\n' "$CUR" "$LATEST" | sort -V |
       # the notice tells the session to restart itself onto the new code.
       log "OK: daemon now on $CUR (was $STALE_FROM) — the manual upgrade is complete"
       write_upgrade_notice "$STALE_FROM" "$CUR"
-      write_last "$STALE_FROM" "$CUR" true true ""
+      write_last "$STALE_FROM" "$CUR" true true "" true
     elif [ -n "$STALE_FROM" ]; then
       # Same version, newer files: the daemon is reconciled; the session is
       # not told to restart (a touched or restored file must not cost it its
       # context — the version did not change).
       log "OK: daemon restarted onto the installed $CUR code; no session notice (same version)"
-      write_last "$CUR" "$CUR" true true ""
+      write_last "$CUR" "$CUR" true true "" true
     elif [ -n "$AHEAD" ]; then
       # The DAEMON is well; the FLEET is not. Recorded as ok/healthy — because
       # it is — with the finding in `reason`, so the org learns that this seat
@@ -609,23 +892,49 @@ if [ "$CUR" = "$LATEST" ] || [ "$(printf '%s\n%s\n' "$CUR" "$LATEST" | sort -V |
       # per episode: unlike an unhealthy daemon this does not clear itself, and
       # a fact that ages out of one log file is how six days went by.
       log "OK: healthy on $CUR, but AHEAD of the registry ($LATEST) — recorded for the org as ahead-of-registry"
-      write_last "$CUR" "$CUR" true true "ahead-of-registry: npm latest is $LATEST"
+      write_last "$CUR" "$CUR" true true "ahead-of-registry: npm latest is $LATEST" true
     else
-      case "$PREV_REASON" in unhealthy-current*)
+      case "$PREV_REASON" in
+      unhealthy-current*)
         log "recovered: last.json no longer reports unhealthy-current"
-        write_last "$CUR" "$CUR" true true "" ;;
+        write_last "$CUR" "$CUR" true true "" true ;;
       ahead-of-registry*)
         log "recovered: $CUR is on the registry now (latest $LATEST) — last.json no longer reports ahead-of-registry"
-        write_last "$CUR" "$CUR" true true "" ;;
+        write_last "$CUR" "$CUR" true true "" true ;;
+      *)
+        # A healthy seat that is simply up to date writes NOTHING — there is
+        # nothing to report and an empty last.json is the honest state.
+        #
+        # The one exception is a standing failStreak, and it is the reason this
+        # branch exists at all. A streak only ever reset on ARRIVAL, i.e. on a
+        # successful hop. If a bad release is YANKED and @latest rolls back to
+        # the version this seat already runs, the seat stops attempting
+        # anything: every hour it takes this quiet path, writes nothing, and the
+        # count stands for ever — `upgrade_stuck` firing critical on a seat that
+        # is, in fact, exactly where the registry wants it. Being AT @latest is
+        # an arrival however you got there, so record one and say so.
+        STREAK="$(last_streak)"
+        if [ -n "$STREAK" ]; then
+          log "streak cleared: $CUR is @latest and this seat is healthy on it, but last.json still carried failStreak=$STREAK — @latest has come back to this version (a yank or a withdrawn release). Recording the arrival so the upgrade_stuck alert stops."
+          write_last "$CUR" "$CUR" true true "" true
+        fi ;;
       esac
     fi
   else
+    # Jacob's daemon-down lane: up to date, but the daemon is loaded with NO
+    # running process while the front door kept beating — the seat looked "up
+    # to date" every hour and was dark. The daemon cannot restart itself, so
+    # revive_daemon (the on-host actor for shouldReviveDaemon) kickstarts it
+    # here and asserts the pid came back. Gated to the no-process case, so a
+    # crash-looping or beat-unhealthy daemon (which still has a pid) is left to
+    # the gate above, not kickstarted.
+    revive_daemon || true
     log "UNHEALTHY on $CUR${STALE_FROM:+ (kickstarted from $STALE_FROM)}: $HEALTH_REASON — no upgrade to roll back; the org cannot see this seat. maestro doctor"
     # The `unhealthy-current:` PREFIX is load-bearing (the episode check below
     # and failed_hold_reason both match on it), so the ahead-of-registry fact
     # is appended, never prepended — an unpublished build that is also sick is
     # the worst case and must say both things.
-    write_last "$CUR" "$CUR" false false "unhealthy-current: $HEALTH_REASON${AHEAD:+ (and ahead-of-registry: npm latest is $LATEST)}"
+    write_last "$CUR" "$CUR" false false "unhealthy-current: $HEALTH_REASON${AHEAD:+ (and ahead-of-registry: npm latest is $LATEST)}" true
     case "$PREV_REASON" in
       unhealthy-current*) log "session already notified this episode (last.json was unhealthy-current); notice not rewritten" ;;
       *) write_notice "$CUR" "$CUR" "unhealthy-current: $HEALTH_REASON" ;;
@@ -644,10 +953,19 @@ fi
 # nothing.
 FAILED_HOLD_S="${MAESTRO_AUTOUPDATE_FAILED_HOLD_S:-86400}"
 # How long a front door may go without beating before this run restarts it FOR
-# it. Ten minutes: long enough that a busy session is never interrupted (the
-# heartbeat rides every tool use), short enough that a wedged one is measured in
-# minutes rather than the days it took to notice the last three.
-SESSION_STALE_S="${MAESTRO_SESSION_STALE_S:-600}"
+# it. 90s — the same bar as STABLE_S, the other 90s observer in this gate — so
+# a wedged front door is measured in the same window as a wedged daemon rather
+# than the days it took to notice the last three. `revive_session` (this run,
+# hourly) is the BACKSTOP; it does not need its own, looser number.
+SESSION_STALE_S="${MAESTRO_SESSION_STALE_S:-90}"
+# The daemon confirms a shut front door over TWO reads before it kickstarts
+# (lib/session/revive.mjs DEFAULT_REVIVE_CONFIRM_READS, REVIVE_CHECK_INTERVAL_MS
+# 60s apart). `revive_session`, the hourly backstop, now does the same: a
+# supervisor resume (`claude --resume`) gaps the heartbeat between the old feed
+# dying and the new one re-arming, and a single stale read landing inside that
+# gap must not kickstart a session that is mid-resume. The two reads are 60s
+# apart; the hourly backstop can afford the one wait.
+SESSION_REVIVE_RECHECK_S="${MAESTRO_SESSION_REVIVE_RECHECK_S:-60}"
 failed_hold_reason(){ # prints "<reason> at <at>" when LATEST failed health within FAILED_HOLD_S
   node -e '
     const [file, latest, holdS] = process.argv.slice(1);