@cohortapp/agent-sdk 2.18.15 → 2.18.16

package/scripts/local-triggers/autoupdate.sh

@@ -43,7 +43,17 @@
 #       STABLE_S apart AND every pid's `ps etime` is ≥ STABLE_S. STABLE_S is 90 s
 #       (MAESTRO_AUTOUPDATE_STABLE_S): 3× the observed ~25 s crash interval, so a
 #       loop cannot straddle both samples by luck, and well inside the hourly
-#       cadence. A 25 s loop fails on the second sample and on uptime.
+#       cadence. A 25 s loop fails on the second sample and on uptime. A pid
+#       CHANGE within the window is not, on its own, proof of that loop: this
+#       script's own restart_daemon (or an operator's `launchctl kickstart -k`)
+#       changes the pid deliberately and can land inside the same window —
+#       2.18.14 read that as a crash loop and rolled back a release that had
+#       done nothing wrong. daemon_last_exit() tells the two apart: a crash
+#       loop's reaped exit is a real non-zero every time; a deliberate
+#       kickstart's is 0 (or "-"/unknown, for a launchd too old to say). Only a
+#       real non-zero exit fails here on the pid change alone — a clean or
+#       unknown exit falls through to the uptime check, which still catches a
+#       genuinely unstable respawn (its pid is always younger than STABLE_S).
 #   (b) NO FATAL LINE in the daemon's own log since THIS daemon booted: the
 #       lines lib/util/unhandled.mjs and agent-daemon.mjs write on the way down
 #       ("[DAEMON] uncaughtException:", "[daemon] Fatal:"), the wrapper's
@@ -270,6 +280,18 @@ FATAL_RE='\[DAEMON\] uncaughtException:|\[daemon\] Fatal:|\[wrapper\] FATAL:|ERR
 HEALTH_REASON=""   # set by is_healthy on failure; a phrase, no quotes
 DAEMON_LABEL=""
 SESSION_LABEL=""
+# The pure daemon-revive decision (lib/session/revive.mjs) run by revive_daemon
+# below — the on-host backstop for a daemon that is DOWN, which a live daemon
+# can never observe about itself. Defined up here (not with SESSION_STALE_S,
+# which lives in the not-up-to-date path) because revive_daemon fires on the
+# UP-TO-DATE unhealthy branch too. DAEMON_STALE_S mirrors SESSION_STALE_S / the
+# 90s front-door observers.
+LIB_REVIVE="$AGENT_DIR/lib/session/revive.mjs"
+DAEMON_STALE_S="${MAESTRO_DAEMON_STALE_S:-90}"
+# Gap between the two settle-window reads that prove the kickstarted daemon is
+# one continuous process, not a come-up-then-crash loop. A one-shot read would
+# pass a phantom pid AND a crash loop whose local beat file it keeps re-stamping.
+DAEMON_CONFIRM_SETTLE_S="${MAESTRO_DAEMON_CONFIRM_SETTLE_S:-5}"
 
 daemon_pids(){ # every pid whose argv IS this agent's daemon entry (see DAEMON_ARGV_RE), sorted, space-joined ("" when none)
   pgrep -f "$DAEMON_ARGV_RE" 2>/dev/null | sort -n | tr '\n' ' ' | sed 's/ *$//'
@@ -355,14 +377,46 @@ sibling_job_audit(){ # log-only: every OTHER ai.maestro.<first>-* job whose LAST
   [ -n "$bad" ] && log "sibling jobs with a non-zero last exit (not a gate reason; see logs/launchd and logs/polling): $bad"
   return 0
 }
+daemon_last_exit(){ # the DAEMON_LABEL's last exit status (column 2 of `launchctl list`, the same column sibling_job_audit reads), or nothing when the label is unknown or not in the list
+  [ -n "$DAEMON_LABEL" ] || resolve_labels   # is_healthy on the up-to-date/not-stale path never called restart_daemon, so labels may still be unresolved here
+  [ -n "$DAEMON_LABEL" ] || return 0
+  launchctl list 2>/dev/null | awk -v l="$DAEMON_LABEL" 'NR>1 && $3==l { print $2; exit }'
+}
 is_healthy(){ # (a)+(b)+(c) above. Sets HEALTH_REASON on failure.
-  local p1 p2 pid up oldest oldest_pid dlog fatal why offset boot
+  local p1 p2 pid up oldest oldest_pid dlog fatal why offset boot exit_code
   HEALTH_REASON=""
   p1="$(daemon_pids)"
   [ -n "$p1" ] || { HEALTH_REASON="no daemon process for $DAEMON_PATTERN"; return 1; }
   [ "$STABLE_GAP_S" -gt 0 ] && sleep "$STABLE_GAP_S"
   p2="$(daemon_pids)"
-  [ "$p1" = "$p2" ] || { HEALTH_REASON="daemon pid changed within ${STABLE_GAP_S}s (${p1} -> ${p2:-none}) — crash loop"; return 1; }
+  if [ "$p1" != "$p2" ]; then
+    # A pid change alone is not proof of a crash loop: `launchctl kickstart -k`
+    # (this same script's own restart_daemon, or an operator's) changes the pid
+    # deliberately and can land inside this very window. 2.18.14 read that as a
+    # crash loop and rolled back a release that had done nothing wrong. The
+    # daemon's OWN last exit says which one happened: a crash-looping process
+    # exits non-zero every time launchd reaps it; a clean kickstart's reaped
+    # exit is 0 (or "-"/unknown, for a launchd too old to say). Only a REAL
+    # non-zero exit is a crash loop here; a clean or unknown exit is treated as
+    # an intended restart and falls through to the uptime check below, which
+    # still catches a genuinely unstable respawn (a crash-looping daemon's new
+    # pid is always younger than STABLE_S).
+    exit_code="$(daemon_last_exit)"
+    case "$exit_code" in
+    ""|"-"|"0")
+      # A clean/unknown exit with NO new pid at all is not a restart in
+      # progress, it is the daemon gone — the "no daemon process" case, not a
+      # crash loop and not healthy.
+      [ -n "$p2" ] || { HEALTH_REASON="daemon pid $p1 exited (last exit ${exit_code:-unknown}) and no replacement is running"; return 1; }
+      log "daemon pid changed within ${STABLE_GAP_S}s (${p1} -> ${p2}) but its last exit was ${exit_code:-unknown} — treating as an intended restart (deliberate kickstart), not a crash loop; checking the new pid's uptime"
+      p1="$p2"
+      ;;
+    *)
+      HEALTH_REASON="daemon pid changed within ${STABLE_GAP_S}s (${p1} -> ${p2:-none}), last exit ${exit_code} — crash loop"
+      return 1
+      ;;
+    esac
+  fi
   oldest=""
   for pid in $p1; do
     up="$(pid_uptime_s "$pid")"
@@ -450,19 +504,133 @@ revive_session(){ # a wedged front door cannot restart itself; restart it FOR it
   # Leave that one to session_reconciled, which already names it.
   label_loaded "$SESSION_LABEL" || return 0
   session_beating && return 0
-  log "front door $SESSION_LABEL is loaded but not beating (> ${SESSION_STALE_S}s) — kickstarting it rather than waiting for a session that cannot answer"
+  # Two-read hysteresis (see SESSION_REVIVE_RECHECK_S): the first stale read is
+  # not enough to drop a session, because a supervisor resume gaps the beat for
+  # a moment. Confirm the door is STILL silent on a second read 60s later
+  # before kickstarting — if it beat again in between, that was a resume gap,
+  # not a wedge, and the session keeps its context.
+  log "front door $SESSION_LABEL is loaded but not beating (> ${SESSION_STALE_S}s) — confirming over a second read ${SESSION_REVIVE_RECHECK_S}s apart before kickstarting"
+  sleep "$SESSION_REVIVE_RECHECK_S"
+  session_beating && { log "front door $SESSION_LABEL beat again on the second read — a resume gap, not a wedge; leaving it"; return 0; }
+  # Supervisor coordination via the resume loop's OWN lock (Jacob), not a soft
+  # note. The supervisor holds the `session` singleton (state/locks/process/
+  # session.pid) for its whole life and beats as a healthy resume progresses.
+  # We are past the FULL grace and two reads apart, so a resume gap is already
+  # excluded by time — a live holder here is a supervisor PARKED in its launch
+  # probe (the 3.5-day / 15-day case), for which the hard kickstart is right.
+  # Consulting the lock tells a parked supervisor (override, logged) from a dead
+  # one (clean restart); it never blocks the parked case a hard skip would.
+  local slock="$AGENT_DIR/state/locks/process/session.pid" spid=""
+  [ -f "$slock" ] && spid="$(head -n1 "$slock" 2>/dev/null | tr -dc '0-9')"
+  if [ -n "$spid" ] && kill -0 "$spid" 2>/dev/null; then
+    log "front door $SESSION_LABEL dark past grace while a supervisor (pid $spid) still holds the session lock — parked in its launch probe; hard-kickstarting over it"
+  fi
+  log "front door $SESSION_LABEL not beating on two reads ${SESSION_REVIVE_RECHECK_S}s apart — kickstarting it rather than waiting for a session that cannot answer"
   launchctl kickstart -k "gui/$(id -u)/$SESSION_LABEL" >> "$LOG" 2>&1 \
     && { log "front door $SESSION_LABEL restarted"; return 0; }
   log "WARN: could not kickstart $SESSION_LABEL (maestro session restart --force)"
   return 1
 }
 
+daemon_launchctl_state(){ # "pid" | "dash" | "absent" for DAEMON_LABEL, read from column 1 (PID) of `launchctl list` — a number is a live pid, "-" is loaded-with-no-pid, missing is absent
+  [ -n "$DAEMON_LABEL" ] || { printf 'absent'; return 0; }
+  launchctl list 2>/dev/null | awk -v l="$DAEMON_LABEL" '
+    NR>1 && $3==l { print ($1 ~ /^[0-9]+$/ ? "pid" : "dash"); f=1; exit }
+    END { if (!f) print "absent" }'
+}
+daemon_beat_age_ms(){ # age of the daemon dashboard beat file in ms; a large number when it is absent (never a small/fresh value)
+  node -e 'try { const s = require("fs").statSync(process.argv[1]); process.stdout.write(String(Math.max(0, Date.now() - s.mtimeMs))); } catch { process.stdout.write("999999999"); }' \
+    "$AGENT_DIR/state/dashboards/daemon-health.yaml" 2>/dev/null || printf '999999999'
+}
+REVIVE_NOTE_REL="state/telemetry/revive-note.json"
+write_revive_note(){ # $1=reason $2=target $3=detail — a failed revive must land ON THE BEAT (collect.mjs → machine.reviveNote), not only in this log; a person reading the fleet sees the failed escalation
+  local dir="$AGENT_DIR/state/telemetry" tmp
+  mkdir -p "$dir" 2>/dev/null || return 0
+  tmp="$(mktemp "$dir/.revive-note.XXXXXX" 2>/dev/null)" || return 0
+  printf '{\n  "reason": "%s",\n  "target": "%s",\n  "detail": "%s",\n  "at": "%s"\n}\n' \
+    "$(json_str "$1")" "$(json_str "$2")" "$(json_str "$3")" "$(date -u +%FT%TZ)" > "$tmp" \
+    && mv -f "$tmp" "$dir/$(basename "$REVIVE_NOTE_REL")"
+  return 0
+}
+daemon_launchctl_pid(){ # the numeric pid launchctl lists for DAEMON_LABEL (empty when it lists a dash or is absent)
+  [ -n "$DAEMON_LABEL" ] || return 0
+  launchctl list 2>/dev/null | awk -v l="$DAEMON_LABEL" 'NR>1 && $3==l { if ($1 ~ /^[0-9]+$/) print $1; exit }'
+}
+daemon_sample_json(){ # one settle-window observation: {"pid":N|null,"alive":bool,"startTime":"…"|null}. A LISTED pid is not a RUNNING one — kill -0 verifies; ps -o lstart fingerprints the instance so a relaunch (crash loop) shows a different start-time.
+  local pid start
+  pid="$(daemon_launchctl_pid)"
+  if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then
+    start="$(ps -o lstart= -p "$pid" 2>/dev/null | tr -d '\n"')"
+    printf '{"pid":%s,"alive":true,"startTime":"%s"}' "$pid" "$start"
+  elif [ -n "$pid" ]; then
+    printf '{"pid":%s,"alive":false,"startTime":null}' "$pid"   # phantom: launchctl lists it, ps says dead
+  else
+    printf '{"pid":null,"alive":false,"startTime":null}'
+  fi
+}
+revive_daemon(){ # the daemon-DOWN backstop. A daemon cannot restart ITSELF — a process alive to ask shows a live pid — so this hourly job is the on-host actor for shouldReviveDaemon (lib/session/revive.mjs). Jacob 2026-09-25: heartbeat.json ticking, no overdue handoff, daemon dead; visible on-host ONLY here.
+  # is_healthy bails at the empty-pid check before daemon_last_exit (its label
+  # resolver) runs, so resolve them here — the down case is exactly the one
+  # where nothing upstream did.
+  [ -n "$DAEMON_LABEL" ] || resolve_labels
+  [ -n "$DAEMON_LABEL" ] || return 0
+  [ -f "$LIB_REVIVE" ] || return 0
+  # ONLY the no-process case. A crash-looping, young, or beat-unhealthy daemon
+  # still has a pid — that is the health gate's business, and a kickstart would
+  # fight the very restart it is doing. pgrep-empty is Jacob's lane.
+  [ -z "$(daemon_pids)" ] || return 0
+  local st beat lpid palive verdict
+  st="$(daemon_launchctl_state)"
+  beat="$(daemon_beat_age_ms)"
+  # PHANTOM PID (Hannah): launchctl can list a pid whose process is dead.
+  # A listed pid is not a running daemon — verify it, and pass pidAlive so
+  # shouldReviveDaemon demotes a phantom to the dash gate instead of rubber-
+  # stamping it alive (which would deadlock: pgrep-empty says down, launchctl
+  # says pid, and nothing revives).
+  lpid="$(daemon_launchctl_pid)"
+  if [ -n "$lpid" ] && kill -0 "$lpid" 2>/dev/null; then palive="true"; else palive="false"; fi
+  verdict="$(node --input-type=module -e 'import { pathToFileURL } from "node:url"; const m = await import(pathToFileURL(process.argv[1]).href); const v = m.shouldReviveDaemon({ launchctl: process.argv[2], beatAgeMs: Number(process.argv[3]), staleMs: Number(process.argv[4]), pidAlive: process.argv[5] === "true" }); process.stdout.write(v.revive ? "revive" : ("no:" + v.reason));' "$LIB_REVIVE" "$st" "$beat" "$((DAEMON_STALE_S * 1000))" "$palive" 2>/dev/null)"
+  case "$verdict" in
+    revive) : ;;
+    *) log "daemon $DAEMON_LABEL down-check: launchctl=$st pidAlive=$palive beat=${beat}ms — not reviving (${verdict:-no})"; return 0 ;;
+  esac
+  log "daemon $DAEMON_LABEL is loaded but down (launchctl=$st pidAlive=$palive, daemon beat ${beat}ms stale) — kickstarting it; a daemon cannot restart itself, so the hourly backstop does"
+  launchctl kickstart -k "gui/$UID_/$DAEMON_LABEL" >> "$LOG" 2>&1 || { log "WARN: could not kickstart $DAEMON_LABEL"; write_revive_note "daemon-revive-failed" "$DAEMON_LABEL" "kickstart returned nonzero"; return 1; }
+  # ASSERT across a SETTLE WINDOW (Jacob, incident-validated): a one-shot read
+  # passes a phantom pid AND a come-up-then-crash whose local beat it keeps
+  # stamping. Two reads DAEMON_CONFIRM_SETTLE_S apart prove one continuous,
+  # live process (same pid, same start-time, alive on both) — the uptime
+  # continuity a crash loop cannot fake. serverBeatFresh is left unset here (an
+  # hourly backstop does not round-trip the org just to confirm); continuity
+  # carries it, and the daemon's own presence publish is the fleet-visible proof.
+  local s1 s2 samples ok
+  s1="$(daemon_sample_json)"
+  sleep "$DAEMON_CONFIRM_SETTLE_S"
+  s2="$(daemon_sample_json)"
+  samples="[$s1,$s2]"
+  ok="$(node --input-type=module -e 'import { pathToFileURL } from "node:url"; const m = await import(pathToFileURL(process.argv[1]).href); const v = m.confirmRevived({ samples: JSON.parse(process.argv[2]), staleMs: Number(process.argv[3]) }); process.stdout.write(v.ok ? "ok" : v.reason);' "$LIB_REVIVE" "$samples" "$((DAEMON_STALE_S * 1000))" 2>/dev/null)"
+  if [ "$ok" = "ok" ]; then
+    log "daemon $DAEMON_LABEL restarted and holding one continuous live pid across ${DAEMON_CONFIRM_SETTLE_S}s — the backstop revived it"
+    rm -f "$AGENT_DIR/$REVIVE_NOTE_REL" 2>/dev/null   # recovered — clear the beat's failed-revive note
+    return 0
+  fi
+  # ESCALATE WITH EVIDENCE, and STOP — no blind second kickstart (it would bury
+  # this failure's log under the loop). One attempt, then hand up with what it
+  # saw: the launchctl list line and the tail of the daemon's own log.
+  local lc_line dlog_tail evidence
+  lc_line="$(launchctl list 2>/dev/null | awk -v l="$DAEMON_LABEL" 'NR>1 && $3==l { print; exit }')"
+  dlog_tail="$(tail -n 3 "$DLOG_DIR/daemon-$(date +%Y-%m-%d).log" 2>/dev/null | tr '\n' '|')"
+  evidence="assert=${ok:-unknown} samples=${samples} launchctl=[${lc_line:-none}] dlog=[${dlog_tail:-none}]"
+  log "daemon $DAEMON_LABEL kickstart did NOT revive it (${ok:-unknown}) — a helper's exit code is not a revive; this seat needs a person (failure to escalate). ${evidence}"
+  write_revive_note "daemon-revive-failed" "$DAEMON_LABEL" "$evidence"
+  return 1
+}
 session_reconciled(){ # only asked when a -session plist was generated; never a rollback reason
   label_loaded "$SESSION_LABEL" || { log "reconcile-failed: session job $SESSION_LABEL is not loaded (maestro session start)"; return 1; }
   pgrep -f "$AGENT_DIR/scripts/session/supervisor" >/dev/null 2>&1 || { log "reconcile-failed: $SESSION_LABEL is loaded but no supervisor process is alive for $AGENT_DIR (maestro session status)"; return 1; }
   return 0
 }
-json_str(){ printf '%s' "$1" | tr -d '"\\' | tr '\n' ' '; } # a value safe inside "…" in the JSON files below
+json_str(){ printf '%s' "$1" | tr -d '"\\' | tr '\n\t\r' '   ' | tr -d '\000-\037'; } # a value safe inside "…" in the JSON files below — quotes/backslashes stripped, and EVERY control char (tabs from `launchctl list` columns, CR, others) folded to a space or removed, so an evidence string built from raw tool output never breaks JSON.parse
 write_notice(){ # $1 = from, $2 = to, [$3 = reason → healthy:false] — tell the front-door session; never restart it here
   local dir="$AGENT_DIR/state/session" tmp
   mkdir -p "$dir" 2>/dev/null || return 0
@@ -753,6 +921,14 @@ if [ "$CUR" = "$LATEST" ] || [ "$(printf '%s\n%s\n' "$CUR" "$LATEST" | sort -V |
       esac
     fi
   else
+    # Jacob's daemon-down lane: up to date, but the daemon is loaded with NO
+    # running process while the front door kept beating — the seat looked "up
+    # to date" every hour and was dark. The daemon cannot restart itself, so
+    # revive_daemon (the on-host actor for shouldReviveDaemon) kickstarts it
+    # here and asserts the pid came back. Gated to the no-process case, so a
+    # crash-looping or beat-unhealthy daemon (which still has a pid) is left to
+    # the gate above, not kickstarted.
+    revive_daemon || true
     log "UNHEALTHY on $CUR${STALE_FROM:+ (kickstarted from $STALE_FROM)}: $HEALTH_REASON — no upgrade to roll back; the org cannot see this seat. maestro doctor"
     # The `unhealthy-current:` PREFIX is load-bearing (the episode check below
     # and failed_hold_reason both match on it), so the ahead-of-registry fact
@@ -777,10 +953,19 @@ fi
 # nothing.
 FAILED_HOLD_S="${MAESTRO_AUTOUPDATE_FAILED_HOLD_S:-86400}"
 # How long a front door may go without beating before this run restarts it FOR
-# it. Ten minutes: long enough that a busy session is never interrupted (the
-# heartbeat rides every tool use), short enough that a wedged one is measured in
-# minutes rather than the days it took to notice the last three.
-SESSION_STALE_S="${MAESTRO_SESSION_STALE_S:-600}"
+# it. 90s — the same bar as STABLE_S, the other 90s observer in this gate — so
+# a wedged front door is measured in the same window as a wedged daemon rather
+# than the days it took to notice the last three. `revive_session` (this run,
+# hourly) is the BACKSTOP; it does not need its own, looser number.
+SESSION_STALE_S="${MAESTRO_SESSION_STALE_S:-90}"
+# The daemon confirms a shut front door over TWO reads before it kickstarts
+# (lib/session/revive.mjs DEFAULT_REVIVE_CONFIRM_READS, REVIVE_CHECK_INTERVAL_MS
+# 60s apart). `revive_session`, the hourly backstop, now does the same: a
+# supervisor resume (`claude --resume`) gaps the heartbeat between the old feed
+# dying and the new one re-arming, and a single stale read landing inside that
+# gap must not kickstart a session that is mid-resume. The two reads are 60s
+# apart; the hourly backstop can afford the one wait.
+SESSION_REVIVE_RECHECK_S="${MAESTRO_SESSION_REVIVE_RECHECK_S:-60}"
 failed_hold_reason(){ # prints "<reason> at <at>" when LATEST failed health within FAILED_HOLD_S
   node -e '
     const [file, latest, holdS] = process.argv.slice(1);