@cohortapp/agent-sdk 2.16.0 → 2.18.4

package/scripts/local-triggers/autoupdate.sh

@@ -30,19 +30,101 @@
 #     state/autoupdate/last.json remembers it and the run holds for 24 h
 #     (MAESTRO_AUTOUPDATE_FAILED_HOLD_S; delete last.json to retry now).
 #
-# The health gate is the DAEMON: alive + a fresh "org-mesh connected" /
-# "[daemon] Running" line. When the generated plist set carries a -session
-# job, the gate ALSO checks the session label is loaded and a supervisor is
-# alive — but a missing session is logged as `reconcile-failed` and does NOT
-# roll back (the daemon's --print lane is the front door meanwhile; doctor
-# and `maestro upgrade --verify` say what to do).
+# THE HEALTH GATE means HEALTH, not "running". It used to be "daemon pid alive
+# + a fresh 'org-mesh connected' / '[daemon] Running' line in today's log" —
+# and a CRASH-LOOPING daemon satisfies both: launchd respawns it, so it is
+# alive at any instant, and every respawn prints the connect line again. One
+# seat logged "up to date (2.17.0)" every hour for SIX DAYS (2026-09-15..21)
+# while its daemon died ~25 s after every start; the org saw no presence beat
+# from it for ten days and nothing on the box said a word. The gate now asks
+# four questions, and the daemon must answer all of them (is_healthy):
+#
+#   (a) PID STABILITY — the set of daemon pids is unchanged across two samples
+#       STABLE_S apart AND every pid's `ps etime` is ≥ STABLE_S. STABLE_S is 90 s
+#       (MAESTRO_AUTOUPDATE_STABLE_S): 3× the observed ~25 s crash interval, so a
+#       loop cannot straddle both samples by luck, and well inside the hourly
+#       cadence. A 25 s loop fails on the second sample and on uptime.
+#   (b) NO FATAL LINE in the daemon's own log since THIS daemon booted: the
+#       lines lib/util/unhandled.mjs and agent-daemon.mjs write on the way down
+#       ("[DAEMON] uncaughtException:", "[daemon] Fatal:"), the wrapper's
+#       "[wrapper] FATAL:", and Node's ERR_MODULE_NOT_FOUND (a bad release's
+#       import failure). The log carries no per-line timestamps, the wrapper
+#       names the file for the day the process STARTED and APPENDS to it, so
+#       one file holds every incarnation that started that day. The scan
+#       starts at the later of: the line count captured before a kickstart,
+#       and the running daemon's own boot marker — "[DAEMON] boot pid=<pid>"
+#       (maestro-daemon.mjs, from the release that added this gate), else the last "[daemon] Running" /
+#       "org-mesh connected" line. A crash earlier the same day that launchd
+#       (or an operator) already recovered from is history, not health. A
+#       missing log file has no fatal lines (fail-open here — (a) and (c)
+#       carry the gate).
+#   (c) A SERVER-ACKNOWLEDGED BEAT — state/org/last-beat.json {at, ok, code?,
+#       sdkVersion?}, which lib/org/mesh.mjs overwrites on EVERY presence beat
+#       (ok:true only when the org answered 2xx; a rejected or unreachable beat
+#       writes ok:false). The record must say ok:true, be dated after the
+#       daemon's start, and be younger than BEAT_FRESH_S (300 s,
+#       MAESTRO_AUTOUPDATE_BEAT_FRESH_S — the same 5 min after which hq's
+#       presence view calls a seat offline). When the record carries
+#       `sdkVersion` — the version that beat TOLD the org, which hq persists to
+#       AgentStatus.machine.sdkVersion — it must equal the installed version:
+#       an accepted beat that reported the previous version proves the old
+#       process is alive, not that the org sees the new one. A record without
+#       the field (a mesh older than this check, or a beat whose status
+#       collector failed) is liveness proof only and is accepted as before.
+#       Skipped only when config/org.yaml does not enroll this seat (an
+#       un-enrolled daemon never beats).
+#   (d) The UP-TO-DATE path is not exempt. When the installed version is already
+#       @latest the run still runs (a)-(c); an unhealthy daemon is logged
+#       loudly, written to state/autoupdate/last.json {healthy:false, reason:
+#       "unhealthy-current: …"} (the beat carries it as machine.upgrade), and —
+#       once per unhealthy episode, on the transition — announced through
+#       state/session/upgrade-notice.json {from:CUR,to:CUR,healthy:false,reason}
+#       so the front-door session's feed surfaces it as a `daemon-unhealthy`
+#       directive (never `upgrade-available` — nothing to restart onto).
+#       Nothing is rolled back: there is no upgrade to undo. A recovery
+#       restores last.json.
+#   (f) AHEAD OF THE REGISTRY is reported, not silently read as current. When
+#       the installed version is strictly GREATER than npm's latest, this build
+#       was never published: it exists on this box and nowhere else, and every
+#       other seat — which follows @latest — will never receive it. The
+#       up-to-date branch used to swallow that case into "up to date (<CUR>)".
+#       That is exactly how THIS gate came to be running on zero machines: the
+#       seat that wrote it installed 2.18.0 locally, npm's latest stayed at
+#       2.17.0, and fifteen boxes logged a reassuring hourly line while the
+#       crash-loop gate reached none of them. Now it is logged loudly and
+#       written to state/autoupdate/last.json {healthy:true, reason:
+#       "ahead-of-registry: npm latest is <LATEST>"} — healthy because the
+#       DAEMON is, reason because the FLEET is not — which the beat carries to
+#       the org as machine.upgrade. It never blocks, never rolls back and never
+#       writes a session notice: there is no upgrade to restart onto, and the
+#       fix (publish, or roll this seat back) is a human's.
+#   (e) A STALE DAEMON is reconciled on the up-to-date path. A hand-run
+#       `maestro upgrade` installs the package and reconciles the seat but
+#       never restarts the daemon (post-steps.mjs: "restarted by autoupdate,
+#       never here"; its verify row promises the new version "takes effect
+#       when autoupdate kickstarts it"). When state/dashboards/daemon-health.yaml
+#       says the LIVE daemon (its pid is one of ours) started on a different
+#       sdk_version than the installed one, this run kickstarts it, waits, and
+#       runs the gate exactly as after an automatic install — healthy →
+#       upgrade-notice {from:<old>,to:CUR} + last.json, as for any completed
+#       hop. Without this a manual upgrade ran old daemon code until the next
+#       release, and the gate misread its missing beat record as unhealthy.
+#
+# When the generated plist set carries a -session job, the gate ALSO checks the
+# session label is loaded and a supervisor is alive — but a missing session is
+# logged as `reconcile-failed` and does NOT roll back (the daemon's --print lane
+# is the front door meanwhile; doctor and `maestro upgrade --verify` say what
+# to do).
 #
 # The front-door session (ai.maestro.<first>-session) is NEVER restarted or
 # killed here: after a HEALTHY upgrade this writes
-# state/session/upgrade-notice.json {from,to,at}; the session's feed surfaces
-# it as an `upgrade-available` directive and the session restarts itself at an
-# idle moment via `maestro session restart` (the supervisor relaunches on the
-# new code). A rollback leaves no notice. Every attempt also writes
+# state/session/upgrade-notice.json {from,to,at}; the session's feed
+# (lib/session/feed-core.mjs) surfaces it as an `upgrade-available` directive
+# and the session restarts itself at an idle moment via `maestro session
+# restart` (the supervisor relaunches on the new code). The same file with
+# {healthy:false, reason} is a REPORT — the feed emits `daemon-unhealthy`
+# for it, a distinct action, so no session restarts on a health report. A
+# rollback leaves no notice. Every attempt also writes
 # state/autoupdate/last.json {from,to,at,ok,healthy,reason}, which the beat
 # reports as machine.upgrade. See
 # docs/superpowers/specs/2026-09-08-front-door-session-design.md §3.10 + WP-M6.
@@ -53,14 +135,18 @@
 #
 # Test knobs (scripts/local-triggers/autoupdate.test.mjs): MAESTRO_AUTOUPDATE_JITTER_MAX
 # (default 600) and MAESTRO_AUTOUPDATE_HEALTH_WAIT (default 30), both seconds;
-# MAESTRO_AUTOUPDATE_RETRY_SLEEP (default 20 s, ×attempt); MAESTRO_AUTOUPDATE_LOCK_STALE_S
-# (default 7200); MAESTRO_AUTOUPDATE_FAILED_HOLD_S (default 86400);
-# MAESTRO_AUTOUPDATE_PATH_PREFIX (a stub-binary dir that wins over PATH);
+# MAESTRO_AUTOUPDATE_STABLE_S (default 90; MAESTRO_AUTOUPDATE_STABLE_GAP_S is the
+# sleep between the two pid samples, default = STABLE_S, zeroed by the test) and
+# MAESTRO_AUTOUPDATE_BEAT_FRESH_S (default 300); MAESTRO_AUTOUPDATE_RETRY_SLEEP
+# (default 20 s, ×attempt);
+# MAESTRO_AUTOUPDATE_LOCK_STALE_S (default 7200); MAESTRO_AUTOUPDATE_FAILED_HOLD_S
+# (default 86400); MAESTRO_AUTOUPDATE_PATH_PREFIX (a stub-binary dir that wins
+# over PATH — the test stubs npm, launchctl, pgrep AND ps there);
 # MAESTRO_SKIP_GLOBAL_INSTALL=1 (never touch the global install).
 # =============================================================================
 set -uo pipefail
 # MAESTRO_AUTOUPDATE_PATH_PREFIX: a dir that wins over the standard prepend
-# (the test puts its npm/launchctl/pgrep stubs there so a run never reaches
+# (the test puts its npm/launchctl/pgrep/ps stubs there so a run never reaches
 # the real registry or launchd).
 export PATH="${MAESTRO_AUTOUPDATE_PATH_PREFIX:+$MAESTRO_AUTOUPDATE_PATH_PREFIX:}/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:${PATH:-}"
 
@@ -68,6 +154,11 @@ export PATH="${MAESTRO_AUTOUPDATE_PATH_PREFIX:+$MAESTRO_AUTOUPDATE_PATH_PREFIX:}
 # The copy removes itself on exit (REEXEC_COPY is only ever a mktemp'd
 # maestro-autoupdate.* file — never the agent dir's script).
 if [ "${MAESTRO_AUTOUPDATE_REEXEC:-}" != "1" ]; then
+  # Pin the agent dir BEFORE re-exec'ing: the copy lives in $TMPDIR, so its own
+  # location says nothing about the agent. launchd's plist sets AGENT_ROOT; a
+  # hand-run `bash scripts/local-triggers/autoupdate.sh` did not, resolved
+  # AGENT_DIR to $TMPDIR/../.., and exited 0 having done nothing.
+  export AGENT_ROOT="${AGENT_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}"
   _tmp="$(mktemp "${TMPDIR:-/tmp}/maestro-autoupdate.XXXXXX")" || exit 0
   cp "${BASH_SOURCE[0]}" "$_tmp" 2>/dev/null || exit 0
   MAESTRO_AUTOUPDATE_REEXEC=1 exec /bin/bash "$_tmp" "$@"
@@ -144,46 +235,150 @@ npm_view_latest(){ # prints the latest version, or nothing after RETRIES attempt
   done
 }
 
-# ── version check ────────────────────────────────────────────────────────────
-CUR="$(node -p "require('$AGENT_DIR/node_modules/$PKG/package.json').version" 2>/dev/null || echo 0.0.0)"
-LATEST="$(npm_view_latest)" || LATEST=""
-[ -z "$LATEST" ] && { log "npm view failed after $RETRIES attempts; skip (still $CUR)"; exit 0; }
-if [ "$CUR" = "$LATEST" ] || [ "$(printf '%s\n%s\n' "$CUR" "$LATEST" | sort -V | tail -1)" != "$LATEST" ]; then
-  log "up to date ($CUR)"; exit 0
-fi
-
-# ── failed-target hold ───────────────────────────────────────────────────────
-# last.json is the memory of the health gate: a LATEST that was installed here
-# and rolled back (or could not even be rolled back) is not retried every hour
-# — that would be two installs, two global refreshes and two daemon restarts an
-# hour until someone touched the kill-switch. An install-failed (network) entry
-# does not hold. Fail-open: an unreadable last.json holds nothing.
-FAILED_HOLD_S="${MAESTRO_AUTOUPDATE_FAILED_HOLD_S:-86400}"
-failed_hold_reason(){ # prints "<reason> at <at>" when LATEST failed health within FAILED_HOLD_S
-  node -e '
-    const [file, latest, holdS] = process.argv.slice(1);
-    try {
-      const j = JSON.parse(require("fs").readFileSync(file, "utf8"));
-      const at = Date.parse(j.at);
-      if (j.to === latest && j.ok === false && /unhealthy/.test(String(j.reason)) && Number.isFinite(at) && Date.now() - at < Number(holdS) * 1000) process.stdout.write(`${j.reason} at ${j.at}`);
-    } catch {}
-  ' "$AGENT_DIR/state/autoupdate/last.json" "$LATEST" "$FAILED_HOLD_S" 2>/dev/null
-}
-HOLD="$(failed_hold_reason)"
-if [ -n "$HOLD" ]; then
-  log "SKIP: $LATEST already failed the health gate here ($HOLD); holding ${FAILED_HOLD_S}s (still $CUR) — rm state/autoupdate/last.json to retry now"
-  exit 0
-fi
-log "UPDATE $CUR -> $LATEST"
-
+# ── this agent, its daemon, and the health gate ──────────────────────────────
+# Defined BEFORE the version check: the up-to-date path runs the gate too (d).
 UID_="$(id -u)"
 AGENT_FIRST="$(node -p "String(require('$AGENT_DIR/config/agent.json').firstName || '').toLowerCase()" 2>/dev/null)"
 [ -n "$AGENT_FIRST" ] && log "agent: $AGENT_FIRST (launchd/pgrep questions are scoped to ai.maestro.$AGENT_FIRST-*)"
-DLOG="$AGENT_DIR/logs/daemon/daemon-$(date +%Y-%m-%d).log"
+DAEMON_PATTERN="$AGENT_DIR/scripts/daemon/maestro-daemon.mjs"
+# The daemon's argv is exactly `<node> <AGENT_DIR>/scripts/daemon/maestro-daemon.mjs`
+# (launchd-wrapper.sh execs it that way), so the pgrep pattern is anchored to
+# that whole argument list. `pgrep -f <path>` alone matches ANY process whose
+# argv merely contains the path — an editor, a grep, a `node --inspect` by an
+# operator — and one of those appearing between the two pid samples reads as
+# a pid-set change, i.e. "crash loop".
+DAEMON_ARGV_RE="^[^ ]*/node $(printf '%s' "$DAEMON_PATTERN" | sed 's/[][\.*^$]/\\&/g')\$"
+DLOG_DIR="$AGENT_DIR/logs/daemon"
 HEALTH_WAIT="${MAESTRO_AUTOUPDATE_HEALTH_WAIT:-30}"
+STABLE_S="${MAESTRO_AUTOUPDATE_STABLE_S:-90}"
+STABLE_GAP_S="${MAESTRO_AUTOUPDATE_STABLE_GAP_S:-$STABLE_S}"   # the gap between the two pid samples; the test zeroes it (its stub changes per CALL)
+BEAT_FRESH_S="${MAESTRO_AUTOUPDATE_BEAT_FRESH_S:-300}"
+LAST_BEAT="$AGENT_DIR/state/org/last-beat.json"
+MESH_FILE="$AGENT_DIR/lib/org/mesh.mjs"
+mesh_records_beats(){ grep -q "LAST_BEAT_FILE" "$MESH_FILE" 2>/dev/null; } # the installed mesh writes state/org/last-beat.json (the release that added this gate)
+# The lines the daemon writes on its way down. Tight on purpose: the daemon log
+# is free text (LLM output included), so every pattern here is a prefix the
+# code emits — lib/util/unhandled.mjs via maestro-daemon.mjs's onEvent, the
+# main() catch in agent-daemon.mjs, launchd-wrapper.sh, and Node's own code
+# for an import that cannot resolve (how a bad release most often dies).
+FATAL_RE='\[DAEMON\] uncaughtException:|\[daemon\] Fatal:|\[wrapper\] FATAL:|ERR_MODULE_NOT_FOUND'
+HEALTH_REASON=""   # set by is_healthy on failure; a phrase, no quotes
 DAEMON_LABEL=""
 SESSION_LABEL=""
 
+daemon_pids(){ # every pid whose argv IS this agent's daemon entry (see DAEMON_ARGV_RE), sorted, space-joined ("" when none)
+  pgrep -f "$DAEMON_ARGV_RE" 2>/dev/null | sort -n | tr '\n' ' ' | sed 's/ *$//'
+}
+etime_seconds(){ # $1 = ps etime ([[dd-]hh:]mm:ss) → seconds; "" when unparseable
+  local s="$1" d=0 h=0 m=0 sec=0 a b c
+  [ -n "$s" ] || return 0
+  case "$s" in *-*) d="${s%%-*}"; s="${s#*-}" ;; esac
+  IFS=: read -r a b c <<< "$s"
+  if [ -n "${c:-}" ]; then h="$a"; m="$b"; sec="$c"; else m="$a"; sec="${b:-0}"; fi
+  case "$d$h$m$sec" in *[!0-9]*) return 0 ;; esac
+  echo $(( 10#$d * 86400 + 10#$h * 3600 + 10#$m * 60 + 10#$sec ))
+}
+pid_uptime_s(){ # $1 = pid → seconds since it started, "" when ps cannot say
+  etime_seconds "$(ps -o etime= -p "$1" 2>/dev/null | tr -d ' ')"
+}
+day_of_epoch(){ # $1 = epoch seconds → YYYY-MM-DD in local time (the wrapper's `date +%Y-%m-%d` basis)
+  date -r "$1" +%Y-%m-%d 2>/dev/null || date -d "@$1" +%Y-%m-%d 2>/dev/null
+}
+daemon_log_for(){ # $1 = uptime seconds → the log file the wrapper opened for THIS incarnation (named for its start day)
+  echo "$DLOG_DIR/daemon-$(day_of_epoch $(( $(date +%s) - $1 ))).log"
+}
+first_fatal_after(){ # $1 = log file, $2 = line offset (lines up to it predate this daemon) → the first fatal line, or nothing
+  [ -f "$1" ] || return 0
+  tail -n +"$(( $2 + 1 ))" "$1" 2>/dev/null | grep -m1 -E "$FATAL_RE"
+  return 0
+}
+# The daemon log is APPENDED to (launchd-wrapper.sh `>>`) and shared by every
+# incarnation that starts on the same day, so "since this daemon" needs a
+# marker. maestro-daemon.mjs prints `[DAEMON] boot pid=<pid> at=<iso>` as its
+# first line (since the release that added this gate); for a daemon older than that the LAST generic boot line
+# — "[daemon] Running" / "[DAEMON] org-mesh connected", which only a process
+# that got through boot prints — is the best available split. Without either,
+# a one-off crash at 09:00 and the healthy respawn at 09:01 share a file and
+# the crash read as "fatal since restart" every hour until midnight.
+BOOT_RE='\[daemon\] Running|\[DAEMON\] org-mesh connected'
+boot_offset(){ # $1 = log file, $2 = pid → the line count that precedes THIS incarnation's boot marker (0 when there is none)
+  local n
+  [ -f "$1" ] || { echo 0; return 0; }
+  n="$(grep -n -F "[DAEMON] boot pid=$2 " "$1" 2>/dev/null | tail -1 | cut -d: -f1)"
+  [ -n "$n" ] || n="$(grep -n -E "$BOOT_RE" "$1" 2>/dev/null | tail -1 | cut -d: -f1)"
+  echo $(( ${n:-1} - 1 ))
+}
+org_enrolled(){ # the SAME answer the daemon gives: lib/org/client.mjs loadOrgConfig + isEnabled — org.cohort.enabled AND a base (or apiUrl+orgId). A seat that fails that test never beats, so the gate does not ask it to.
+  node --input-type=module -e '
+    const [root] = process.argv.slice(1);
+    const { loadOrgConfig, isEnabled } = await import(root + "/lib/org/client.mjs");
+    process.exit(isEnabled(loadOrgConfig(root)) ? 0 : 3);
+  ' "$AGENT_DIR" 2>/dev/null
+  case $? in 0) return 0 ;; 3) return 1 ;; esac
+  log "WARN: lib/org/client.mjs could not answer whether this seat is enrolled — treating it as not enrolled (the daemon would too)"
+  return 1
+}
+beat_not_acknowledged(){ # $1 = epoch seconds the daemon started, $2 = installed version → prints WHY the org has not acknowledged this daemon ON THIS VERSION, or nothing when it has
+  node -e '
+    const [file, sinceS, freshS, meshRecords, installed] = process.argv.slice(1);
+    const out = (s) => process.stdout.write(s);
+    let j;
+    try { j = JSON.parse(require("fs").readFileSync(file, "utf8")); }
+    catch (e) { out(`no beat recorded (${e && e.code === "ENOENT" ? (meshRecords === "1" ? "state/org/last-beat.json absent — this daemon process has never completed a beat on this code" : "state/org/last-beat.json absent, and the installed lib/org/mesh.mjs does not record beats at all — a preserved local copy older than this gate? maestro upgrade --verify") : "state/org/last-beat.json unreadable"})`); process.exit(0); }
+    const at = Date.parse(j && j.at);
+    if (!Number.isFinite(at)) { out("last-beat.json carries no parseable at"); process.exit(0); }
+    const ageS = Math.round((Date.now() - at) / 1000);
+    if (j.ok !== true) out(`last beat NOT accepted by the org (${j.code || "?"}${j.message ? ": " + String(j.message).replace(/["\\\\]/g, "") : ""}) at ${j.at}`);
+    else if (at < Number(sinceS) * 1000) out(`last accepted beat (${j.at}) predates this daemon process`);
+    else if (ageS > Number(freshS)) out(`last accepted beat is ${ageS}s old (> ${freshS}s; the org already calls this seat offline)`);
+    else if (installed && typeof j.sdkVersion === "string" && j.sdkVersion && j.sdkVersion !== installed) out(`last accepted beat (${j.at}) told the org sdkVersion ${j.sdkVersion}, but ${installed} is installed — the org sees the old version`);
+  ' "$LAST_BEAT" "$1" "$BEAT_FRESH_S" "$(mesh_records_beats && echo 1 || echo 0)" "${2:-}" 2>/dev/null
+}
+# The daemon log offset "since the restart" — captured right before a kickstart
+# (see restart_daemon). is_healthy takes the LATER of this and the boot marker
+# (boot_offset), so with no restart the scan still starts at the running
+# daemon's own boot line, never at the top of a shared start-day file.
+PRE_LOG=""; PRE_LOG_LINES=0
+installed_version(){ # the version in node_modules RIGHT NOW — re-read per call, because is_healthy runs after an install and again after a rollback
+  node -p "require('$AGENT_DIR/node_modules/$PKG/package.json').version" 2>/dev/null || true
+}
+sibling_job_audit(){ # log-only: every OTHER ai.maestro.<first>-* job whose LAST exit was non-zero. Never a gate reason — a broken relay poller is not a reason to roll the SDK back — but the daemon log is not where those jobs fail, and nothing else reads `launchctl list` for them.
+  [ -n "${AGENT_FIRST:-}" ] || return 0
+  local bad
+  # The -session job's 75 is EX_TEMPFAIL: the supervisor's DESIGNED "the session ended, relaunch me" exit (scripts/session/supervisor.mjs), the signature of every clean relaunch — not a failure.
+  bad="$(launchctl list 2>/dev/null | awk -v p="ai.maestro.${AGENT_FIRST}-" -v sess="ai.maestro.${AGENT_FIRST}-session" 'NR>1 && index($3, p)==1 && $2 != "0" && $2 != "-" && !($3 == sess && $2 == "75") { printf "%s%s(%s)", (n++ ? ", " : ""), $3, $2 }')"
+  [ -n "$bad" ] && log "sibling jobs with a non-zero last exit (not a gate reason; see logs/launchd and logs/polling): $bad"
+  return 0
+}
+is_healthy(){ # (a)+(b)+(c) above. Sets HEALTH_REASON on failure.
+  local p1 p2 pid up oldest oldest_pid dlog fatal why offset boot
+  HEALTH_REASON=""
+  p1="$(daemon_pids)"
+  [ -n "$p1" ] || { HEALTH_REASON="no daemon process for $DAEMON_PATTERN"; return 1; }
+  [ "$STABLE_GAP_S" -gt 0 ] && sleep "$STABLE_GAP_S"
+  p2="$(daemon_pids)"
+  [ "$p1" = "$p2" ] || { HEALTH_REASON="daemon pid changed within ${STABLE_GAP_S}s (${p1} -> ${p2:-none}) — crash loop"; return 1; }
+  oldest=""
+  for pid in $p1; do
+    up="$(pid_uptime_s "$pid")"
+    [ -n "$up" ] && [ "$up" -ge "$STABLE_S" ] || { HEALTH_REASON="daemon pid $pid uptime ${up:-?}s < ${STABLE_S}s — crash loop"; return 1; }
+    [ -n "$oldest" ] && [ "$oldest" -ge "$up" ] || { oldest="$up"; oldest_pid="$pid"; }
+  done
+  dlog="$(daemon_log_for "$oldest")"
+  offset=0; [ "$dlog" = "$PRE_LOG" ] && offset="$PRE_LOG_LINES"
+  boot="$(boot_offset "$dlog" "$oldest_pid")"
+  [ "$boot" -gt "$offset" ] && offset="$boot"
+  fatal="$(first_fatal_after "$dlog" "$offset")"
+  [ -z "$fatal" ] || { HEALTH_REASON="fatal in $(basename "$dlog") since this daemon booted (scanned from line $(( offset + 1 ))): $(printf '%s' "$fatal" | tr -d '"\\' | cut -c1-160)"; return 1; }
+  if org_enrolled; then
+    why="$(beat_not_acknowledged $(( $(date +%s) - oldest - 2 )) "$(installed_version)")"
+    [ -z "$why" ] || { HEALTH_REASON="$why"; return 1; }
+  else
+    log "org not enrolled in config/org.yaml — beat acknowledgement not required"
+  fi
+  return 0
+}
+
 first_plist(){ # $1 = suffix, $2 = dir — this agent's plist for that suffix, or nothing (unquoted so `*` globs when no name is known)
   ls "$2"/ai.maestro.${AGENT_FIRST:-*}-"$1".plist 2>/dev/null | head -1
 }
@@ -208,9 +403,212 @@ label_loaded(){ # $1 = label — measured in the gui domain (right from ssh too)
 }
 restart_daemon(){
   resolve_labels
+  # Mark "since the restart" for the fatal scan: the new process appends to
+  # today's file (the wrapper names it for its start day), after these lines.
+  PRE_LOG="$DLOG_DIR/daemon-$(date +%Y-%m-%d).log"
+  PRE_LOG_LINES=0
+  [ -f "$PRE_LOG" ] && PRE_LOG_LINES="$(wc -l < "$PRE_LOG" 2>/dev/null | tr -d ' ')"
+  PRE_LOG_LINES="${PRE_LOG_LINES:-0}"
   if [ -z "$DAEMON_LABEL" ]; then log "no -daemon plist under ~/Library/LaunchAgents — nothing to kickstart (maestro upgrade installs it; doctor will say)"; return 0; fi
   launchctl kickstart -k "gui/$UID_/$DAEMON_LABEL" >> "$LOG" 2>&1 || log "WARN: kickstart $DAEMON_LABEL nonzero"
 }
+session_reconciled(){ # only asked when a -session plist was generated; never a rollback reason
+  label_loaded "$SESSION_LABEL" || { log "reconcile-failed: session job $SESSION_LABEL is not loaded (maestro session start)"; return 1; }
+  pgrep -f "$AGENT_DIR/scripts/session/supervisor" >/dev/null 2>&1 || { log "reconcile-failed: $SESSION_LABEL is loaded but no supervisor process is alive for $AGENT_DIR (maestro session status)"; return 1; }
+  return 0
+}
+json_str(){ printf '%s' "$1" | tr -d '"\\' | tr '\n' ' '; } # a value safe inside "…" in the JSON files below
+write_notice(){ # $1 = from, $2 = to, [$3 = reason → healthy:false] — tell the front-door session; never restart it here
+  local dir="$AGENT_DIR/state/session" tmp
+  mkdir -p "$dir" 2>/dev/null || return 0
+  tmp="$(mktemp "$dir/.upgrade-notice.XXXXXX" 2>/dev/null)" || return 0
+  if [ -n "${3:-}" ]; then
+    printf '{\n  "from": "%s",\n  "to": "%s",\n  "at": "%s",\n  "healthy": false,\n  "reason": "%s"\n}\n' "$1" "$2" "$(date -u +%FT%TZ)" "$(json_str "$3")" > "$tmp" \
+      && mv -f "$tmp" "$dir/upgrade-notice.json" \
+      && log "session notified: upgrade-notice.json healthy:false on $2 (no restart — the session surfaces it)"
+  else
+    printf '{\n  "from": "%s",\n  "to": "%s",\n  "at": "%s"\n}\n' "$1" "$2" "$(date -u +%FT%TZ)" > "$tmp" \
+      && mv -f "$tmp" "$dir/upgrade-notice.json" \
+      && log "session notified: upgrade-notice.json $1 -> $2 (session restarts itself when idle)"
+  fi
+  rm -f "$tmp" 2>/dev/null
+  return 0
+}
+write_upgrade_notice(){ write_notice "$1" "$2"; }
+LAST_JSON="$AGENT_DIR/state/autoupdate/last.json"
+write_last(){ # $1 from, $2 to, $3 ok, $4 healthy, $5 reason — state/autoupdate/last.json (→ beat machine.upgrade)
+  local dir="$AGENT_DIR/state/autoupdate" tmp
+  mkdir -p "$dir" 2>/dev/null || return 0
+  tmp="$(mktemp "$dir/.last.XXXXXX" 2>/dev/null)" || return 0
+  printf '{\n  "from": "%s",\n  "to": "%s",\n  "at": "%s",\n  "ok": %s,\n  "healthy": %s,\n  "reason": "%s"\n}\n' \
+    "$1" "$2" "$(date -u +%FT%TZ)" "$3" "$4" "$(json_str "$5")" > "$tmp" && mv -f "$tmp" "$LAST_JSON"
+  rm -f "$tmp" 2>/dev/null
+  return 0
+}
+last_reason(){ # the reason field of last.json, or nothing
+  node -e 'try { process.stdout.write(String(JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")).reason || "")); } catch {}' "$LAST_JSON" 2>/dev/null
+}
+# The version the RUNNING daemon started on. scripts/daemon/health.mjs resolves
+# sdk_version once at load and writes it to state/dashboards/daemon-health.yaml
+# with its pid, precisely so a reader can tell the installed package from the
+# code a live process is executing. Printed only when the dashboard's pid is
+# one of the sampled daemon pids — a dashboard left by a dead process says
+# nothing about this one.
+DAEMON_HEALTH="$AGENT_DIR/state/dashboards/daemon-health.yaml"
+running_daemon_version(){ # → the sdk_version of the LIVE daemon (dashboard pid ∈ daemon_pids), or nothing
+  local v pid
+  [ -f "$DAEMON_HEALTH" ] || return 0
+  v="$(sed -nE 's/^sdk_version:[[:space:]]*([^[:space:]]+).*$/\1/p' "$DAEMON_HEALTH" 2>/dev/null | head -1)"
+  pid="$(sed -nE 's/^pid:[[:space:]]*([0-9]+).*$/\1/p' "$DAEMON_HEALTH" 2>/dev/null | head -1)"
+  [ -n "$v" ] && [ -n "$pid" ] || return 0
+  case " $(daemon_pids) " in *" $pid "*) printf '%s' "$v" ;; esac
+}
+daemon_start_epoch(){ # of the longest-running daemon pid, "" when none
+  local p up best=""
+  for p in $(daemon_pids); do up="$(pid_uptime_s "$p")"; [ -n "$up" ] || continue; [ -n "$best" ] && [ "$best" -ge "$up" ] || best="$up"; done
+  [ -n "$best" ] && echo $(( $(date +%s) - best ))
+  return 0
+}
+daemon_predates_mesh(){ # no beat record, and the live daemon started BEFORE the installed lib/org/mesh.mjs was written → it cannot be running that code
+  local start mt
+  [ -f "$LAST_BEAT" ] && return 1
+  mesh_records_beats || return 1
+  start="$(daemon_start_epoch)"; [ -n "$start" ] || return 1
+  mt="$(stat -f %m "$MESH_FILE" 2>/dev/null || stat -c %Y "$MESH_FILE" 2>/dev/null)"; [ -n "$mt" ] || return 1
+  [ "$start" -lt "$mt" ]
+}
+
+# ── version check ────────────────────────────────────────────────────────────
+CUR="$(node -p "require('$AGENT_DIR/node_modules/$PKG/package.json').version" 2>/dev/null || echo 0.0.0)"
+LATEST="$(npm_view_latest)" || LATEST=""
+[ -z "$LATEST" ] && { log "npm view failed after $RETRIES attempts; skip (still $CUR)"; exit 0; }
+# (f) AHEAD OF THE REGISTRY is not "up to date", and the difference is the whole
+# fleet. The branch below treats `CUR >= LATEST` as one outcome, so a seat
+# carrying a build that was never published logs the same reassuring
+# "up to date (<CUR>)" as a seat that is genuinely current — which is how the
+# health gate written for the six-day crash loop came to be running on ZERO
+# machines: the authoring seat installed 2.18.0 from its own tree, npm's latest
+# stopped at 2.17.0, and every hourly run on every box said "up to date". A
+# gate that lives only on the machine that wrote it is live nowhere, and
+# nothing said so. Say it, in the log AND in last.json — which the beat carries
+# to the org as machine.upgrade, so the fleet holds the fact rather than the
+# one box that already has the code.
+AHEAD=""
+if [ "$CUR" != "$LATEST" ] && [ "$(printf '%s\n%s\n' "$CUR" "$LATEST" | sort -V | tail -1)" = "$CUR" ]; then
+  AHEAD=1
+  log "AHEAD OF REGISTRY: installed $CUR, npm latest is $LATEST — this build was never published. Every OTHER seat follows @latest and is pinned at $LATEST, so nothing in $CUR has reached the fleet. Publish it (npm publish) or roll this seat back; do not read the line below as the fleet being current."
+fi
+if [ "$CUR" = "$LATEST" ] || [ "$(printf '%s\n%s\n' "$CUR" "$LATEST" | sort -V | tail -1)" != "$LATEST" ]; then
+  # (d) up to date is not the same as well. Run the gate on the CURRENT
+  # version; there is nothing to roll back, so the outcome is evidence —
+  # loud in this log, in last.json (→ the beat's machine.upgrade), and in the
+  # session's feed on the transition into an unhealthy episode.
+  log "up to date ($CUR) — checking daemon health: pid stability ${STABLE_S}s, fatal scan, org-acknowledged beat (< ${BEAT_FRESH_S}s)"
+  PREV_REASON="$(last_reason)"
+  # (e) A daemon still running an OLDER version than the one installed. That
+  # is what a hand-run `maestro upgrade` leaves behind: its post-steps never
+  # restart the daemon (lib/upgrade/post-steps.mjs — "the daemon is restarted
+  # by autoupdate, never here") and its verify row says the new version "takes
+  # effect when autoupdate kickstarts it". Until this branch existed that was
+  # a promise nobody kept: the installed version was already @latest, so this
+  # path never kickstarted anything, the daemon ran the old code until the
+  # NEXT release, and — because an earlier mesh.mjs never wrote
+  # state/org/last-beat.json — the gate below read the missing record as
+  # "no beat recorded" and declared a stable, beating daemon unhealthy. The
+  # kickstart is the upgrade's missing last step; the gate then runs as after
+  # any restart, and a healthy outcome completes the hop the same way an
+  # automatic one does (notice → the session restarts itself when idle).
+  # Two signals, either is enough: the dashboard's sdk_version differs from
+  # the installed one (a version hop), or — same version — there is no beat
+  # record and the daemon started before the installed lib/org/mesh.mjs was
+  # written (a file change without a version bump: it cannot be running that
+  # code, and that code is the only thing that writes the record).
+  RUNNING_V="$(running_daemon_version)"
+  STALE_FROM=""; STALE_HOP=""
+  if [ -n "$RUNNING_V" ] && [ "$RUNNING_V" != "$CUR" ]; then
+    STALE_FROM="$RUNNING_V"; STALE_HOP=1
+    log "stale daemon: running $RUNNING_V, installed $CUR — a hand-run maestro upgrade left the daemon restart to this job; kickstarting"
+  elif daemon_predates_mesh; then
+    STALE_FROM="${RUNNING_V:-$CUR}"
+    log "stale daemon: started $(daemon_start_epoch | xargs -I{} date -u -r {} +%FT%TZ 2>/dev/null) — before the installed lib/org/mesh.mjs was written — so it runs code that never records a beat; kickstarting onto the installed code"
+  fi
+  if [ -n "$STALE_FROM" ]; then
+    restart_daemon
+    [ "$HEALTH_WAIT" -gt 0 ] && sleep "$HEALTH_WAIT"
+  fi
+  if is_healthy; then
+    log "healthy on $CUR"
+    sibling_job_audit
+    if [ -n "$STALE_HOP" ]; then
+      # A real version hop: complete it the way an automatic one completes —
+      # the notice tells the session to restart itself onto the new code.
+      log "OK: daemon now on $CUR (was $STALE_FROM) — the manual upgrade is complete"
+      write_upgrade_notice "$STALE_FROM" "$CUR"
+      write_last "$STALE_FROM" "$CUR" true true ""
+    elif [ -n "$STALE_FROM" ]; then
+      # Same version, newer files: the daemon is reconciled; the session is
+      # not told to restart (a touched or restored file must not cost it its
+      # context — the version did not change).
+      log "OK: daemon restarted onto the installed $CUR code; no session notice (same version)"
+      write_last "$CUR" "$CUR" true true ""
+    elif [ -n "$AHEAD" ]; then
+      # The DAEMON is well; the FLEET is not. Recorded as ok/healthy — because
+      # it is — with the finding in `reason`, so the org learns that this seat
+      # executes code no other seat can install. Written every run, not once
+      # per episode: unlike an unhealthy daemon this does not clear itself, and
+      # a fact that ages out of one log file is how six days went by.
+      log "OK: healthy on $CUR, but AHEAD of the registry ($LATEST) — recorded for the org as ahead-of-registry"
+      write_last "$CUR" "$CUR" true true "ahead-of-registry: npm latest is $LATEST"
+    else
+      case "$PREV_REASON" in unhealthy-current*)
+        log "recovered: last.json no longer reports unhealthy-current"
+        write_last "$CUR" "$CUR" true true "" ;;
+      ahead-of-registry*)
+        log "recovered: $CUR is on the registry now (latest $LATEST) — last.json no longer reports ahead-of-registry"
+        write_last "$CUR" "$CUR" true true "" ;;
+      esac
+    fi
+  else
+    log "UNHEALTHY on $CUR${STALE_FROM:+ (kickstarted from $STALE_FROM)}: $HEALTH_REASON — no upgrade to roll back; the org cannot see this seat. maestro doctor"
+    # The `unhealthy-current:` PREFIX is load-bearing (the episode check below
+    # and failed_hold_reason both match on it), so the ahead-of-registry fact
+    # is appended, never prepended — an unpublished build that is also sick is
+    # the worst case and must say both things.
+    write_last "$CUR" "$CUR" false false "unhealthy-current: $HEALTH_REASON${AHEAD:+ (and ahead-of-registry: npm latest is $LATEST)}"
+    case "$PREV_REASON" in
+      unhealthy-current*) log "session already notified this episode (last.json was unhealthy-current); notice not rewritten" ;;
+      *) write_notice "$CUR" "$CUR" "unhealthy-current: $HEALTH_REASON" ;;
+    esac
+  fi
+  exit 0
+fi
+
+# ── failed-target hold ───────────────────────────────────────────────────────
+# last.json is the memory of the health gate: a LATEST that was installed here
+# and rolled back (or could not even be rolled back) is not retried every hour
+# — that would be two installs, two global refreshes and two daemon restarts an
+# hour until someone touched the kill-switch. An install-failed (network) entry
+# does not hold, and neither does an unhealthy-current entry (its `to` is the
+# installed version, never a LATEST). Fail-open: an unreadable last.json holds
+# nothing.
+FAILED_HOLD_S="${MAESTRO_AUTOUPDATE_FAILED_HOLD_S:-86400}"
+failed_hold_reason(){ # prints "<reason> at <at>" when LATEST failed health within FAILED_HOLD_S
+  node -e '
+    const [file, latest, holdS] = process.argv.slice(1);
+    try {
+      const j = JSON.parse(require("fs").readFileSync(file, "utf8"));
+      const at = Date.parse(j.at);
+      if (j.to === latest && j.ok === false && /unhealthy/.test(String(j.reason)) && Number.isFinite(at) && Date.now() - at < Number(holdS) * 1000) process.stdout.write(`${j.reason} at ${j.at}`);
+    } catch {}
+  ' "$LAST_JSON" "$LATEST" "$FAILED_HOLD_S" 2>/dev/null
+}
+HOLD="$(failed_hold_reason)"
+if [ -n "$HOLD" ]; then
+  log "SKIP: $LATEST already failed the health gate here ($HOLD); holding ${FAILED_HOLD_S}s (still $CUR) — rm state/autoupdate/last.json to retry now"
+  exit 0
+fi
+log "UPDATE $CUR -> $LATEST"
+
 # What -session plists THIS agent had installed BEFORE this run. A rollback
 # reverts only a session job this run's upgrade installed — never one that
 # was already there (that one is the operator's, and `maestro session stop`
@@ -240,35 +638,6 @@ refresh_global(){ # $1 = version — the rollback's counterpart of upgrade() ste
   retry "npm install -g $PKG@$1" npm install -g "$PKG@$1" --no-audit --no-fund || log "WARN: global refresh to $1 failed — run: npm install -g $PKG@$1"
   return 0
 }
-is_healthy(){ # THIS agent's daemon process alive AND a fresh boot/connect line
-  pgrep -f "$AGENT_DIR/scripts/daemon/maestro-daemon.mjs" >/dev/null 2>&1 || return 1
-  tail -80 "$DLOG" 2>/dev/null | grep -qE "org-mesh connected|daemon\] Running" || return 1
-  return 0
-}
-session_reconciled(){ # only asked when a -session plist was generated; never a rollback reason
-  label_loaded "$SESSION_LABEL" || { log "reconcile-failed: session job $SESSION_LABEL is not loaded (maestro session start)"; return 1; }
-  pgrep -f "$AGENT_DIR/scripts/session/supervisor" >/dev/null 2>&1 || { log "reconcile-failed: $SESSION_LABEL is loaded but no supervisor process is alive for $AGENT_DIR (maestro session status)"; return 1; }
-  return 0
-}
-write_upgrade_notice(){ # $1 = from, $2 = to — tell the front-door session; never restart it here
-  local dir="$AGENT_DIR/state/session" tmp
-  mkdir -p "$dir" 2>/dev/null || return 0
-  tmp="$(mktemp "$dir/.upgrade-notice.XXXXXX" 2>/dev/null)" || return 0
-  printf '{\n  "from": "%s",\n  "to": "%s",\n  "at": "%s"\n}\n' "$1" "$2" "$(date -u +%FT%TZ)" > "$tmp" \
-    && mv -f "$tmp" "$dir/upgrade-notice.json" \
-    && log "session notified: upgrade-notice.json $1 -> $2 (session restarts itself when idle)"
-  rm -f "$tmp" 2>/dev/null
-  return 0
-}
-write_last(){ # $1 from, $2 to, $3 ok, $4 healthy, $5 reason — state/autoupdate/last.json (→ beat machine.upgrade)
-  local dir="$AGENT_DIR/state/autoupdate" tmp
-  mkdir -p "$dir" 2>/dev/null || return 0
-  tmp="$(mktemp "$dir/.last.XXXXXX" 2>/dev/null)" || return 0
-  printf '{\n  "from": "%s",\n  "to": "%s",\n  "at": "%s",\n  "ok": %s,\n  "healthy": %s,\n  "reason": "%s"\n}\n' \
-    "$1" "$2" "$(date -u +%FT%TZ)" "$3" "$4" "$5" > "$tmp" && mv -f "$tmp" "$dir/last.json"
-  rm -f "$tmp" 2>/dev/null
-  return 0
-}
 migrate_poller_plist(){ # audit F6 follow-through: an older install left SLACK_USER_TOKEN in a 0644 poller plist; nothing else rewrites it
   [ -f "$AGENT_DIR/scripts/poller-launchd/migrate.sh" ] || return 0
   bash "$AGENT_DIR/scripts/poller-launchd/migrate.sh" >> "$LOG" 2>&1 || log "WARN: poller plist migration nonzero (see above)"
@@ -287,11 +656,12 @@ if is_healthy; then
   REASON=""
   if [ -n "$SESSION_LABEL" ] && ! session_reconciled; then REASON="session-reconcile-failed"; fi
   log "OK: healthy on $LATEST${REASON:+ ($REASON)}"
+  sibling_job_audit
   write_upgrade_notice "$CUR" "$LATEST"
   write_last "$CUR" "$LATEST" true true "$REASON"
   migrate_poller_plist
 else
-  log "UNHEALTHY on $LATEST -> ROLLBACK to $CUR"
+  log "UNHEALTHY on $LATEST ($HEALTH_REASON) -> ROLLBACK to $CUR"
   apply_version "$CUR" || log "WARN: rollback install nonzero"
   refresh_global "$CUR"
   revert_session_install
@@ -301,7 +671,7 @@ else
     log "rolled back to $CUR (healthy)"
     write_last "$CUR" "$LATEST" false true "unhealthy-rolled-back"
   else
-    log "ROLLBACK health still failing on $CUR — needs operator"
+    log "ROLLBACK health still failing on $CUR ($HEALTH_REASON) — needs operator"
     write_last "$CUR" "$LATEST" false false "rollback-unhealthy"
   fi
 fi

package/scripts/local-triggers/run-trigger.sh

@@ -35,6 +35,19 @@ TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
 DATE=$(date +%Y-%m-%d)
 LOG_FILE="$LOG_DIR/$DATE-$TRIGGER_NAME.jsonl"
 
+# Retired on engine-cohort seats (W4-E1): this lane runs claude outside the
+# runtime adapter. The guard exits 3 with the reason when runtime.engine is cohort.
+# Fail closed: when the guard cannot run (no node on PATH — launchd's
+# /usr/bin:/bin — or the guard file is missing), config/agent.json is checked
+# here and an engine-cohort seat is refused all the same.
+LEGACY_GUARD="$AGENT_REPO_DIR/lib/runtime/legacy-shell-guard.mjs"
+if [ -f "$LEGACY_GUARD" ] && command -v node >/dev/null 2>&1; then
+    node "$LEGACY_GUARD" run-trigger.sh "$AGENT_REPO_DIR" || exit $?
+elif [ -f "$AGENT_REPO_DIR/config/agent.json" ] && grep -Eq '"engine"[[:space:]]*:[[:space:]]*"cohort"' "$AGENT_REPO_DIR/config/agent.json"; then
+    echo "run-trigger.sh is retired on this seat: runtime.engine is \"cohort\", and this legacy lane runs the claude binary directly, outside the runtime adapter (the engine guard could not run: node or $LEGACY_GUARD is unavailable)." >&2
+    exit 3
+fi
+
 # Check emergency stop
 if [ -f "$AGENT_REPO_DIR/.emergency-stop" ]; then
     echo "{\"timestamp\":\"$TIMESTAMP\",\"trigger\":\"$TRIGGER_NAME\",\"status\":\"blocked\",\"reason\":\"emergency-stop\"}" >> "$LOG_FILE"