omp-conductor 0.15.10 → 0.15.12

package/systemd/omp-conductor-recover.sh

@@ -0,0 +1,433 @@
+#!/usr/bin/env bash
+#
+# omp-conductor fleet recovery — the OnFailure= playbook for the two fleet
+# units (omp-conductor.service, herdr-fleet.service).
+#
+# Why it exists. Both units crash-loop under systemd's default start limit and
+# then sit in `failed` indefinitely, in silence: the daemon cannot report its
+# own crash loop, and the orchestrator cannot report a session host that is not
+# hosting it (#485). Each unit carries OnFailure=omp-conductor-recover.service,
+# so the moment one of them is given up on, systemd starts this unit — outside
+# the failed unit's cgroup and process tree, so the restarts below can never
+# take the recovery down with them.
+#
+# The playbook is fixed and bounded, because it runs at the moment the system
+# is least healthy with no orchestrator alive to review it:
+#
+#   1. Collect evidence durably, before anything else. journal tail, unit
+#      state, `omp-conductor doctor --json`, installed versions, and the
+#      pre-upgrade snapshot list land in $RECOVER_DIR so the record survives
+#      no matter what happens next.
+#   2. Attempt at most ONE known-safe restorative action, chosen mechanically
+#      from that evidence:
+#        ROLLBACK     — a fresh pre-upgrade config snapshot exists (an upgrade
+#                       just landed and the daemon died after it): restore the
+#                       durable bytes `upgrade` keeps, then start the daemon
+#                       unit once.
+#        RECOVER_PANE — herdr-fleet.service failed: start it (reset-failed +
+#                       start); its startup hook resumes the fleet pane by
+#                       exact identity.
+#        RESTART_PEER — the daemon failed with no upgrade evidence: restart
+#                       the peer unit (herdr-fleet.service) once. The fleet
+#                       units share state; the peer is the only unit the
+#                       recovery may touch that cannot re-loop the failure
+#                       that already exhausted the failed unit's start limit.
+#      Nothing improvised: no diagnosis, no repo edits, no installed-tree
+#      writes, no blind restarts of the failed unit. If the evidence points at
+#      a code defect, the right output is the escalation — the normal fleet
+#      loop repairs it once the fleet is back.
+#   3. Escalate tier-2 through the durable outbox (`omp-conductor report
+#      --kind tier2`), whether or not the action worked — a fleet that fell
+#      over and got itself up is still news.
+#   4. Never loop. Consecutive failed attempts are capped at
+#      RECOVER_MAX_FAILED_ATTEMPTS via a durable counter (expired after
+#      RECOVER_FAIL_WINDOW_S so a genuinely new episode gets fresh attempts),
+#      the recovery unit itself carries StartLimitBurst=2 (rendered by
+#      setup-host.ts), and this unit never carries OnFailure= — there is no
+#      path where recovery re-triggers recovery.
+#
+# Runs as root (journalctl and systemctl need it). OMP_CONDUCTOR_HOME is set
+# by the unit to the fleet's conductor home, so every omp-conductor call
+# touches the fleet's own config and store. RECOVER_DRY_RUN=1 prints every
+# decision and mutates nothing.
+
+set -uo pipefail
+
+PROG=omp-conductor-recover
+
+# Every external command is overridable so tests can inject stubs; the unit
+# renders an explicit PATH (this host's non-interactive PATH lacks `omp`,
+# `herdr` and `omp-conductor`).
+SYSTEMCTL=${SYSTEMCTL_BIN:-systemctl}
+JOURNALCTL=${JOURNALCTL_BIN:-journalctl}
+CONDUCTOR=${OMP_CONDUCTOR_BIN:-omp-conductor}
+
+# The fleet's conductor home: where config.json, the sqlite store and the
+# upgrade's durable snapshots live. The unit renders OMP_CONDUCTOR_HOME and
+# RECOVER_STATE_DIR; a standalone run defaults to ~/.omp/conductor.
+STATE_DIR=${RECOVER_STATE_DIR:-${OMP_CONDUCTOR_HOME:-$HOME/.omp/conductor}}
+CONFIG_PATH=$STATE_DIR/config.json
+BACKUP_ROOT=$STATE_DIR/backups/config
+RECOVER_DIR=${RECOVER_DIR:-$STATE_DIR/recovery}
+
+# The two fleet units, daemon first. Tests may narrow this list.
+DAEMON_UNIT=${DAEMON_UNIT:-omp-conductor.service}
+HERDR_UNIT=${HERDR_UNIT:-herdr-fleet.service}
+FLEET_UNITS=${FLEET_UNITS:-$DAEMON_UNIT:$HERDR_UNIT}
+
+# Bounds (see the header). The rendered unit's StartLimitIntervalSec should
+# stay in agreement with RECOVER_FAIL_WINDOW_S.
+MAX_FAILED_ATTEMPTS=${RECOVER_MAX_FAILED_ATTEMPTS:-2}
+FAIL_WINDOW_S=${RECOVER_FAIL_WINDOW_S:-1800}
+ROLLBACK_AGE_S=${RECOVER_ROLLBACK_AGE_S:-3600}
+JOURNAL_LINES=${RECOVER_JOURNAL_LINES:-200}
+VERIFY_POLL_S=${RECOVER_VERIFY_POLL_S:-2}
+VERIFY_TRIES=${RECOVER_VERIFY_TRIES:-10}
+PROJECT=${RECOVER_PROJECT:-}
+
+# --------------------------------------------------------------------------
+# output
+# --------------------------------------------------------------------------
+
+say() { printf '%s: %s\n' "$PROG" "$*"; }
+log() { printf '%s: %s\n' "$PROG" "$*" >&2; }
+
+dry_run() { [[ ${RECOVER_DRY_RUN:-0} == 1 ]]; }
+
+stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; }
+epoch_now() { date +%s; }
+
+# --------------------------------------------------------------------------
+# systemd access (the only live-host surface; tests inject stubs)
+# --------------------------------------------------------------------------
+
+# `is-failed` prints "failed" only for the failed state; an absent unit prints
+# "inactive" and exits 3, which is not a failure.
+unit_is_failed() { # <unit>
+  [[ $("$SYSTEMCTL" is-failed "$1" 2>/dev/null | tr -d '[:space:]') == failed ]]
+}
+
+unit_says_failed() { # <unit> — "is-failed" verdict + exit code, for logs
+  "$SYSTEMCTL" is-failed "$1" 2>/dev/null | tr -d '[:space:]'
+}
+
+# One bounded boot of a unit: clear the start limit, then start. Only ever
+# used on the failed unit with fresh evidence behind it (the rollback leg) or
+# on herdr-fleet (RECOVER_PANE), never as a blind daemon restart.
+start_unit() { # <unit>
+  say "systemd: reset-failed $1"
+  "$SYSTEMCTL" reset-failed "$1" 2>/dev/null || true
+  say "systemd: start $1"
+  "$SYSTEMCTL" start "$1"
+}
+
+# Wait until the unit leaves the failed state, bounded by the poll window.
+# Returns 0 when it did.
+wait_no_longer_failed() { # <unit>
+  local tries=$VERIFY_TRIES
+  while (( tries > 0 )); do
+    if ! unit_is_failed "$1"; then
+      return 0
+    fi
+    sleep "$VERIFY_POLL_S"
+    tries=$((tries - 1))
+  done
+  return 1
+}
+
+# --------------------------------------------------------------------------
+# durable evidence (written before anything else)
+# --------------------------------------------------------------------------
+
+# Write the full evidence bundle for one run and print its path. The file is
+# the record a later human or the ordinary fleet loop reads (#378 wants it
+# provenance-carrying): who, when, what was installed, what the units said,
+# what the doctor said, and the journal tail of the failed unit.
+write_evidence() { # <failed-unit or empty>
+  local unit=${1:-none} states="" snapshots="" doctor="" versions="" journal=""
+  local u
+  if ! mkdir -p "$RECOVER_DIR" 2>/dev/null; then
+    log "cannot create recovery dir $RECOVER_DIR"
+    return 2
+  fi
+  for u in ${FLEET_UNITS//:/ }; do
+    states="$states $u=$(unit_says_failed "$u")"
+  done
+  snapshots=$(ls -t "$BACKUP_ROOT"/config.json.pre-upgrade-* 2>/dev/null | head -n 5 || true)
+  doctor=$("$CONDUCTOR" doctor --json 2>&1 | head -c 4000 || true)
+  versions=$("$CONDUCTOR" --version 2>&1 | head -n 1 || true)
+  if [[ $unit != none ]]; then
+    journal=$("$JOURNALCTL" -u "$unit" -n "$JOURNAL_LINES" --no-pager 2>&1 || true)
+  fi
+  local file="$RECOVER_DIR/evidence-$(stamp)-$unit.txt"
+  {
+    printf '%s\n' "=== omp-conductor fleet recovery evidence (#485) ==="
+    printf 'failed unit:    %s\n' "$unit"
+    printf 'triggered at:   %s (epoch %s)\n' "$(stamp)" "$(epoch_now)"
+    printf 'recover script: %s\n' "$(readlink -f "$0" 2>/dev/null || printf '%s' "$0")"
+    printf 'units:         %s\n' "$states"
+    printf 'installed:      %s\n' "$versions"
+    printf '%s\n' 'pre-upgrade snapshots (newest first):'
+    printf '%s\n' "${snapshots:-  (none)}"
+    printf '%s\n' '--- doctor ---'
+    printf '%s\n' "$doctor"
+    printf '%s\n' '--- journal tail ---'
+    printf '%s\n' "$journal"
+    printf '%s\n' '--- decision ---'
+  } >"$file" 2>/dev/null
+  if [[ ! -s $file ]]; then
+    log "could not write evidence to $file"
+    return 2
+  fi
+  printf '%s\n' "$file"
+}
+
+append_evidence() { # <evidence path> <text>
+  printf '%s\n' "$2" >>"$1" 2>/dev/null || true
+}
+
+# --------------------------------------------------------------------------
+# escalation — always, through the durable outbox (#123, #288)
+# --------------------------------------------------------------------------
+
+# The single escalation path this repo has for "something must reach a human":
+# `omp-conductor report --kind tier2` persists the row in the fleet's store
+# (the durable outbox) before anything is sent, and the daemon owns delivery
+# with bounded retries once it is back. This script only ever enqueues.
+escalation_text() { # <failed-unit or empty> <action> <outcome> <evidence>
+  if [[ ${1:-none} == none ]]; then
+    printf '%s\n' \
+      "fleet recovery (#485): a recovery trigger fired but no fleet unit was failing when it ran — the fleet got itself up or was started externally; evidence=$4"
+    return
+  fi
+  printf '%s\n' \
+    "fleet recovery (#485): $1 entered failed state; recovery=${2:-none}; outcome=${3}; evidence=${4}; next: read the evidence file (and its doctor output) before touching the fleet"
+}
+
+escalate() { # <failed-unit or empty> <action-name> <outcome> <evidence-path>
+  local unit=${1:-none} text args
+  text=$(escalation_text "$unit" "$2" "$3" "$4")
+  if [[ -n $PROJECT ]]; then
+    args=("$CONDUCTOR" report --kind tier2 --project "$PROJECT" --text "$text")
+  else
+    args=("$CONDUCTOR" report --kind tier2 --text "$text")
+  fi
+  say "escalation: ${args[*]}"
+  if dry_run; then
+    return 0
+  fi
+  local out
+  out=$("${args[@]}" 2>&1)
+  local code=$?
+  if (( code != 0 )); then
+    log "escalation failed (exit $code): $out"
+    return 1
+  fi
+  printf '%s: escalation: %s\n' "$PROG" "$out" >&2
+  return 0
+}
+
+# --------------------------------------------------------------------------
+# bounds — the durable counter ("a recovery that fails twice stops trying")
+# --------------------------------------------------------------------------
+
+MARKER="$RECOVER_DIR/attempts"
+
+read_failed_attempts() {
+  local count ts now
+  [[ -f $MARKER ]] || { printf '0\n'; return; }
+  count=$(sed -n '1p' "$MARKER" 2>/dev/null || true)
+  ts=$(sed -n '2p' "$MARKER" 2>/dev/null || true)
+  now=$(epoch_now)
+  if [[ ! $count =~ ^[0-9]+$ || ! $ts =~ ^[0-9]+$ ]] || (( now - ts > FAIL_WINDOW_S )); then
+    printf '0\n'
+    return
+  fi
+  printf '%s\n' "$count"
+}
+
+bump_failed_attempts() {
+  local next
+  next=$(read_failed_attempts)
+  mkdir -p "$RECOVER_DIR" 2>/dev/null || true
+  printf '%s\n%s\n' "$((next + 1))" "$(epoch_now)" >"$MARKER"
+}
+
+clear_failed_attempts() {
+  rm -f "$MARKER"
+}
+
+# --------------------------------------------------------------------------
+# decisions (pure; the test suite drives these directly)
+# --------------------------------------------------------------------------
+
+# The newest pre-upgrade config snapshot recent enough to implicate the current
+# failure. `upgrade` persists these durably before it touches anything
+# (configBackupDir(), config.json.pre-upgrade-*). A fresh one plus a dead
+# daemon is the one mechanical "the upgrade did it" signal we are allowed to
+# act on — fixing the new version's mistakes is a normal daemon-side repair,
+# but reinstalling the previous package pinned to a durable version is not
+# something this bounded playbook may improvise.
+fresh_rollback_snapshot() {
+  local newest ts now
+  newest=$(ls -t "$BACKUP_ROOT"/config.json.pre-upgrade-* 2>/dev/null | head -n 1 || true)
+  [[ -n $newest ]] || return 1
+  now=$(epoch_now)
+  ts=$(stat -c %Y "$newest" 2>/dev/null || true)
+  [[ -n $ts && $((now - ts)) -le $ROLLBACK_AGE_S ]] || return 1
+  printf '%s\n' "$newest"
+}
+
+# The single action for one failed unit: `ROLLBACK|<snapshot>`,
+# `RESTART_PEER|<herdr unit>`, `RECOVER_PANE|<herdr unit>` or `NONE|`.
+# Mechanically selected, never improvised.
+choose_action() { # <failed-unit>
+  local unit=$1 snap
+  if [[ $unit == "$DAEMON_UNIT" ]]; then
+    if snap=$(fresh_rollback_snapshot); then
+      printf 'ROLLBACK|%s\n' "$snap"
+      return
+    fi
+    printf 'RESTART_PEER|%s\n' "$HERDR_UNIT"
+    return
+  fi
+  if [[ $unit == "$HERDR_UNIT" ]]; then
+    printf 'RECOVER_PANE|%s\n' "$HERDR_UNIT"
+    return
+  fi
+  printf 'NONE|\n'
+}
+
+# --------------------------------------------------------------------------
+# main
+# --------------------------------------------------------------------------
+
+run_recovery() {
+  local failed="" u action_name="" action_arg="" ev="" outcome="" ok=1
+  for u in ${FLEET_UNITS//:/ }; do
+    if unit_is_failed "$u"; then
+      failed=$u
+      break
+    fi
+  done
+
+  if dry_run; then
+    say "dry run: nothing will be written or executed"
+    [[ -z $failed ]] && say "plan: no unit failed — escalate only"
+    if [[ -n $failed ]]; then
+      IFS='|' read -r action_name action_arg <<<"$(choose_action "$failed")"
+      say "plan: failed unit $failed; action: $action_name"
+    fi
+    say "plan: escalation — $(escalation_text "$failed" "${action_name:-none}" "dry run" "DRY-RUN")"
+    return 0
+  fi
+
+  ev=$(write_evidence "$failed")
+  if [[ -z $ev ]]; then
+    log "could not write evidence under $RECOVER_DIR — treating the recovery as failed"
+    return 1
+  fi
+
+  local attempts
+  if [[ -z $failed ]]; then
+    # Nothing failed when recovery ran (the fleet got itself up, or an
+    # external start beat this activation): always escalate, and a previously
+    # exhausted episode counter gets a fresh start.
+    say "no unit failed when this recovery ran — the fleet got itself up (or was started externally); nothing to attempt"
+    append_evidence "$ev" "decision: NONE — no unit failed"
+    if escalate none none "no unit failed when the recovery ran" "$ev"; then
+      clear_failed_attempts
+      return 0
+    fi
+    return 1
+  fi
+
+  attempts=$(read_failed_attempts)
+  if [[ $attempts -ge $MAX_FAILED_ATTEMPTS ]]; then
+    say "attempts exhausted: $attempts consecutive failed recoveries inside the ${FAIL_WINDOW_S}s window; escalating only"
+    append_evidence "$ev" "decision: NONE (attempts exhausted, $attempts)"
+    if escalate "$failed" none "attempts exhausted ($attempts); no further recovery" "$ev"; then
+      return 0
+    fi
+    return 1
+  fi
+
+  say "failed unit: $failed"
+  IFS='|' read -r action_name action_arg <<<"$(choose_action "$failed")"
+  action_name=${action_name:-NONE}
+  append_evidence "$ev" "decision: $action_name"
+  if [[ $action_name == NONE ]]; then
+    say "no safe mechanical action for $failed; escalating"
+    if escalate "$failed" none "no known-safe action" "$ev"; then
+      return 0
+    fi
+    return 1
+  fi
+
+  # Exactly one action, no matter how many units are down.
+  case "$action_name" in
+    ROLLBACK)
+      if cp "$action_arg" "$CONFIG_PATH" && start_unit "$failed"; then
+        if wait_no_longer_failed "$failed"; then
+          ok=0
+          outcome="restored pre-upgrade snapshot ${action_arg##*/} and started $failed"
+        else
+          outcome="restored pre-upgrade snapshot and started $failed, but it is still failing"
+        fi
+      else
+        outcome="could not restore snapshot ${action_arg##*/} or start $failed"
+      fi
+      ;;
+    RECOVER_PANE)
+      if start_unit "$HERDR_UNIT"; then
+        if wait_no_longer_failed "$HERDR_UNIT"; then
+          ok=0
+          outcome="restarted session host $HERDR_UNIT; its startup hook recovers the fleet pane"
+        else
+          outcome="$HERDR_UNIT started but is still failing"
+        fi
+      else
+        outcome="could not start $HERDR_UNIT"
+      fi
+      ;;
+    RESTART_PEER)
+      if "$SYSTEMCTL" restart "$action_arg"; then
+        outcome="restarted peer unit $action_arg"
+      else
+        outcome="could not restart peer unit $action_arg"
+      fi
+      ;;
+  esac
+  say "outcome: $outcome"
+  append_evidence "$ev" "outcome: $outcome"
+
+  if (( ok == 0 )); then
+    append_evidence "$ev" "verify: $failed no longer failed"
+    say "recovery succeeded: $failed no longer failed"
+    clear_failed_attempts
+  else
+    append_evidence "$ev" "verify: $failed still failed"
+    say "recovery failed: $failed still failed; attempts remaining until exhausted"
+    bump_failed_attempts
+  fi
+
+  if escalate "$failed" "$action_name" "$outcome" "$ev"; then
+    if (( ok == 0 )); then
+      return 0
+    fi
+    return 1
+  fi
+  return 1
+}
+
+main() {
+  dry_run && say "dry run: printing decisions, mutating nothing"
+  if run_recovery; then
+    exit 0
+  fi
+  exit 1
+}
+
+if [[ ${BASH_SOURCE[0]} == "$0" ]]; then
+  main
+fi

package/systemd/omp-conductor.service.example

@@ -16,6 +16,12 @@
 #   4. Do not co-locate ClickHouse + other multi-GB services beside a 2-worker
 #      fleet on a ≤8 GB box.
 #
+# OnFailure (issue #485): this line arms the fleet recovery oneshot that #485
+# ships via `setup host` — `omp-conductor-recover.service`, staged from
+# omp-conductor-recover.sh. Do not remove it: a crash-looped unit used to sit in
+# `failed` indefinitely with nobody watching it. Re-running `setup host`
+# provisions the recovery unit and installs the playbook itself.
+#
 # Install:
 #   sudo install -m 0644 omp-conductor.service.example /etc/systemd/system/omp-conductor.service
 #   sudo systemctl daemon-reload
@@ -27,6 +33,7 @@ Description=omp-conductor dispatch daemon
 Documentation=https://github.com/TerrifiedBug/conductor
 After=network-online.target
 Wants=network-online.target
+OnFailure=omp-conductor-recover.service
 
 [Service]
 Type=simple
@@ -44,9 +51,13 @@ WorkingDirectory=/home/fleet
 # under a unit, call `daemon` directly.
 ExecStart=/home/fleet/.local/bin/omp-conductor daemon --port 8787
 
-Restart=on-failure
-# Handled SIGTERM exits 143; without this, Restart=on-failure brings the unit
-# straight back after `systemctl stop` / `omp-conductor stop`.
+Restart=always
+# Restart on any exit — clean, signalled or crashed — except an explicit
+# `systemctl stop`, which systemd records as intentional and never undoes
+# (issue #546). A stray SIGTERM (e.g. a worker's `bun test`) then costs
+# seconds of downtime instead of leaving the fleet down until a human;
+# a crash loop still trips the start-limit burst and reaches OnFailure above.
+# SuccessExitStatus=0 143 keeps a handled SIGTERM shutdown logged as success.
 SuccessExitStatus=0 143
 
 MemoryAccounting=yes