omp-conductor 0.15.9 → 0.15.11

package/systemd/recover-unit-test.sh

@@ -0,0 +1,428 @@
+#!/usr/bin/env bash
+#
+# omp-conductor fleet recovery — offline tests for omp/systemd/omp-conductor-recover.sh
+#
+# No network, no real systemctl/journalctl/omp-conductor, and never a touch on
+# this host's units. Every external call goes through stub binaries on a
+# fixture PATH; a fixture state dir drives the unit state machine, so what is
+# under test is the playbook's decision: evidence written before anything,
+# exactly one bounded action from the fixed set, always an escalation through
+# the outbox, and no loop.
+#
+# Prints one line per case and exits non-zero if any of them fails.
+
+set -uo pipefail
+
+here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+recover="$here/omp-conductor-recover.sh"
+
+failures=0
+pass() { printf 'ok    %s\n' "$1"; }
+fail() {
+  printf 'FAIL  %s\n' "$1"
+  failures=$((failures + 1))
+}
+
+check() { # <name> <expected substring> <actual>
+  if [[ $3 == *"$2"* ]]; then
+    pass "$1"
+  else
+    fail "$1"
+    printf '      expected substring: %s\n' "$2"
+    printf '      actual:            %s\n' "$3"
+  fi
+}
+
+# --------------------------------------------------------------------------
+# syntax
+# --------------------------------------------------------------------------
+
+syntax_errors=$(
+  bash -n "$recover" 2>&1
+  bash -n "$here/recover-unit-test.sh" 2>&1
+)
+if [[ -z $syntax_errors ]]; then
+  pass 'syntax: recovery playbook and test suite'
+else
+  fail 'syntax: recovery playbook and test suite'
+  printf '%s\n' "$syntax_errors"
+fi
+
+tmp=$(mktemp -d "${TMPDIR:-/tmp}/omp-recover-test.XXXXXX")
+trap 'rm -rf "$tmp"' EXIT
+mkdir -p "$tmp/bin"
+
+# --------------------------------------------------------------------------
+# fixture stubs. The systemctl stub reads per-unit state files so tests can
+# decide which unit is failed and whether `start` heals it; every call is
+# logged so tests pin call ORDER (evidence before action) and the single
+# action rule.
+# --------------------------------------------------------------------------
+
+cat >"$tmp/bin/systemctl" <<'STUB'
+#!/usr/bin/env bash
+echo "systemctl $*" >>"$STUB_CALLS"
+unit="$2"
+state_path="$STUB_UNITS/$unit"
+state=$(cat "$state_path" 2>/dev/null || echo absent)
+case "$1" in
+  is-failed)
+    [[ $state == failed ]] && echo failed || echo inactive ;;
+  is-active)
+    if [[ $state == failed ]]; then echo failed; elif [[ $state == absent ]]; then echo inactive; else echo active; fi ;;
+  reset-failed) : ;;
+  start)
+    [[ $unit == "$STUB_HEAL_ON_START" ]] && echo active > "$state_path" ;;
+  restart)
+    echo active > "$state_path" ;;
+  *)
+    echo "unexpected systemctl $*" >&2 ;;
+esac
+STUB
+
+cat >"$tmp/bin/journalctl" <<'STUB'
+#!/usr/bin/env bash
+echo "journalctl $*" >>"$STUB_CALLS"
+echo "journal: fleet crashed here"
+echo "journal: last line before death"
+STUB
+
+cat >"$tmp/bin/omp-conductor" <<'STUB'
+#!/usr/bin/env bash
+echo "omp-conductor $*" >>"$STUB_CALLS"
+case "$1" in
+  doctor)
+    printf '%s\n' '{"project":"demo","status":"fail","findings":[{"id":"systemd-unit","status":"fail","summary":"installed differs"}]}'
+    ;;
+  --version)
+    echo "0.15.11"
+    ;;
+  report)
+    echo "report 485-abc queued for demo (tier2)"
+    ;;
+  *)
+    exit 2 ;;
+esac
+STUB
+
+chmod +x "$tmp/bin/"*
+
+# --------------------------------------------------------------------------
+# fixture helpers
+# --------------------------------------------------------------------------
+
+newcase() { # <name> -> case dir
+  local d="$tmp/case-$1"
+  mkdir -p "$d/state/backups/config" "$d/units"
+  printf '%s\n' "$d"
+}
+
+unit_state() { # <case-dir> <unit> <state|absent>
+  if [[ $3 == absent ]]; then
+    rm -f "$1/units/$2"
+  else
+    printf '%s\n' "$3" >"$1/units/$2"
+  fi
+}
+
+# A pre-upgrade snapshot `upgrade` durably keeps (configBackupDir()):
+# config.json.pre-upgrade-<ts>. The test pass the age in seconds.
+write_snapshot() { # <case-dir> <age-seconds>
+  local d="$1" age="${2:-0}" now
+  now=$(date +%s)
+  local snap="$d/state/backups/config/config.json.pre-upgrade-$now"
+  printf '%s\n' '{"preUpgrade":true}' >"$snap"
+  if (( age > 0 )); then
+    touch -d "@$(( now - age ))" "$snap"
+  fi
+  printf '%s\n' "$snap"
+}
+
+marker() { # <case-dir> -> attempts count or 0
+  local f="$1/state/recovery/attempts"
+  if [[ -f $f ]]; then
+    sed -n '1p' "$f"
+  else
+    echo 0
+  fi
+}
+
+run_recover() { # <case-dir> [env assignments…]
+  local d="$1"
+  shift
+  (
+    cd "$here" || exit 1
+    env \
+      PATH="$tmp/bin:/usr/bin:/bin" \
+      SYSTEMCTL_BIN="$tmp/bin/systemctl" \
+      JOURNALCTL_BIN="$tmp/bin/journalctl" \
+      OMP_CONDUCTOR_BIN="$tmp/bin/omp-conductor" \
+      RECOVER_STATE_DIR="$d/state" \
+      RECOVER_PROJECT=demo \
+      RECOVER_VERIFY_POLL_S=0 \
+      RECOVER_VERIFY_TRIES=3 \
+      STUB_CALLS="$d/calls" \
+      STUB_UNITS="$d/units" \
+      "$@" \
+      bash "$recover" >"$d/out" 2>"$d/stderr"
+  )
+  echo $?
+}
+
+# `systemctl restart` never heals the FAILED unit in this fixture (it only
+# marks the named unit active, which is what a real peer restart does to a
+# healthy peer), so tests wanting daemon-side healing pass HEAL_ON_START=<unit>
+# and the stub's `start` flips that unit back to active.
+run_with_heal() { # <case-dir> <heal-on-start>
+  (
+    cd "$here" || exit 1
+    env \
+      PATH="$tmp/bin:/usr/bin:/bin" \
+      SYSTEMCTL_BIN="$tmp/bin/systemctl" \
+      JOURNALCTL_BIN="$tmp/bin/journalctl" \
+      OMP_CONDUCTOR_BIN="$tmp/bin/omp-conductor" \
+      RECOVER_STATE_DIR="$1/state" \
+      RECOVER_PROJECT=demo \
+      RECOVER_VERIFY_POLL_S=0 \
+      RECOVER_VERIFY_TRIES=3 \
+      STUB_CALLS="$1/calls" \
+      STUB_UNITS="$1/units" \
+      STUB_HEAL_ON_START="$2" \
+      bash "$recover" >"$1/out" 2>"$1/stderr"
+  )
+  echo $?
+}
+
+# --------------------------------------------------------------------------
+# the decision functions, driven directly. The unit text holds the other half
+# of the bounds (StartLimitBurst=2, no OnFailure=) — setup-host.test.ts pins it.
+# --------------------------------------------------------------------------
+
+SCAFFOLD=$(newcase decisions)
+FLEET_UNITS=herdr-fleet.service
+RECOVER_STATE_DIR="$SCAFFOLD/state"
+. "$recover"
+
+# RECOVER_PANE is the only action for a failed herdr-fleet unit: herdr's own
+# startup hook recovers the fleet pane, so the pane leg is `start herdr`.
+check 'herdr failure chooses RECOVER_PANE' \
+  'RECOVER_PANE|herdr-fleet.service' "$(choose_action herdr-fleet.service)"
+
+D2=$(newcase decisions-daemon)
+RECOVER_STATE_DIR="$D2/state"
+BACKUP_ROOT="$D2/state/backups/config"
+
+# The daemon's action hinges on a fresh pre-upgrade snapshot: with one, the
+# mechanical conclusion is "the upgrade did it" and the action is ROLLBACK.
+write_snapshot "$D2" 0 >/dev/null
+check 'a fresh pre-upgrade snapshot chooses ROLLBACK' \
+  "ROLLBACK|$BACKUP_ROOT/config.json.pre-upgrade-" "$(choose_action omp-conductor.service)"
+
+# Without one, the only safe motion is restarting the peer unit.
+rm -f "$BACKUP_ROOT"/config.json.pre-upgrade-*
+check 'no snapshot chooses RESTART_PEER' \
+  'RESTART_PEER|herdr-fleet.service' "$(choose_action omp-conductor.service)"
+
+# A snapshot too old to implicate THIS failure is not evidence.
+write_snapshot "$D2" 7200 >/dev/null
+check 'a stale snapshot does not choose ROLLBACK' \
+  'RESTART_PEER|herdr-fleet.service' "$(choose_action omp-conductor.service)"
+unset FLEET_UNITS RECOVER_STATE_DIR BACKUP_ROOT
+
+# ---------------------------------------------------------------------------
+# case A — RECOVER_PANE: herdr-fleet failed and the start heals it in time.
+# Evidence must be written before a single systemd mutation is attempted.
+# ---------------------------------------------------------------------------
+
+d=$(newcase recover-pane)
+unit_state "$d" herdr-fleet.service failed
+unit_state "$d" omp-conductor.service active
+code=$(run_with_heal "$d" herdr-fleet.service)
+check 'RECOVER_PANE exits 0 when the unit heals' '0' "$code"
+check 'RECOVER_PANE reports success and names the pane hook' \
+  'startup hook recovers the fleet pane' "$(grep 'outcome:' "$d/out")"
+
+# Order the acceptance pins: journal + doctor + version (evidence) strictly
+# before the first systemd mutation (reset-failed/start).
+journal_line=$(grep -n '^journalctl' "$d/calls" | head -n 1 | cut -d: -f1 | tr -d ' ')
+reset_line=$(grep -n '^systemctl reset-failed' "$d/calls" | head -n 1 | cut -d: -f1 | tr -d ' ')
+if [[ -n $journal_line && -n $reset_line && $journal_line -lt $reset_line ]]; then
+  pass 'evidence collection precedes the single action'
+else
+  fail "evidence collection precedes the single action (journal=$journal_line reset=$reset_line)"
+fi
+
+# Exactly one restorative action: one reset-failed + one start, no restart.
+starts=$(grep -c '^systemctl start ' "$d/calls" || true)
+resets=$(grep -c '^systemctl reset-failed ' "$d/calls" || true)
+restarts=$(grep -c '^systemctl restart ' "$d/calls" || true)
+if [[ $starts == 1 && $resets == 1 && $restarts == 0 ]]; then
+  pass 'exactly one restorative action (start herdr-fleet)'
+else
+  fail "exactly one restorative action — starts=$starts resets=$resets restarts=$restarts"
+fi
+
+check 'successful recovery still escalates tier-2 through the outbox' \
+  'omp-conductor report --kind tier2' "$(grep '^omp-conductor report' "$d/calls" | head -n 1)"
+check 'the escalation names the durable evidence file' \
+  "$d/state/recovery/evidence-" "$(grep '^omp-conductor report' "$d/calls" | head -n 1)"
+# The evidence bundle itself carries the collected facts.
+ev=$(ls "$d/state/recovery"/evidence-* | head -n 1)
+check 'the evidence bundle holds unit state, doctor JSON, versions and journal' \
+  '=== omp-conductor fleet recovery evidence' "$(head -n 1 "$ev")"
+check 'the evidence bundle names the failed unit' \
+  'herdr-fleet.service' "$(grep '^failed unit:' "$ev")"
+check 'the evidence bundle carries the doctor output' \
+  '"systemd-unit"' "$(grep 'installed differs' "$ev")"
+check 'the evidence bundle carries the journal tail' \
+  'journal: last line before death' "$(grep 'journal:' "$ev")"
+
+[[ $(marker "$d") == 0 ]]
+pass 'the attempts counter is cleared after a successful recovery'
+
+# ---------------------------------------------------------------------------
+# case ROLLBACK: daemon failed AND a fresh pre-upgrade snapshot exists.
+# ---------------------------------------------------------------------------
+
+d=$(newcase rollback)
+write_snapshot "$d" 5 >/dev/null
+printf '%s\n' '{"live":true}' >"$d/state/config.json"
+unit_state "$d" omp-conductor.service failed
+unit_state "$d" herdr-fleet.service active
+code=$(run_with_heal "$d" omp-conductor.service)
+check 'ROLLBACK exits 0 when the daemon heals' '0' "$code"
+check 'ROLLBACK restores the pre-upgrade bytes over config.json' \
+  '{"preUpgrade":true}' "$(cat "$d/state/config.json")"
+check 'ROLLBACK starts the daemon unit' \
+  'systemctl start omp-conductor.service' "$(grep '^systemctl start ' "$d/calls" | head -n 1)"
+check 'ROLLBACK escalates as well' \
+  'omp-conductor report --kind tier2' "$(grep '^omp-conductor report' "$d/calls" | tail -n 1)"
+check 'the pre-upgrade snapshot survives the rollback for the operator' \
+  '{"preUpgrade":true}' "$(cat "$d/state/backups/config"/config.json.pre-upgrade-*)"
+
+# ---------------------------------------------------------------------------
+# case RESTART_PEER and the bound: attempts cap at two, then escalate-only.
+# A daemon crash with no upgrade evidence must never get a blind daemon start.
+# ---------------------------------------------------------------------------
+
+d=$(newcase restart-peer)
+unit_state "$d" omp-conductor.service failed
+unit_state "$d" herdr-fleet.service active
+
+code=$(run_recover "$d")
+check 'RESTART_PEER restarts exactly the peer unit and nothing else' \
+  'systemctl restart herdr-fleet.service' "$(cat "$d/calls")"
+check 'RESTART_PEER never starts the failed daemon' \
+  '' "$(grep '^systemctl start omp-conductor' "$d/calls" || true)"
+check 'a peer restart that leaves the daemon failed exits 1' '1' "$code"
+check 'the failed attempt is recorded' '1' "$(marker "$d")"
+
+# A second trigger in the same episode is one more attempt, not a loop…
+code=$(run_recover "$d")
+check 'a second attempt is allowed below the cap and also exits 1' '1' "$code"
+check 'two failed attempts are recorded' '2' "$(marker "$d")"
+
+# …and the third trigger only escalates: no mutating systemd call at all
+# (is-failed probes for evidence are reads, not actions).
+mutating() { grep -c '^systemctl \(start\|restart\|reset-failed\) ' "$1" || true; }
+before=$(mutating "$d/calls")
+code=$(run_recover "$d")
+after=$(mutating "$d/calls")
+check 'attempts exhausted — the run says so' \
+  'attempts exhausted' "$(cat "$d/out")"
+check 'an exhausted run performs no systemd action' \
+  "$before" "$after"
+check 'the escalation is still queued after exhaustion' \
+  'omp-conductor report --kind tier2' "$(grep '^omp-conductor report' "$d/calls" | tail -n 1)"
+
+# ---------------------------------------------------------------------------
+# case — nothing failed when recovery ran (the "got itself up" case): always
+# escalate, and a previously exhausted episode counter gets a fresh start.
+# ---------------------------------------------------------------------------
+
+d=$(newcase nothing-failed)
+mkdir -p "$d/state/recovery"
+printf '3\n%s\n' "$(date +%s)" >"$d/state/recovery/attempts"
+unit_state "$d" omp-conductor.service active
+unit_state "$d" herdr-fleet.service active
+code=$(run_recover "$d")
+check 'a trigger with nothing failed escalates (fleet came back)' \
+  'no unit failed' "$(cat "$d/out")"
+check 'a trigger with nothing failed exits 0' '0' "$code"
+check 'a trigger with nothing failed clears the old counter' '0' "$(marker "$d")"
+
+# ---------------------------------------------------------------------------
+# case DRY-RUN — the operator's rehearsal mode prints decisions and mutates
+# nothing, not even the recovery dir or the fixture call log.
+# ---------------------------------------------------------------------------
+
+d=$(newcase dry-run)
+unit_state "$d" herdr-fleet.service failed
+unit_state "$d" omp-conductor.service active
+(
+  cd "$here" || exit 1
+  env \
+    PATH="$tmp/bin:/usr/bin:/bin" \
+    SYSTEMCTL_BIN="$tmp/bin/systemctl" \
+    JOURNALCTL_BIN="$tmp/bin/journalctl" \
+    OMP_CONDUCTOR_BIN="$tmp/bin/omp-conductor" \
+    RECOVER_STATE_DIR="$d/state" \
+    RECOVER_PROJECT=demo \
+    RECOVER_DRY_RUN=1 \
+    STUB_CALLS="$d/calls" \
+    STUB_UNITS="$d/units" \
+    bash "$recover" >"$d/out" 2>"$d/stderr"
+)
+check 'a dry run names the action and the escalation' \
+  'action: RECOVER_PANE' "$(cat "$d/out")"
+check 'a dry run performs no systemctl call' \
+  '' "$(cat "$d/calls" 2>/dev/null || true)"
+check 'a dry run writes no evidence or counter' \
+  '' "$(ls "$d/state/recovery" 2>/dev/null || true)"
+
+# ---------------------------------------------------------------------------
+# case — escalation failure fails the recovery (bounded), evidence still there.
+# ---------------------------------------------------------------------------
+
+d=$(newcase escalation-fail)
+mkdir -p "$d/bin"
+cat >"$d/bin/omp-conductor" <<'STUB'
+#!/usr/bin/env bash
+echo "omp-conductor $*" >>"$STUB_CALLS"
+case "$1" in
+  doctor) echo '{"status":"pass"}'; exit 0 ;;
+  --version) echo "0.15.11" ;;
+  report) exit 3 ;;
+esac
+STUB
+chmod +x "$d/bin/omp-conductor"
+unit_state "$d" herdr-fleet.service failed
+unit_state "$d" omp-conductor.service absent
+(
+  cd "$here" || exit 1
+  env \
+    PATH="$tmp/bin:/usr/bin:/bin" \
+    SYSTEMCTL_BIN="$tmp/bin/systemctl" \
+    JOURNALCTL_BIN="$tmp/bin/journalctl" \
+    OMP_CONDUCTOR_BIN="$d/bin/omp-conductor" \
+    RECOVER_STATE_DIR="$d/state" \
+    RECOVER_PROJECT=demo \
+    RECOVER_VERIFY_POLL_S=0 \
+    RECOVER_VERIFY_TRIES=2 \
+    STUB_CALLS="$d/calls" \
+    STUB_UNITS="$d/units" \
+    bash "$recover" >"$d/out" 2>"$d/stderr"
+)
+code=$?
+check 'a failed escalation fails the recovery run' '1' "$code"
+check 'the evidence file exists even when the escalation failed' \
+  '=== omp-conductor fleet recovery evidence (#485) ===' "$(head -n 1 "$d/state/recovery"/evidence-*)"
+
+# ---------------------------------------------------------------------------
+
+if (( failures > 0 )); then
+  printf 'FAIL: %s test(s) failed\n' "$failures"
+  exit 1
+fi
+echo 'all tests passed'
+exit 0