omp-conductor 0.19.5 → 0.19.6

package/systemd/omp-conductor-recover.sh

@@ -61,6 +61,7 @@ PROG=omp-conductor-recover
 SYSTEMCTL=${SYSTEMCTL_BIN:-systemctl}
 JOURNALCTL=${JOURNALCTL_BIN:-journalctl}
 CONDUCTOR=${OMP_CONDUCTOR_BIN:-omp-conductor}
+BUN=${BUN_BIN:-bun}
 
 # The fleet's conductor home: where config.json, the sqlite store and the
 # upgrade's durable snapshots live. The unit renders OMP_CONDUCTOR_HOME and
@@ -84,6 +85,12 @@ JOURNAL_LINES=${RECOVER_JOURNAL_LINES:-200}
 VERIFY_POLL_S=${RECOVER_VERIFY_POLL_S:-2}
 VERIFY_TRIES=${RECOVER_VERIFY_TRIES:-10}
 PROJECT=${RECOVER_PROJECT:-}
+# Recovery action scope and durable-report ownership are deliberately separate.
+# `RECOVER_PROJECT` is only safe on a single-project host; the report still
+# needs one configured project to own its outbox row on a shared host.
+REPORT_PROJECT=${RECOVER_REPORT_PROJECT:-$PROJECT}
+AFFECTED_PROJECTS=${RECOVER_PROJECTS:-$PROJECT}
+REPORT_ROUTING_NOTE=
 # The bound on the post-restore re-arm. `arm` with a claim-only proof performs
 # no Telegram send or wait, so it completes in milliseconds; the timeout exists
 # so a misbehaving conductor can never hang this bounded playbook (#613).
@@ -190,6 +197,40 @@ append_evidence() { # <evidence path> <text>
   printf '%s\n' "$2" >>"$1" 2>/dev/null || true
 }
 
+
+# Resolve the host-global report owner from the live machine-written config.
+# Bun is not an extra prerequisite: omp-conductor itself runs under Bun, and
+# the recovery unit's canonical PATH already contains that executable. When a
+# manual/degraded invocation cannot read the config, the script still attempts
+# the legacy unscoped report and records that degradation in the evidence.
+resolve_report_scope() {
+  if [[ -n $REPORT_PROJECT && -n $AFFECTED_PROJECTS ]]; then
+    return 0
+  fi
+  local lines first all
+  if lines=$(
+    "$BUN" -e '
+      const path = Bun.argv.at(-1);
+      const config = await Bun.file(path).json();
+      const names = Array.isArray(config.projects)
+        ? config.projects.map((project) => project?.name).filter((name) => typeof name === "string" && name.length > 0)
+        : [];
+      if (names.length === 0) process.exit(2);
+      console.log(names[0]);
+      console.log(names.join(","));
+    ' "$CONFIG_PATH" 2>/dev/null
+  ); then
+    first=${lines%%$'\n'*}
+    all=${lines#*$'\n'}
+    REPORT_PROJECT=${REPORT_PROJECT:-$first}
+    AFFECTED_PROJECTS=${AFFECTED_PROJECTS:-$all}
+  fi
+  if [[ -z $REPORT_PROJECT || -z $AFFECTED_PROJECTS ]]; then
+    AFFECTED_PROJECTS=${AFFECTED_PROJECTS:-unknown}
+    REPORT_ROUTING_NOTE="configured project resolution failed for $CONFIG_PATH; attempting the report without a project selector"
+  fi
+  return 0
+}
 # --------------------------------------------------------------------------
 # escalation — always, through the durable outbox (#123, #288)
 # --------------------------------------------------------------------------
@@ -199,23 +240,27 @@ append_evidence() { # <evidence path> <text>
 # (the durable outbox) before anything is sent, and the daemon owns delivery
 # with bounded retries once it is back. This script only ever enqueues.
 escalation_text() { # <failed-unit or empty> <action> <outcome> <evidence>
+  local projects=${AFFECTED_PROJECTS:-unknown}
   if [[ ${1:-none} == none ]]; then
     printf '%s\n' \
-      "fleet recovery (#485): a recovery trigger fired but no fleet unit was failing when it ran — the fleet got itself up or was started externally; evidence=$4"
+      "fleet recovery (#485): a recovery trigger fired but no fleet unit was failing when it ran — the fleet got itself up or was started externally; affected-projects=$projects; evidence=$4"
     return
   fi
   printf '%s\n' \
-    "fleet recovery (#485): $1 entered failed state; recovery=${2:-none}; outcome=${3}; evidence=${4}; next: read the evidence file (and its doctor output) before touching the fleet"
+    "fleet recovery (#485): $1 entered failed state; affected-projects=$projects; recovery=${2:-none}; outcome=${3}; evidence=${4}; next: read the evidence file (and its doctor output) before touching the fleet"
 }
 
 escalate() { # <failed-unit or empty> <action-name> <outcome> <evidence-path>
   local unit=${1:-none} text args
-  text=$(escalation_text "$unit" "$2" "$3" "$4")
-  if [[ -n $PROJECT ]]; then
-    args=("$CONDUCTOR" report --kind tier2 --project "$PROJECT" --text "$text")
-  else
-    args=("$CONDUCTOR" report --kind tier2 --text "$text")
+  resolve_report_scope
+  if [[ -n $REPORT_ROUTING_NOTE ]]; then
+    log "$REPORT_ROUTING_NOTE"
+    append_evidence "$4" "report routing: $REPORT_ROUTING_NOTE"
   fi
+  text=$(escalation_text "$unit" "$2" "$3" "$4")
+  args=("$CONDUCTOR" report --kind tier2)
+  [[ -n $REPORT_PROJECT ]] && args+=(--project "$REPORT_PROJECT")
+  args+=(--text "$text")
   say "escalation: ${args[*]}"
   if dry_run; then
     return 0
@@ -365,6 +410,9 @@ run_recovery() {
   done
 
   if dry_run; then
+    resolve_report_scope
+    say "plan: report owner — ${REPORT_PROJECT:-unscoped}; affected projects — $AFFECTED_PROJECTS"
+    [[ -n $REPORT_ROUTING_NOTE ]] && say "plan: report routing degraded — $REPORT_ROUTING_NOTE"
     say "dry run: nothing will be written or executed"
     [[ -z $failed ]] && say "plan: no unit failed — escalate only"
     if [[ -n $failed ]]; then

package/systemd/recover-unit-test.sh

@@ -50,6 +50,7 @@ fi
 
 tmp=$(mktemp -d "${TMPDIR:-/tmp}/omp-recover-test.XXXXXX")
 trap 'rm -rf "$tmp"' EXIT
+bun_bin=$(command -v bun)
 mkdir -p "$tmp/bin"
 
 # --------------------------------------------------------------------------
@@ -156,6 +157,18 @@ write_project_config() { # <case-dir> <proof>
   printf '{\n  "version": 2,\n  "projects": [\n    { "name": "demo", "arm": { "proof": "%s" } }\n  ]\n}\n' "$2" >"$1/state/config.json"
 }
 
+write_multi_project_config() { # <case-dir>
+  cat >"$1/state/config.json" <<'JSON'
+{
+  "version": 2,
+  "projects": [
+    { "name": "alpha" },
+    { "name": "beta" }
+  ]
+}
+JSON
+}
+
 run_recover() { # <case-dir> [env assignments…]
   local d="$1"
   shift
@@ -182,7 +195,9 @@ run_recover() { # <case-dir> [env assignments…]
 # marks the named unit active, which is what a real peer restart does to a
 # healthy peer), so tests wanting daemon-side healing pass HEAL_ON_START=<unit>
 # and the stub's `start` flips that unit back to active.
-run_with_heal() { # <case-dir> <heal-on-start>
+run_with_heal() { # <case-dir> <heal-on-start> [env assignments…]
+  local d="$1" heal="$2"
+  shift 2
   (
     cd "$here" || exit 1
     env \
@@ -190,14 +205,15 @@ run_with_heal() { # <case-dir> <heal-on-start>
       SYSTEMCTL_BIN="$tmp/bin/systemctl" \
       JOURNALCTL_BIN="$tmp/bin/journalctl" \
       OMP_CONDUCTOR_BIN="$tmp/bin/omp-conductor" \
-      RECOVER_STATE_DIR="$1/state" \
+      RECOVER_STATE_DIR="$d/state" \
       RECOVER_PROJECT=demo \
       RECOVER_VERIFY_POLL_S=0 \
       RECOVER_VERIFY_TRIES=3 \
-      STUB_CALLS="$1/calls" \
-      STUB_UNITS="$1/units" \
-      STUB_HEAL_ON_START="$2" \
-      bash "$recover" >"$1/out" 2>"$1/stderr"
+      STUB_CALLS="$d/calls" \
+      STUB_UNITS="$d/units" \
+      STUB_HEAL_ON_START="$heal" \
+      "$@" \
+      bash "$recover" >"$d/out" 2>"$d/stderr"
   )
   echo $?
 }
@@ -412,6 +428,83 @@ check 'a trigger with nothing failed escalates (fleet came back)' \
 check 'a trigger with nothing failed exits 0' '0' "$code"
 check 'a trigger with nothing failed clears the old counter' '0' "$(marker "$d")"
 
+# ---------------------------------------------------------------------------
+# case — a host-global recovery on a multi-project config still has one
+# deterministic outbox owner, while its body names every affected project.
+# ---------------------------------------------------------------------------
+
+d=$(newcase multi-project-report)
+write_multi_project_config "$d"
+unit_state "$d" omp-conductor.service active
+unit_state "$d" herdr-fleet.service active
+code=$(run_recover "$d" \
+  RECOVER_PROJECT= \
+  RECOVER_REPORT_PROJECT= \
+  RECOVER_PROJECTS= \
+  BUN_BIN="$bun_bin")
+report_call=$(grep '^omp-conductor report' "$d/calls" | head -n 1)
+check 'multi-project recovery exits after queuing its report' '0' "$code"
+check 'multi-project recovery uses the first configured outbox owner' \
+  '--project alpha' "$report_call"
+check 'multi-project recovery names every affected project' \
+  'affected-projects=alpha,beta' "$report_call"
+check 'multi-project recovery queues exactly one report' \
+  '1' "$(grep -c '^omp-conductor report' "$d/calls")"
+
+d=$(newcase multi-project-recovery-action)
+write_multi_project_config "$d"
+unit_state "$d" omp-conductor.service active
+unit_state "$d" herdr-fleet.service failed
+code=$(run_with_heal "$d" herdr-fleet.service \
+  RECOVER_PROJECT= \
+  RECOVER_REPORT_PROJECT= \
+  RECOVER_PROJECTS= \
+  BUN_BIN="$bun_bin")
+report_call=$(grep '^omp-conductor report' "$d/calls" | head -n 1)
+check 'verified multi-project recovery action exits cleanly' '0' "$code"
+check 'verified multi-project recovery action queues one scoped report' \
+  '1' "$(grep -c '^omp-conductor report' "$d/calls")"
+check 'verified multi-project recovery action uses configured owner' \
+  '--project alpha' "$report_call"
+check 'verified multi-project recovery action names every project' \
+  'affected-projects=alpha,beta' "$report_call"
+
+d=$(newcase multi-project-dry-run)
+write_multi_project_config "$d"
+unit_state "$d" omp-conductor.service active
+unit_state "$d" herdr-fleet.service failed
+code=$(run_recover "$d" \
+  RECOVER_PROJECT= \
+  RECOVER_REPORT_PROJECT= \
+  RECOVER_PROJECTS= \
+  RECOVER_DRY_RUN=1 \
+  BUN_BIN="$bun_bin")
+check 'multi-project dry run exits cleanly' '0' "$code"
+check 'multi-project dry run names its report owner' \
+  'plan: report owner — alpha' "$(cat "$d/out")"
+check 'multi-project dry run names every affected project' \
+  'affected projects — alpha,beta' "$(cat "$d/out")"
+
+d=$(newcase report-routing-fallback)
+unit_state "$d" omp-conductor.service active
+unit_state "$d" herdr-fleet.service active
+code=$(run_recover "$d" \
+  RECOVER_PROJECT= \
+  RECOVER_REPORT_PROJECT= \
+  RECOVER_PROJECTS= \
+  BUN_BIN="$bun_bin")
+report_call=$(grep '^omp-conductor report' "$d/calls" | head -n 1)
+check 'unreadable config routing still attempts a durable report' \
+  'omp-conductor report --kind tier2 --text' "$report_call"
+if [[ $report_call == *"--project "* ]]; then
+  fail "routing fallback does not guess an unconfigured project — $report_call"
+else
+  pass 'routing fallback does not guess an unconfigured project'
+fi
+check 'routing fallback remains a successful recovered run' '0' "$code"
+check 'routing fallback is preserved in durable evidence' \
+  'attempting the report without a project selector' "$(cat "$d/state/recovery"/evidence-*)"
+
 # ---------------------------------------------------------------------------
 # case DRY-RUN — the operator's rehearsal mode prints decisions and mutates
 # nothing, not even the recovery dir or the fixture call log.