mandrel-platform 1.0.1 → 1.2.0

package/templates/runner/check-runner-env-drift.sh

@@ -0,0 +1,248 @@
+#!/usr/bin/env bash
+#
+# check-runner-env-drift.sh — report per-runner `.env` configuration drift
+# across one host's runner pool (mandrel-platform runner kit).
+#
+# ── WHY THIS EXISTS ─────────────────────────────────────────────────────────
+#
+# Nothing else observes a runner's LOCAL configuration. The fleet monitor
+# (`scripts/check-runner-health.mjs`) reaches runners through
+# `GET /repos/{owner}/{repo}/actions/runners`, which reports a runner's name,
+# labels and online status — the endpoint cannot see `<RUNNER_DIR>/.env`, so
+# hook configuration is invisible to it.
+#
+# The consequence is that a partially-provisioned pool never presents as a
+# configuration fault. It presents as an unattributable behavioural difference
+# between two runs of the SAME job on the SAME repo. That is issue #343: on one
+# host, 16 of 19 runners carried ACTIONS_RUNNER_HOOK_JOB_STARTED; two of the
+# hooked ones sat 5m29s in `Set up runner` while the same job on an unhooked
+# runner finished in 54 seconds. Attributing that took far longer than reading
+# nineteen `.env` files would have — which is precisely what this script does.
+#
+# ── WHAT IT REPORTS ─────────────────────────────────────────────────────────
+#
+# PRESENCE — never value — of the four keys `.env.example` mandates:
+# ACTIONS_RUNNER_HOOK_JOB_STARTED, RUNNER_TOOL_CACHE, AGENT_TOOLSDIRECTORY,
+# LANG. Values are deliberately not compared: every one of them embeds the
+# runner's own absolute root path, so they are SUPPOSED to differ per runner.
+#
+# The drift signal is a key set on SOME runners but not all — the 16-of-19
+# shape. A key absent from EVERY runner is a uniform gap: reported as such, and
+# not on its own a non-zero exit. A fleet that has deliberately not adopted a
+# key must not be a standing alarm, or the operator learns to ignore the exit
+# code and the signal is worth nothing when it does fire.
+#
+# ── OPERATOR CONTRACT ───────────────────────────────────────────────────────
+#
+#   check-runner-env-drift.sh [--pool-root <dir>]
+#
+#   --pool-root <dir>  Directory holding one subdirectory per runner. Defaults
+#                      to the PARENT of the directory containing this script:
+#                      the kit installs it into <RUNNER_DIR>, and the runbook
+#                      mandates one directory per runner under a common root,
+#                      so the default is correct on any kit-provisioned host.
+#
+#   A child directory counts as a runner iff it contains `config.sh`. That
+#   predicate keeps unrelated siblings (shared caches, scratch dirs) out of the
+#   report without inventing a naming convention.
+#
+#   exit 0 — no drift: every mandated key is uniform across the pool (set
+#            everywhere, or unset everywhere).
+#   exit 1 — drift: at least one key is set on some runners but not all. The
+#            non-zero exit IS the alert channel, matching the posture
+#            `scripts/check-runner-health.mjs` already uses, so this can be
+#            scheduled.
+#   exit 2 — usage error: unknown flag, or a pool root that is not a directory
+#            or holds no runners. Deliberately distinct from 0: reporting "no
+#            drift" over an empty walk would read as evidence the fleet is
+#            uniform.
+#
+# READ-ONLY, and never fails soft on a broken runner. It writes nothing into a
+# runner root and never touches a launchd service. A runner whose `.env` is
+# missing or unreadable is recorded as all four keys unset and the walk
+# continues — one broken runner must not shrink the sample the verdict is
+# computed over.
+#
+# This is an OPERATOR-run tool, not a job hook. Do not wire it into
+# ACTIONS_RUNNER_HOOK_JOB_STARTED: that hook runs inside the job's clock, where
+# every read is billed to `Set up runner` and counts against the job's
+# `timeout-minutes` (issue #343). A pool-wide walk belongs outside that clock.
+#
+# ── PORTABILITY ─────────────────────────────────────────────────────────────
+#
+# Runs on the host with no repo checkout and no Node runtime, and stays
+# compatible with macOS's system bash 3.2 — the same constraint
+# `.github/actions/gitleaks-scan/action.yml` documents for this fleet. That
+# rules out `declare -A`, `mapfile`/`readarray`, and `${var,,}`; it does NOT
+# rule out plain INDEXED arrays, which 3.2 supports and which the accumulators
+# below use. Only the per-key tally needs a second pass, because keeping a
+# key->count table is the one thing an indexed array cannot do.
+#
+# Accumulating into arrays rather than splitting a delimited string on a
+# reassigned `IFS` is deliberate and load-bearing: reassigning IFS globally is
+# flagged by the platform's own SAST ruleset (`bash.lang.security.ifs-tampering`)
+# because it silently changes the splitting behaviour of every later unquoted
+# expansion in the script. Arrays give the same grouping with no global state
+# and no quoting hazard for a runner directory whose name contains whitespace.
+
+set -u
+
+MANDATED_KEYS=(
+  ACTIONS_RUNNER_HOOK_JOB_STARTED
+  RUNNER_TOOL_CACHE
+  AGENT_TOOLSDIRECTORY
+  LANG
+)
+
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+POOL_ROOT=$(dirname "$SCRIPT_DIR")
+
+usage() {
+  cat <<'USAGE'
+Usage: check-runner-env-drift.sh [--pool-root <dir>]
+
+Reports which runners in a pool are missing the `.env` keys the runner kit
+mandates. Read-only.
+
+  --pool-root <dir>   Pool root (default: the parent of this script's dir).
+  -h, --help          Show this help.
+
+Exit: 0 no drift · 1 drift · 2 usage error.
+USAGE
+}
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --pool-root)
+      if [ $# -lt 2 ]; then
+        printf 'check-runner-env-drift: --pool-root requires a directory\n' >&2
+        exit 2
+      fi
+      POOL_ROOT=$2
+      shift 2
+      ;;
+    --pool-root=*)
+      POOL_ROOT=${1#--pool-root=}
+      shift
+      ;;
+    -h | --help)
+      usage
+      exit 0
+      ;;
+    *)
+      printf 'check-runner-env-drift: unknown argument: %s\n' "$1" >&2
+      usage >&2
+      exit 2
+      ;;
+  esac
+done
+
+if [ ! -d "$POOL_ROOT" ]; then
+  printf 'check-runner-env-drift: pool root is not a directory: %s\n' "$POOL_ROOT" >&2
+  exit 2
+fi
+POOL_ROOT=$(cd "$POOL_ROOT" && pwd)
+
+# Presence test for one key in one runner's `.env`.
+#
+# The assignment form is `^[[:space:]]*KEY=` on a non-comment line. A commented
+# line cannot match (the `#` is not whitespace), which is the case that matters:
+# `.env.example` ships every key inside a block of explanatory prose, so a
+# half-applied copy where the operator never uncommented a line is the most
+# likely real drift shape — and matching the key name anywhere in the file would
+# report that runner as fully provisioned. The trailing `=` is equally
+# load-bearing: without it `LANGUAGE=` would satisfy `LANG`.
+#
+# A missing, unreadable, or non-regular `.env` returns "unset" rather than
+# aborting, so the caller records all keys unset and keeps walking.
+env_has_key() {
+  env_file=$1
+  env_key=$2
+
+  [ -f "$env_file" ] || return 1
+  [ -r "$env_file" ] || return 1
+  grep -Eq "^[[:space:]]*${env_key}=" "$env_file" 2>/dev/null
+}
+
+key_total=${#MANDATED_KEYS[@]}
+
+# Enumerate runners. A glob matching nothing stays literal and fails the `-d`
+# test, so an empty pool root falls through to the exit-2 branch below.
+runner_names=()
+for candidate in "$POOL_ROOT"/*/; do
+  [ -d "$candidate" ] || continue
+  [ -f "${candidate}config.sh" ] || continue
+  runner_names+=("$(basename "$candidate")")
+done
+runner_count=${#runner_names[@]}
+
+if [ "$runner_count" -eq 0 ]; then
+  printf 'check-runner-env-drift: no runner directories under %s\n' "$POOL_ROOT" >&2
+  printf 'check-runner-env-drift: a runner is a child directory containing config.sh — is this the pool root?\n' >&2
+  exit 2
+fi
+
+printf 'runner .env configuration drift report\n'
+printf '  pool root: %s\n' "$POOL_ROOT"
+printf '  runners:   %d\n' "$runner_count"
+printf '\n'
+
+printf 'per-runner:\n'
+for name in "${runner_names[@]}"; do
+  unset_keys=()
+  for key in "${MANDATED_KEYS[@]}"; do
+    if ! env_has_key "$POOL_ROOT/$name/.env" "$key"; then
+      unset_keys+=("$key")
+    fi
+  done
+
+  if [ "${#unset_keys[@]}" -eq 0 ]; then
+    printf '  %s: all %d mandated keys set\n' "$name" "$key_total"
+  else
+    # `${arr[*]}` joins on the first character of IFS — a space, since this
+    # script never reassigns it. Safe for key names, which carry no whitespace;
+    # runner names are printed one per line below for exactly that reason.
+    printf '  %s: unset %s\n' "$name" "${unset_keys[*]}"
+  fi
+done
+printf '\n'
+
+printf 'per-key:\n'
+drift_count=0
+for key in "${MANDATED_KEYS[@]}"; do
+  set_count=0
+  unset_names=()
+  for name in "${runner_names[@]}"; do
+    if env_has_key "$POOL_ROOT/$name/.env" "$key"; then
+      set_count=$((set_count + 1))
+    else
+      unset_names+=("$name")
+    fi
+  done
+
+  if [ "$set_count" -eq "$runner_count" ]; then
+    printf '  %s: ok — set on %d of %d runners\n' "$key" "$set_count" "$runner_count"
+  elif [ "$set_count" -eq 0 ]; then
+    printf '  %s: uniformly unset — set on 0 of %d runners; a uniform gap, not drift\n' "$key" "$runner_count"
+  else
+    # Reached only when 0 < set_count < runner_count, so unset_names is
+    # non-empty here — one name per line, because a runner directory name may
+    # contain whitespace and a joined list would make it unactionable.
+    drift_count=$((drift_count + 1))
+    printf '  %s: DRIFT — set on %d of %d runners; unset on:\n' "$key" "$set_count" "$runner_count"
+    for name in "${unset_names[@]}"; do
+      printf '      %s\n' "$name"
+    done
+  fi
+done
+printf '\n'
+
+if [ "$drift_count" -eq 0 ]; then
+  printf 'no drift: every mandated key is uniform across the pool.\n'
+  exit 0
+fi
+
+printf 'DRIFT: %d of %d mandated keys are set on some runners but not all.\n' "$drift_count" "$key_total"
+printf 'Provision the runners named above from templates/runner/.env.example,\n'
+printf 'then restart each one so it reloads .env: ./svc.sh stop && ./svc.sh start\n'
+exit 1

package/templates/runner/job-cleanup.sh

@@ -10,8 +10,8 @@
 #
 #   - an orphaned `pnpm`/`node` process (e.g. a hung install or lint) still
 #     mutating the pnpm shim install, corrupting the pnpm CLI for the next job;
-#   - leftover `gitleaks.tmp` / `gitleaks-*` artifacts in the shared $TMPDIR
-#     blocking the next gitleaks download.
+#   - leftover tool-download temp dirs (gitleaks, OSV-scanner, the semgrep
+#     venv) accumulating in the runner's own job temp.
 #
 # Running this before every job gives each job a clean slate ("fresh per job"
 # without the cost of re-registering an ephemeral runner).
@@ -35,9 +35,36 @@
 #      own work tree (`<RUNNER_DIR>/_work/...`). Every path below derives
 #      from RUNNER_DIR, which is unique per runner, so a co-resident
 #      runner's processes and files are never matched.
-#   3. Age-gates cleanup of the genuinely shared $TMPDIR gitleaks artifacts,
-#      so a fresh (in-flight) download owned by a concurrent job is never
-#      deleted — only stale leftovers are.
+#   3. NEVER READS the shared OS temp root, either. Reading is not free: the
+#      hook runs inside the JOB's clock, so any cost here is charged to
+#      `Set up runner` and counts against the job's own `timeout-minutes`.
+#      $TMPDIR is unbounded and shared with every other process on the host,
+#      so a sweep rooted there costs a function of how much UNRELATED junk
+#      the host has accumulated — see the incident note below.
+#
+# ── WHY THE SHARED-$TMPDIR SWEEP IS GONE (issue #343) ───────────────────────
+#
+# This hook used to age-gate two `find "$TMPDIR" -maxdepth 1 -name …` sweeps
+# for `gitleaks.tmp` / `gitleaks-*`. `-maxdepth 1 -name <literal>` is a FULL
+# directory enumeration for what is really an existence check, so its cost
+# scaled with host churn. On the swarm-os runner host $TMPDIR reached 841,690
+# entries; one scan measured 42s, the hook ran two of them, and up to 16
+# co-resident runners ran it concurrently. `Set up runner` reached 5m29s, and
+# every job whose `timeout-minutes` sat at or below that was killed before its
+# first real step — surfacing as `cancelled` on an innocent diff.
+#
+# It was also a no-op: the platform's actions extract via `mktemp -d`, so
+# nothing ever created `gitleaks.tmp` or `gitleaks-*`. The sweep paid an
+# unbounded cost hunting names that never existed, while the dirs the actions
+# DID leave went unswept.
+#
+# The fix is ownership, not tuning: every platform action now extracts into
+# `${RUNNER_TEMP}` (== RUNNER_TMP below), which is unique per runner. A
+# co-resident runner's in-flight download is therefore unreachable from here
+# by construction — which is what retired the age gate outright (along with
+# the stale-minutes env knob that tuned it), rather than merely shrinking its
+# blast radius. Keep it that way: a sweep added here MUST be rooted at
+# RUNNER_TMP.
 #
 # ── PARAMETERIZATION ────────────────────────────────────────────────────────
 #
@@ -48,11 +75,8 @@
 #                  runner root, next to config.sh / run.sh). Override via env
 #                  only if you install the hook elsewhere.
 #   RUNNER_TMP   — the runner's per-runner job temp (`runner.temp`), always
-#                  `${RUNNER_DIR}/_work/_temp`.
-#   JOB_CLEANUP_STALE_MINUTES
-#                — age threshold (minutes) for the shared-$TMPDIR gitleaks
-#                  sweep. Default 60. Artifacts younger than this are assumed
-#                  in-flight and left alone.
+#                  `${RUNNER_DIR}/_work/_temp`. Every path this hook touches
+#                  lives under it.
 #
 # Configured via `ACTIONS_RUNNER_HOOK_JOB_STARTED=<RUNNER_DIR>/job-cleanup.sh`
 # in the runner's `.env` (see .env.example in this directory).
@@ -64,8 +88,6 @@ set +e
 RUNNER_DIR="${RUNNER_DIR:-$(cd "$(dirname "$0")" && pwd)}"
 RUNNER_WORK="${RUNNER_DIR}/_work"
 RUNNER_TMP="${RUNNER_WORK}/_temp"
-TMP="${TMPDIR:-/tmp}"
-STALE_MINUTES="${JOB_CLEANUP_STALE_MINUTES:-60}"
 
 # 1) Reap orphaned pnpm/node processes from prior jobs — scoped to THIS
 #    runner's work tree only. The patterns target executable paths INSIDE the
@@ -85,13 +107,20 @@ pkill -9 -f "${RUNNER_WORK}/_tool/[^ ]*node_modules" 2>/dev/null
 rm -rf "${RUNNER_TMP}/pnpm"       2>/dev/null
 rm -rf "${RUNNER_TMP}/setup-pnpm" 2>/dev/null
 
-# 3) Sweep stale gitleaks artifacts from the SHARED $TMPDIR. Because this
-#    location is shared by every runner on the host, deletion is age-gated:
-#    only artifacts older than STALE_MINUTES are removed, so a concurrent
-#    runner's in-flight download is never deleted mid-job.
-find "${TMP}" -maxdepth 1 -name 'gitleaks.tmp' -mmin "+${STALE_MINUTES}" \
-  -exec rm -f  {} + 2>/dev/null
-find "${TMP}" -maxdepth 1 -name 'gitleaks-*'   -mmin "+${STALE_MINUTES}" \
-  -exec rm -rf {} + 2>/dev/null
+# 3) Remove this runner's own leftover tool-download temp dirs. The platform's
+#    composite actions and workflows create these via
+#    `mktemp -d "${RUNNER_TEMP}/<tool>.XXXXXX"`, so every one of them is
+#    runner-scoped and a co-resident runner's in-flight download is
+#    unreachable here — no age gate is needed (see the issue #343 note above).
+#
+#    Globbing is what keeps this bounded: the shell expands these against
+#    RUNNER_TMP alone, so the cost is a function of THIS runner's leftovers,
+#    never of host-wide churn. Do not replace it with a `find` over a parent.
+#    A glob that matches nothing stays literal, and `rm -rf` on a nonexistent
+#    path is silent — hence the nullglob-free form plus 2>/dev/null.
+rm -rf "${RUNNER_TMP}"/gitleaks.*    2>/dev/null
+rm -rf "${RUNNER_TMP}"/osv-scanner.* 2>/dev/null
+rm -rf "${RUNNER_TMP}"/semgrep.*     2>/dev/null
+rm -f  "${RUNNER_TMP}"/gh-api-err.*  2>/dev/null
 
 exit 0