@rubytech/create-maxy-code 0.1.476 → 0.1.477

package/payload/platform/scripts/cpu-triage.sh

@@ -0,0 +1,313 @@
+#!/usr/bin/env bash
+# cpu-triage.sh — sustained CPU triage for a Maxy Code device.
+#
+# WHY THIS EXISTS, AND WHY IT SAMPLES FOR MINUTES RATHER THAN SECONDS
+#
+# The booking reconcile loop runs on a 120 s cycle and a tick costs roughly
+# 800 ms of network wait plus a garbage-collection burst. Measured on laptop
+# 192.168.88.10 (2026-07-20), the SAME server process over the SAME method read
+# 0.2%, 0.2%, 0.2% in three consecutive 25 s windows and 10.8% in a 20 s window.
+# Every one of those readings was correct. Any sample shorter than the period
+# either lands on a tick or misses it, so spot checks on this platform produce
+# contradictory answers indefinitely. This tool therefore samples across at
+# least one full cycle and reports a DISTRIBUTION (mean, peak, and what share of
+# the window sat above threshold), never a single number.
+#
+# THREE INSTRUMENTS THIS DELIBERATELY DOES NOT USE, EACH HAVING PRODUCED A FALSE
+# CONCLUSION IN THE 2026-07-20 INVESTIGATION:
+#   * `ps %cpu` / `top` %CPU  — LIFETIME averages. A 40-minute-old process read
+#     8.5% against an 8-day-old peer at 0.1%; both were actually 0.2% live. This
+#     nearly shipped as an "85x regression".
+#   * `iostat -c` first line  — a SINCE-BOOT average. Read as live it reported a
+#     machine "42% busy" that was 87% idle at that moment.
+#   * `ns_last_pid` deltas    — counts THREAD creation as well as forks, and was
+#     compared against a single 10 s sample used as a "baseline". Produced a
+#     false 5.6x churn regression. Fork rate here comes from /proc/stat's
+#     `processes` counter, which counts forks only.
+# Everything below is a DELTA over a stated window. No lifetime averages.
+#
+# Aggregate CPU is also avoided as a headline: two hot cores out of twenty
+# average down to ~3% and vanish, which is exactly how a real, operator-visible
+# condition was dismissed as "nothing changed". Per-core is the primary view.
+#
+# Unprivileged by design: the admin assistant runs without sudo, so this uses
+# only /proc and the user's own cgroup tree. `perf` and `bpftrace` need root and
+# are named in the escalation hint rather than invoked.
+#
+# EXIT CODES (gates, for deterministic callers)
+#   0  no core sustained above threshold — nothing to escalate
+#   1  one or more cores sustained above threshold — findings in the report
+#   2  usage error or unmet precondition (nothing measured)
+
+set -uo pipefail
+
+WINDOW=300          # total sampling window, seconds (>= 2 reconcile cycles)
+INTERVAL=10         # sample cadence, seconds
+THRESHOLD=15        # per-core busy % that counts as "hot"
+SUSTAIN=50          # % of samples above THRESHOLD to call a core "sustained"
+TOPN=12             # processes to report
+JSON=0
+CONTROL=""          # optional peer brand known NOT to have the change
+
+usage() {
+  cat >&2 <<USAGE
+usage: cpu-triage.sh [--window S] [--interval S] [--threshold PCT]
+                     [--sustain PCT] [--top N] [--control BRAND] [--json]
+
+  --window     total sampling window in seconds (default 300, min 120)
+  --interval   sample cadence in seconds (default 10)
+  --threshold  per-core busy %% treated as hot (default 15)
+  --sustain    %% of samples above threshold to call it sustained (default 50)
+  --top        processes to report (default 12)
+  --control    a brand on the PREVIOUS build, for like-for-like comparison.
+               The single most valuable input: on 2026-07-20 an un-upgraded
+               peer collapsed a suspected regression into a measurement artifact.
+  --json       emit machine-readable JSON instead of the text report
+USAGE
+  exit 2
+}
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --window)    WINDOW="${2:-}"; shift 2 ;;
+    --interval)  INTERVAL="${2:-}"; shift 2 ;;
+    --threshold) THRESHOLD="${2:-}"; shift 2 ;;
+    --sustain)   SUSTAIN="${2:-}"; shift 2 ;;
+    --top)       TOPN="${2:-}"; shift 2 ;;
+    --control)   CONTROL="${2:-}"; shift 2 ;;
+    --json)      JSON=1; shift ;;
+    -h|--help)   usage ;;
+    *) echo "cpu-triage: unknown argument '$1'" >&2; usage ;;
+  esac
+done
+
+case "$WINDOW$INTERVAL$THRESHOLD$SUSTAIN$TOPN" in *[!0-9]*) usage ;; esac
+[ "$INTERVAL" -ge 1 ] || usage
+if [ "$WINDOW" -lt 120 ]; then
+  echo "cpu-triage: refusing a window under 120s — shorter than the reconcile cycle," >&2
+  echo "            which is what makes spot readings contradict each other." >&2
+  exit 2
+fi
+[ -r /proc/stat ] || { echo "cpu-triage: /proc/stat unreadable — Linux only" >&2; exit 2; }
+
+SAMPLES=$(( WINDOW / INTERVAL ))
+[ "$SAMPLES" -ge 2 ] || { echo "cpu-triage: window/interval must yield >= 2 samples" >&2; exit 2; }
+
+CLK=$(getconf CLK_TCK 2>/dev/null || echo 100)
+TMP=$(mktemp -d "${TMPDIR:-/tmp}/cpu-triage.XXXXXX") || exit 2
+trap 'rm -rf "$TMP"' EXIT INT TERM
+
+CG_BASE="/sys/fs/cgroup/user.slice/user-$(id -u).slice/user@$(id -u).service/app.slice"
+
+# ---------------------------------------------------------------------------
+# Sampling. One loop, one cadence, every measure derived from the same instants
+# so per-core, per-process and per-service figures are directly comparable.
+# ---------------------------------------------------------------------------
+snapshot() {
+  local idx="$1"
+  # NOTE: every redirect below MUST be `>>`. Each sample is a fresh awk process,
+  # and awk's `>` truncates the file once per invocation — so `>` silently keeps
+  # only the LAST sample, leaving nothing to diff. That bug shipped into the
+  # first test run and produced an empty per-core report with mean 0%.
+  awk -v idx="$idx" -v out="$TMP" '
+    FILENAME == "/proc/stat" {
+      if ($1 ~ /^cpu[0-9]+$/) {
+        total = 0
+        for (i = 2; i <= NF; i++) total += $i
+        # busy excludes idle (field 5) and iowait (field 6)
+        print idx, $1, total - $5 - $6, total >> (out "/cores.raw")
+      }
+      if ($1 == "processes") print idx, $2 >> (out "/forks.raw")
+      next
+    }
+    # /proc/<pid>/stat — comm may contain spaces and parens; the greedy .*\)
+    # anchors on the LAST close paren, which is the only safe split.
+    {
+      if (match($0, /^[0-9]+ \(.*\) /)) {
+        pid  = $1
+        rest = substr($0, RSTART + RLENGTH)
+        n    = split(rest, f, " ")
+        if (n >= 13) print idx, pid, f[12] + f[13] >> (out "/procs.raw")
+      }
+    }
+  ' /proc/stat /proc/[0-9]*/stat 2>/dev/null
+
+  # Per-service cgroup CPU. Counts CHILD processes, which per-process sampling
+  # attributes nowhere — the original wrangler churn lived entirely in children.
+  #
+  # Read every cpu.stat in ONE awk rather than a shell loop forking per service.
+  # This tool reports fork rate, so its own churn is measurement noise it would
+  # otherwise attribute to the box: the loop form cost ~16 forks per sample.
+  if [ -d "$CG_BASE" ]; then
+    awk -v idx="$idx" '
+      /^usage_usec/ {
+        n = split(FILENAME, p, "/")
+        svc = p[n-1]; sub(/\.service$/, "", svc)
+        print idx, svc, $2
+      }
+    ' "$CG_BASE"/*.service/cpu.stat >> "$TMP/cgroup.raw" 2>/dev/null
+  fi
+}
+
+HOST=$(hostname 2>/dev/null || echo unknown)
+NCPU=$(awk '/^cpu[0-9]+/{n++} END{print n+0}' /proc/stat)
+STARTED=$(date -Is 2>/dev/null || date)
+
+if [ "$JSON" -eq 0 ]; then
+  echo "cpu-triage on $HOST — sampling ${WINDOW}s at ${INTERVAL}s cadence (${SAMPLES} samples, ${NCPU} cores)"
+  echo "this spans at least one 120s reconcile cycle; spot readings on this platform are not reliable"
+  echo
+fi
+
+i=0
+while [ "$i" -lt "$SAMPLES" ]; do
+  snapshot "$i"
+  i=$(( i + 1 ))
+  [ "$i" -lt "$SAMPLES" ] && sleep "$INTERVAL"
+done
+ENDED=$(date -Is 2>/dev/null || date)
+
+[ -s "$TMP/cores.raw" ] || { echo "cpu-triage: no samples captured" >&2; exit 2; }
+
+# ---------------------------------------------------------------------------
+# Analysis — distribution per core, not a single number.
+# ---------------------------------------------------------------------------
+awk -v thr="$THRESHOLD" '
+  { key = $2
+    if (prev_total[key] != "") {
+      dt = $4 - prev_total[key]
+      db = $3 - prev_busy[key]
+      if (dt > 0) {
+        pct = db / dt * 100
+        sum[key] += pct; n[key]++
+        if (pct > peak[key]) peak[key] = pct
+        if (pct > thr) hot[key]++
+      }
+    }
+    prev_total[key] = $4; prev_busy[key] = $3
+  }
+  END {
+    for (k in sum)
+      printf "%s %.2f %.2f %.1f %d\n", k, sum[k]/n[k], peak[k], (hot[k]+0)/n[k]*100, n[k]
+  }
+' "$TMP/cores.raw" | sort -k2 -rn > "$TMP/cores.out"
+
+awk -v clk="$CLK" -v iv="$INTERVAL" '
+  { key = $2
+    if (prev[key] != "") {
+      d = $3 - prev[key]
+      if (d >= 0) {
+        pct = d / (clk * iv) * 100
+        sum[key] += pct; n[key]++
+        if (pct > peak[key]) peak[key] = pct
+      }
+    }
+    prev[key] = $3
+  }
+  END { for (k in sum) if (sum[k]/n[k] >= 0.05) printf "%s %.2f %.2f\n", k, sum[k]/n[k], peak[k] }
+' "$TMP/procs.raw" | sort -k2 -rn > "$TMP/procs.out"
+
+if [ -s "${TMP}/cgroup.raw" ]; then
+  awk -v iv="$INTERVAL" '
+    { key = $2
+      if (prev[key] != "") {
+        d = $3 - prev[key]
+        if (d >= 0) { pct = d / (iv * 1000000) * 100; sum[key] += pct; n[key]++
+                      if (pct > peak[key]) peak[key] = pct }
+      }
+      prev[key] = $3
+    }
+    END { for (k in sum) printf "%s %.2f %.2f\n", k, sum[k]/n[k], peak[k] }
+  ' "$TMP/cgroup.raw" | sort -k2 -rn > "$TMP/cgroup.out"
+fi
+
+FORKS_MIN=$(awk -v iv="$INTERVAL" 'NR==1{f=$2;i=$1} END{ if (NR>1) printf "%.0f", ($2-f)/((($1-i)*iv))*60 }' "$TMP/forks.raw" 2>/dev/null)
+[ -n "$FORKS_MIN" ] || FORKS_MIN=0
+
+HOT_COUNT=$(awk -v s="$SUSTAIN" '$4 >= s {n++} END{print n+0}' "$TMP/cores.out")
+MEAN_CORE=$(awk '{s+=$2; n++} END{ if(n) printf "%.2f", s/n; else print "0" }' "$TMP/cores.out")
+
+resolve() { tr '\0' ' ' < "/proc/$1/cmdline" 2>/dev/null | cut -c1-78 || true; }
+
+# ---------------------------------------------------------------------------
+# Report
+# ---------------------------------------------------------------------------
+if [ "$JSON" -eq 1 ]; then
+  printf '{\n  "host": %s,\n  "started": %s,\n  "ended": %s,\n' "\"$HOST\"" "\"$STARTED\"" "\"$ENDED\""
+  printf '  "windowSec": %s, "intervalSec": %s, "samples": %s, "cores": %s,\n' "$WINDOW" "$INTERVAL" "$SAMPLES" "$NCPU"
+  printf '  "thresholdPct": %s, "sustainPct": %s,\n' "$THRESHOLD" "$SUSTAIN"
+  printf '  "meanPerCorePct": %s, "sustainedHotCores": %s, "forksPerMin": %s,\n' "$MEAN_CORE" "$HOT_COUNT" "$FORKS_MIN"
+  printf '  "perCore": ['
+  awk '{ printf "%s{\"cpu\":\"%s\",\"meanPct\":%s,\"peakPct\":%s,\"aboveThresholdPct\":%s}", (NR>1?",":""), $1,$2,$3,$4 }' "$TMP/cores.out"
+  printf '],\n  "topProcesses": ['
+  head -n "$TOPN" "$TMP/procs.out" | while read -r pid mean peak; do
+    cl=$(resolve "$pid" | sed 's/\\/\\\\/g; s/"/\\"/g')
+    printf '{"pid":%s,"meanPct":%s,"peakPct":%s,"cmd":"%s"},' "$pid" "$mean" "$peak" "$cl"
+  done | sed 's/,$//'
+  printf '],\n  "perService": ['
+  if [ -s "${TMP}/cgroup.out" ]; then
+    awk '{ printf "%s{\"service\":\"%s\",\"meanPct\":%s,\"peakPct\":%s}", (NR>1?",":""), $1,$2,$3 }' "$TMP/cgroup.out"
+  fi
+  printf ']\n}\n'
+else
+  echo "PER-CORE over the window (mean / peak / share of window above ${THRESHOLD}%)"
+  echo "  a core is 'sustained' when it exceeds threshold in >= ${SUSTAIN}% of samples"
+  awk -v thr="$THRESHOLD" -v s="$SUSTAIN" '
+    $2 >= 1 || $4 > 0 {
+      flag = ($4 >= s) ? "  <== SUSTAINED" : ""
+      printf "  %-7s mean=%6.2f%%  peak=%6.2f%%  above=%5.1f%% of window%s\n", $1, $2, $3, $4, flag
+    }' "$TMP/cores.out"
+  echo "  ---"
+  printf "  mean per-core %s%%   sustained hot cores: %s of %s   forks %s/min\n\n" "$MEAN_CORE" "$HOT_COUNT" "$NCPU" "$FORKS_MIN"
+
+  echo "TOP PROCESSES by measured delta (100% = one full core; NOT a lifetime average)"
+  head -n "$TOPN" "$TMP/procs.out" | while read -r pid mean peak; do
+    printf "  %6.2f%% mean  %6.2f%% peak  pid=%-8s %s\n" "$mean" "$peak" "$pid" "$(resolve "$pid")"
+  done
+  echo
+
+  if [ -s "${TMP}/cgroup.out" ]; then
+    echo "PER-SERVICE cgroup CPU (child processes included — the platform's true footprint)"
+    tot=$(awk '{s+=$2} END{printf "%.2f", s}' "$TMP/cgroup.out")
+    awk '$2 >= 0.01 { printf "  %-32s mean=%6.2f%%  peak=%6.2f%%\n", $1, $2, $3 }' "$TMP/cgroup.out"
+    printf "  ---\n  all services %s%% of one core (%.2f%% of the %s-core box)\n\n" \
+      "$tot" "$(awk -v t="$tot" -v c="$NCPU" 'BEGIN{print t/c}')" "$NCPU"
+    if [ -n "$CONTROL" ]; then
+      cval=$(awk -v c="$CONTROL" '$1 == c {print $2}' "$TMP/cgroup.out")
+      if [ -n "$cval" ]; then
+        echo "CONTROL: '$CONTROL' is on the previous build and measured ${cval}% of one core."
+        echo "  Compare peers against this, not against their own past readings. A peer that"
+        echo "  did not get the change is worth more than any before/after inference."
+        echo
+      else
+        echo "CONTROL: '$CONTROL' not found among running services — comparison skipped."; echo
+      fi
+    fi
+  fi
+
+  if command -v sar >/dev/null 2>&1; then
+    y=$(date -d yesterday +%d 2>/dev/null)
+    if [ -n "$y" ] && [ -r "/var/log/sysstat/sa$y" ]; then
+      echo "HISTORY — same instrument, yesterday, for like-for-like regression checking"
+      sar -P ALL -f "/var/log/sysstat/sa$y" 2>/dev/null | awk -v thr="$THRESHOLD" '
+        /Average/ && $2 ~ /^[0-9]+$/ { b = 100 - $8; s += b; n++; if (b > thr) h++ }
+        END { if (n) printf "  yesterday: mean per-core %.2f%%   cores above %s%%: %d/%d\n", s/n, thr, h+0, n }'
+      echo
+    fi
+  fi
+
+  if [ "$HOT_COUNT" -gt 0 ]; then
+    echo "VERDICT: ${HOT_COUNT} core(s) sustained above ${THRESHOLD}%. Findings above."
+    echo "  Before calling it a regression, check the desktop stack in the process list."
+    echo "  On 2026-07-20 gnome-shell + gnome-system-monitor + Xorg accounted for ~41% of"
+    echo "  a core, roughly three times the entire four-brand platform, and the System"
+    echo "  Monitor window doing the observing was itself a third of that."
+    echo "  To attribute short-lived process churn, escalate (needs root):"
+    echo "    bpftrace -e 'tracepoint:syscalls:sys_enter_execve { @[comm, str(args->filename)] = count(); }'"
+  else
+    echo "VERDICT: no core sustained above ${THRESHOLD}%."
+  fi
+fi
+
+[ "$HOT_COUNT" -gt 0 ] && exit 1
+exit 0