create-walle 0.9.30 → 0.9.32

package/template/wall-e/training/training-coexistence.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+# training-coexistence.sh — reclaim RAM headroom for a heavy training run by
+# stopping *memory-heavy, non-essential* apps, then restoring them afterward.
+#
+# Design constraints (learned the hard way):
+#   - NEVER stop the browser: the CTM/Wall-E UI runs in it (localhost:3456).
+#   - NEVER stop CTM, Wall-E, or the EDR agent.
+#   - Only stop apps that actually FREE RAM on exit (SIGSTOP does not reclaim
+#     memory — it only freezes — so we quit/stop, not pause).
+#
+# Config (user-customizable): ~/.walle/training-coexistence.json
+#   { "stopApps": ["OrbStack"], "minFreeGb": 8 }
+# Anything in the hard denylist below is refused even if listed.
+#
+# Usage: training-coexistence.sh pause   (stops configured apps, prints what it did)
+#        training-coexistence.sh restore (restarts what pause stopped)
+set -u
+CFG="${WALLE_COEXIST_CONFIG:-$HOME/.walle/training-coexistence.json}"
+STATE="$HOME/.walle/.training-coexistence-stopped"
+# ERE (macOS grep has no -P); matched case-insensitively via grep -i.
+DENY_RE='chrome|chromium|safari|arc|brave|edge|firefox|claude task|ctm|wall-e|walle|node|cortex|traps|paloalto|windowserver|finder|terminal|iterm'
+
+read_list(){ # stopApps from config, or default OrbStack
+  if [ -f "$CFG" ] && command -v node >/dev/null 2>&1; then
+    node -e 'try{const c=require(process.argv[1]);process.stdout.write((c.stopApps||["OrbStack"]).join("\n"))}catch(e){process.stdout.write("OrbStack")}' "$CFG"
+  else echo "OrbStack"; fi
+}
+
+stop_app(){
+  local app="$1"
+  if echo "$app" | grep -qiE 'orbstack|orb'; then
+    command -v orb >/dev/null 2>&1 && { orb stop >/dev/null 2>&1 && echo "OrbStack"; return; }
+  fi
+  # generic: graceful quit via AppleScript (frees RAM on exit)
+  osascript -e "quit app \"$app\"" >/dev/null 2>&1 && echo "$app"
+}
+
+stop_ollama_server(){ # stop the SERVER (not just unload) so Wall-E can't reload it mid-run.
+  command -v ollama >/dev/null 2>&1 || return
+  if brew services list 2>/dev/null | grep -qiE '^ollama +started'; then
+    brew services stop ollama >/dev/null 2>&1 && { echo "OLLAMA:brew" >> "$STATE"; echo "stopped ollama (brew service)"; return; }
+  fi
+  if pgrep -fl 'Ollama.app' >/dev/null 2>&1; then
+    osascript -e 'quit app "Ollama"' >/dev/null 2>&1 && { echo "OLLAMA:app" >> "$STATE"; echo "stopped Ollama.app"; return; }
+  fi
+  if pgrep -f 'ollama serve' >/dev/null 2>&1; then
+    pkill -f 'ollama serve' >/dev/null 2>&1 && { echo "OLLAMA:bare" >> "$STATE"; echo "stopped ollama serve (bare)"; return; }
+  fi
+}
+
+restore_ollama(){
+  case "$1" in
+    OLLAMA:brew) brew services start ollama >/dev/null 2>&1 && echo "restarted ollama (brew service)";;
+    OLLAMA:app)  open -a Ollama >/dev/null 2>&1 && echo "restarted Ollama.app";;
+    OLLAMA:bare) nohup ollama serve >/dev/null 2>&1 & echo "restarted ollama serve (bare)";;
+  esac
+}
+
+want_ollama(){ # default ON; config { "unloadOllama": false } to disable
+  if [ -f "$CFG" ] && command -v node >/dev/null 2>&1; then
+    node -e 'try{const c=require(process.argv[1]);process.stdout.write(String(c.unloadOllama!==false))}catch(e){process.stdout.write("true")}' "$CFG"
+  else echo "true"; fi
+}
+
+case "${1:-}" in
+  pause)
+    : > "$STATE"
+    [ "$(want_ollama)" = "true" ] && stop_ollama_server
+    while IFS= read -r app; do
+      [ -z "$app" ] && continue
+      if echo "$app" | grep -qiE "$DENY_RE"; then echo "REFUSED (denylist): $app" >&2; continue; fi
+      done_app=$(stop_app "$app")
+      [ -n "$done_app" ] && { echo "$done_app" >> "$STATE"; echo "stopped: $done_app"; }
+    done < <(read_list)
+    ;;
+  restore)
+    [ -f "$STATE" ] || { echo "nothing to restore"; exit 0; }
+    while IFS= read -r app; do
+      [ -z "$app" ] && continue
+      case "$app" in OLLAMA:*) restore_ollama "$app"; continue;; esac
+      if echo "$app" | grep -qiE 'orbstack|orb'; then command -v orb >/dev/null 2>&1 && orb start >/dev/null 2>&1 && echo "restarted: OrbStack"
+      else open -a "$app" >/dev/null 2>&1 && echo "restarted: $app"; fi
+    done < "$STATE"
+    rm -f "$STATE"
+    ;;
+  *) echo "usage: $0 pause|restore" >&2; exit 2;;
+esac

package/template/wall-e/training/training-mem-guard.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# training-mem-guard.sh — host-OOM watchdog for MLX training runs.
+#
+# MLX's set_memory_limit is a SOFT guideline (it only raises once RAM+swap is
+# exhausted), so a runaway training process can still drive the whole machine
+# into a jetsam low-memory kill / restart — which on this box would take down
+# the live CTM/Wall-E sessions. This watchdog is the hard guarantee: it polls
+# the trainer's RSS and system available memory and SIGTERMs the trainer BEFORE
+# the host is endangered. Pair it with frequent --save-every so a kill costs
+# only minutes.
+#
+# Usage:
+#   training-mem-guard.sh --match <pgrep-pattern> --cap-gb N --min-free-gb M \
+#       [--interval S] [--log PATH]
+#
+# Kills the matched process (and its tree) if EITHER:
+#   - its RSS exceeds --cap-gb, or
+#   - system available memory drops below --min-free-gb.
+set -u
+MATCH="mlx_lora_launch"; CAP_GB=40; MIN_FREE_GB=8; INTERVAL=5; LOG=/dev/stderr
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --match) MATCH="$2"; shift 2;;
+    --cap-gb) CAP_GB="$2"; shift 2;;
+    --min-free-gb) MIN_FREE_GB="$2"; shift 2;;
+    --interval) INTERVAL="$2"; shift 2;;
+    --log) LOG="$2"; shift 2;;
+    *) echo "unknown arg $1" >&2; exit 2;;
+  esac
+done
+PAGE=$(sysctl -n hw.pagesize 2>/dev/null || echo 16384)
+log(){ echo "[mem-guard $(date '+%H:%M:%S')] $*" >> "$LOG"; }
+
+avail_gb(){
+  # available ≈ (free + inactive + speculative + purgeable) pages * pagesize
+  vm_stat 2>/dev/null | awk -v pg="$PAGE" '
+    /Pages free/        {f=$3}
+    /Pages inactive/    {i=$3}
+    /Pages speculative/ {s=$3}
+    /Pages purgeable/   {p=$3}
+    END { gsub(/\./,"",f); gsub(/\./,"",i); gsub(/\./,"",s); gsub(/\./,"",p);
+          printf "%.2f", (f+i+s+p)*pg/1073741824 }'
+}
+
+log "armed: match=$MATCH cap=${CAP_GB}GB min_free=${MIN_FREE_GB}GB interval=${INTERVAL}s"
+# Startup grace: the trainer (node -> python child) may not exist yet. Wait up
+# to ~180s for it to first appear before the "gone -> exit" logic applies, so we
+# don't exit immediately on a launch race.
+appeared=0
+for _ in $(seq 1 45); do
+  if [ -n "$(pgrep -f "$MATCH" | head -1)" ]; then appeared=1; break; fi
+  sleep 4
+done
+[ "$appeared" = 1 ] && log "trainer detected — monitoring" || log "trainer never appeared in grace window — exiting"
+[ "$appeared" = 1 ] || exit 0
+while true; do
+  PID=$(pgrep -f "$MATCH" | head -1)
+  if [ -z "$PID" ]; then log "trainer gone — exiting clean"; exit 0; fi
+  RSS_KB=$(ps -o rss= -p "$PID" 2>/dev/null | tr -d ' ')
+  [ -z "$RSS_KB" ] && { log "pid $PID vanished"; exit 0; }
+  RSS_GB=$(awk -v k="$RSS_KB" 'BEGIN{printf "%.2f", k/1048576}')
+  AVAIL=$(avail_gb)
+  over_rss=$(awk -v r="$RSS_GB" -v c="$CAP_GB" 'BEGIN{print (r>c)?1:0}')
+  low_free=$(awk -v a="$AVAIL" -v m="$MIN_FREE_GB" 'BEGIN{print (a<m)?1:0}')
+  if [ "$over_rss" = 1 ] || [ "$low_free" = 1 ]; then
+    why="rss=${RSS_GB}GB(cap=${CAP_GB}) avail=${AVAIL}GB(min=${MIN_FREE_GB})"
+    log "BREACH ($why) — SIGTERM trainer tree pid=$PID to protect host"
+    # kill the node parent tree too so the driver stops cleanly
+    PPID_=$(ps -o ppid= -p "$PID" 2>/dev/null | tr -d ' ')
+    kill -TERM "$PID" 2>/dev/null
+    [ -n "$PPID_" ] && kill -TERM "$PPID_" 2>/dev/null
+    sleep 3; kill -KILL "$PID" 2>/dev/null
+    exit 1
+  fi
+  sleep "$INTERVAL"
+done