kampodine 0.3.0 → 0.6.0

package/scripts/metrics.sh

@@ -0,0 +1,127 @@
+#!/usr/bin/env bash
+# metrics.sh — ONE-SHOT VM snapshot over SSH (no continuous monitoring —
+# Beszel owns that; this is the CLI triage view). One SSH round-trip per
+# snapshot: load + CPUs, memory, disk WITH an images/data/other breakdown,
+# per-container CPU/MEM (podman stats), uptime, top-5 processes by RSS.
+#
+# The remote command is busybox-safe (cat/echo/nproc/df/du/podman/ps/sort/
+# head — see METRICS_REMOTE_CMD in common.sh); all parsing/rendering happens
+# client-side (metrics_* helpers in common.sh — the SAME snapshot folds into
+# `status --verbose`). Disk threshold reuses the deploy gate's verdict logic
+# (disk_verdict): --warn-disk <pct> (default 90) exits 1 when usage >= pct.
+#
+# Usage:
+#   kampodine metrics [--profile <name>] [--host <user@ip>] [--ssh-key <path>]
+#                     [--warn-disk <pct>] [--watch <sec>] [--count <n>]
+set -euo pipefail
+
+HERE="$(cd "$(dirname "$0")" && pwd)"
+# shellcheck source=common.sh
+source "$HERE/common.sh"
+
+KAMPODINE_HOST="${KAMPODINE_HOST:-}"
+if [ -n "${KAMPODINE_SSH_KEY:-}" ]; then
+    SSH_KEY="$KAMPODINE_SSH_KEY"
+else
+    SSH_KEY=""
+fi
+PROFILE_FLAG=""
+HOST_FLAG=""
+KEY_FLAG=""
+WARN_DISK="90"
+WATCH=""
+COUNT=""
+
+say() { printf '\033[1;36m[metrics]\033[0m %s\n' "$*"; }
+die() { printf '\033[1;31m[metrics] FAIL:\033[0m %s\n' "$*" >&2; exit 1; }
+usage() {
+  local code="${1:-0}"
+  printf 'Usage:\n'
+  grep '^#   kampodine metrics' "$0" | sed 's/^#   //'
+  cat <<'EOF'
+
+One SSH round-trip per snapshot; busybox-safe remote commands; rendering is
+client-side. NO continuous monitoring (Beszel owns that) — one-shot CLI view;
+--watch N re-snapshots every N seconds (Ctrl-C ends), --count M bounds the
+iterations. --warn-disk <pct> (default 90) exits 1 when root disk usage is at
+or above the threshold — the SAME verdict logic as deploy's fail-closed gate.
+Host resolution: --host | --profile <name> | KAMPODINE_PROFILE |
+config defaultProfile | KAMPODINE_HOST.
+
+Examples:
+  kampodine metrics                              # profile/host from config or env
+  kampodine metrics --host root@203.0.113.10     # explicit target
+  kampodine metrics --profile prod               # per-instance profile
+  kampodine metrics --warn-disk 85               # exit 1 at >= 85% (default 90)
+  kampodine metrics --watch 10 --count 6         # snapshot every 10s, 6 times
+  kampodine status --verbose                     # the same snapshot inside status
+EOF
+  exit "$code"
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --profile) PROFILE_FLAG="$2"; shift 2 ;;
+    --host) HOST_FLAG="$2"; KAMPODINE_HOST="$2"; shift 2 ;;
+    --ssh-key) KEY_FLAG="$2"; SSH_KEY="$2"; shift 2 ;;
+    --warn-disk) WARN_DISK="$2"; shift 2 ;;
+    --watch) WATCH="$2"; shift 2 ;;
+    --count) COUNT="$2"; shift 2 ;;
+    -h|--help) usage 0 ;;
+    *) die "unknown argument: $1 (--help)" ;;
+  esac
+done
+
+# profile resolution (flag > KAMPODINE_PROFILE > defaultProfile); an explicit
+# --host/--ssh-key always beats the profile
+profile_resolve "$PROFILE_FLAG" "$HOST_FLAG" "$KEY_FLAG"
+SSH_KEY="${KEY_FLAG:-${KAMPODINE_SSH_KEY:-}}"
+
+[[ "$WARN_DISK" =~ ^[0-9]+$ && "$WARN_DISK" -ge 1 && "$WARN_DISK" -le 100 ]] \
+  || die "--warn-disk must be a percentage 1-100 (got: $WARN_DISK)"
+[[ -z "$WATCH" || "$WATCH" =~ ^[1-9][0-9]*$ ]] || die "--watch must be a positive-integer number of seconds (got: $WATCH)"
+[[ -z "$COUNT" || "$COUNT" =~ ^[1-9][0-9]*$ ]] || die "--count must be a positive integer (got: $COUNT)"
+[[ -n "$KAMPODINE_HOST" ]] || die "target required: --host root@<ip>, --profile <name>, KAMPODINE_PROFILE, config defaultProfile, or KAMPODINE_HOST=root@<ip>"
+
+SSH_ARGS=(-o ConnectTimeout=10 -o BatchMode=yes)
+[[ -n "$SSH_KEY" ]] && SSH_ARGS+=(-i "$SSH_KEY")
+# $1 is a composed remote command — client-side expansion is the design.
+# shellcheck disable=SC2029
+vm() { ssh "${SSH_ARGS[@]}" "$KAMPODINE_HOST" "$1"; }
+
+# --- macOS ssh-agent quirk (first run from a fresh machine) -------------------
+if [[ "$(uname -s)" == "Darwin" ]]; then
+  SSH_AUTH_SOCK="$(launchctl getenv SSH_AUTH_SOCK 2>/dev/null || true)"
+  export SSH_AUTH_SOCK
+fi
+
+snapshot() {
+  local raw pct verdict
+  raw="$(vm "$METRICS_REMOTE_CMD")" || die "cannot reach VM $KAMPODINE_HOST (ssh failed) — nothing to report"
+  say "== metrics ($KAMPODINE_HOST) =="
+  metrics_render "$raw"
+  pct="$(disk_pct_from_df "$M_DISK")"
+  verdict="$(disk_verdict "$pct" "$WARN_DISK")"
+  case "$verdict" in
+    fail)
+      printf 'WARNING: root disk at %s%% (>= --warn-disk %s%%) — free space first: kampodine deploy prune --dry-run\n' \
+        "${pct:-?}" "$WARN_DISK"
+      return 1
+      ;;
+    warn)
+      printf 'WARNING: root disk above 90%% (%s%%) — old sha-tagged deploy images pile up; reclaim: kampodine deploy prune --dry-run\n' "$pct"
+      ;;
+  esac
+  return 0
+}
+
+n=0
+while :; do
+  n=$(( n + 1 ))
+  if ! snapshot; then
+    exit 1
+  fi
+  [[ -n "$COUNT" && "$n" -ge "$COUNT" ]] && exit 0
+  [[ -z "$WATCH" ]] && exit 0
+  sleep "$WATCH"
+done

package/scripts/migrate.sh

@@ -9,13 +9,16 @@
 # are COLLECTED (files are independent; one bad tenant must not block the others)
 # and the script exits 1 if any failed — the deploy's smoke gate catches a red migration.
 # Runs ON the VM from the repo checkout (also fine locally against .tmp data):
-#   kampodine migrate [--allow-running]
+#   kampodine migrate [--allow-running] [--profile <name>]
 set -euo pipefail
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 REPO_ROOT="${REPO_ROOT:-$(cd "${SCRIPT_DIR}/../../.." && pwd)}"
 TENANT_DIR="${TENANT_DIR:-/data/tenants}"
-SERVICE="${SERVICE:-esellar-api}"
+SERVICE="${SERVICE:-kampodine-api}"
+HERE="$SCRIPT_DIR"
+# shellcheck source=common.sh
+source "$HERE/common.sh"
 
 log() { printf '[migrate-all] %s\n' "$*"; }
 fail() { printf '[migrate-all][FAIL] %s\n' "$*" >&2; exit 1; }
@@ -25,7 +28,7 @@ Usage:
   kampodine migrate [--allow-running]
 
 Examples:
-  kampodine migrate                    # stop the API first (rc-service esellar-api stop)
+  kampodine migrate                    # stop the API first (rc-service kampodine-api stop)
   kampodine migrate --allow-running    # deliberate: migrate while the API serves (SQLITE_BUSY risk)
 
 Runs drizzle migrations over the libSQL dbs under TENANT_DIR (default
@@ -37,13 +40,25 @@ EOF
 }
 
 ALLOW_RUNNING=0
+PROFILE_FLAG=""
+PENDING_PROFILE=0
 for arg in "$@"; do
+  if [[ "$PENDING_PROFILE" -eq 1 ]]; then
+    PROFILE_FLAG="$arg"
+    PENDING_PROFILE=0
+    continue
+  fi
   case "$arg" in
     --allow-running) ALLOW_RUNNING=1 ;;
-    -h|--help) usage ;;
+    --profile) PENDING_PROFILE=1 ;;
+    --help|-h) usage ;;
     *) fail "unknown argument: $arg (--help)" ;;
   esac
 done
+[[ "$PENDING_PROFILE" -eq 0 ]] || fail "--profile requires a value"
+# profile selection is validated for uniformity — migrate consumes no
+# host/sshKey fields (it runs ON the VM from the repo checkout)
+profile_resolve "$PROFILE_FLAG" "" ""
 
 [[ -f "${REPO_ROOT}/apps/api/scripts/libsql-migrate/migrate-db.ts" ]] \
   || fail "repo root not found at ${REPO_ROOT} (set REPO_ROOT=... if the checkout lives elsewhere)"
@@ -51,11 +66,30 @@ command -v pnpm >/dev/null 2>&1 || fail "pnpm not found in PATH"
 [[ -d "$TENANT_DIR" ]] || fail "tenant dir ${TENANT_DIR} does not exist"
 
 # Writing schema while the API serves traffic risks SQLITE_BUSY on a single-writer
-# engine — the service must be stopped first (rc-service esellar-api stop). Override exists
+# engine — the service must be stopped first (rc-service ${SERVICE} stop). Override exists
 # for deliberate local use.
-if systemctl is-active --quiet "$SERVICE" 2>/dev/null; then
+#
+# Init-aware detection (fixes the DEAD guard, 2026-10-08): this script runs ON
+# the VM, whose init is OpenRC (Alpine ships no systemd) — a bare `systemctl
+# is-active` always failed there, so the stop-first guard NEVER fired and
+# migrations could run under the serving API. Mirror common.sh's detection
+# order: rc-service (openrc) first, then systemctl (systemd hosts). On a host
+# with neither (local dev machines), state is unknowable — proceed with a
+# loud warning rather than blocking local runs.
+service_running() {
+  if command -v rc-service >/dev/null 2>&1; then
+    rc-service "$SERVICE" status >/dev/null 2>&1
+  elif command -v systemctl >/dev/null 2>&1; then
+    systemctl is-active --quiet "$SERVICE"
+  else
+    log "service state unknown (neither rc-service nor systemctl found) — assuming ${SERVICE} is not running"
+    return 1
+  fi
+}
+
+if service_running; then
   if [[ $ALLOW_RUNNING -ne 1 ]]; then
-    fail "${SERVICE} is running — stop it first (rc-service esellar-api stop) or pass --allow-running"
+    fail "${SERVICE} is running — stop it first (rc-service ${SERVICE} stop) or pass --allow-running"
   fi
   log "WARNING: migrating while ${SERVICE} is running (--allow-running)"
 fi

package/scripts/status.sh

@@ -1,44 +1,164 @@
 #!/usr/bin/env bash
-# status.sh — one command: what is LIVE (through the proxy) + the blue/green
-# pair view (instances, reserved IP, health).
+# status.sh — one command: deployment count (ledger), what is LIVE (through
+# the proxy), the VM's disk/image/service state (with --host), and the
+# blue/green pair view.
 #
 # Usage:
-#   kampodine status
+#   kampodine status [--host root@<ip>] [--ssh-key <path>]
 #
-# Env: APP_HOST_HEADER (default: app.example.com), OCI_PROFILE, OCI_COMPARTMENT.
+# Without --host: live health + the deployment count (all hosts). With
+# --host: adds VM disk usage (LOUD warning above 90%), the sha-tagged image
+# count with a reclaimable-by-prune estimate, and service states
+# (kampodine-api, kamal-proxy, walshipper if present).
+#
+# Env: APP_HOST_HEADER (default: app.example.com), KAMPODINE_HOST,
+# KAMPODINE_SSH_KEY, OCI_PROFILE, OCI_COMPARTMENT.
 set -euo pipefail
 HERE="$(cd "$(dirname "$0")" && pwd)"
-HOST="${APP_HOST_HEADER:-app.example.com}"
+# shellcheck source=common.sh
+source "$HERE/common.sh"
+
+HOST_HEADER="${PROFILE_PROXY_HOST:-${APP_HOST_HEADER:-app.example.com}}"
+IMAGE="127.0.0.1:5000/kampodine-api"
+DISK_PATH="/var/lib/containers"
+KAMPODINE_HOST="${KAMPODINE_HOST:-}"
+if [ -n "${KAMPODINE_SSH_KEY:-}" ]; then
+    SSH_KEY="$KAMPODINE_SSH_KEY"
+else
+    SSH_KEY=""
+fi
+PROFILE_FLAG=""
+HOST_FLAG=""
+KEY_FLAG=""
+VERBOSE=0
 
 say() { printf '%s\n' "$*"; }
 die() { printf '[status] FAIL: %s\n' "$*" >&2; exit 1; }
 usage() {
   cat <<'EOF'
 Usage:
-  kampodine status
+  kampodine status [--host root@<ip>] [--profile <name>] [--ssh-key <path>] [--verbose]
 
 Examples:
-  kampodine status    # live health (proxy host) + the blue/green pair view
+  kampodine status                          # deployments (ledger) + live health + blue/green pair
+  kampodine status --host root@203.0.113.10 # + VM disk usage, image/prune estimate, service states
+  kampodine status --profile prod           # resolve host/key from a config profile
+  kampodine status --host root@203.0.113.10 --verbose  # + the full metrics snapshot
+                                            #   (load, memory, disk breakdown, containers, top procs)
 
-Env: APP_HOST_HEADER (default app.example.com), OCI_PROFILE, OCI_COMPARTMENT.
+Env: APP_HOST_HEADER (default app.example.com), KAMPODINE_HOST,
+KAMPODINE_SSH_KEY, KAMPODINE_PROFILE, OCI_PROFILE, OCI_COMPARTMENT.
 EOF
   exit 0
 }
 
 while [[ $# -gt 0 ]]; do
   case "$1" in
+    --host) HOST_FLAG="$2"; KAMPODINE_HOST="$2"; shift 2 ;;
+    --ssh-key) KEY_FLAG="$2"; SSH_KEY="$2"; shift 2 ;;
+    --profile) PROFILE_FLAG="$2"; shift 2 ;;
+    --verbose) VERBOSE=1; shift ;;
     -h|--help) usage ;;
     *) die "unknown argument: $1 (--help)" ;;
   esac
 done
 
-say "== live (through the proxy: https://$HOST) =="
-if body="$(curl -sf -m 8 "https://$HOST/api/auth/ok" 2>/dev/null)"; then
+# profile resolution (flag > KAMPODINE_PROFILE > defaultProfile); an explicit
+# --host/--ssh-key flag always beats the profile
+profile_resolve "$PROFILE_FLAG" "$HOST_FLAG" "$KEY_FLAG"
+SSH_KEY="${KEY_FLAG:-${KAMPODINE_SSH_KEY:-}}"
+
+# --- deployment count (the owner's "total deployments", always visible) -------
+if [[ -n "$KAMPODINE_HOST" ]]; then
+  say "== deployments (ledger: $(ledger_file)) =="
+  ledger_total_line "$(ledger_count "$KAMPODINE_HOST")" "deployments to $KAMPODINE_HOST"
+else
+  say "== deployments (ledger: $(ledger_file)) =="
+  ledger_total_line "$(ledger_count)" "across all hosts — pass --host root@<ip> for VM state"
+  if [[ $VERBOSE -eq 1 ]]; then
+    say "metrics snapshot: needs a target — add --host root@<ip> (or --profile <name>)"
+  fi
+fi
+say
+
+say "== live (through the proxy: https://$HOST_HEADER) =="
+if body="$(curl -sf -m 8 "https://$HOST_HEADER/api/auth/ok" 2>/dev/null)"; then
   say "HEALTH OK: $body"
-  say "build-id  : $(curl -sf -m 8 "https://$HOST/build-id.txt" 2>/dev/null || echo unreachable)"
+  say "build-id  : $(curl -sf -m 8 "https://$HOST_HEADER/build-id.txt" 2>/dev/null || echo unreachable)"
 else
   say "UNREACHABLE or unhealthy — check the VM (ssh) and kamal-proxy"
 fi
+
+if [[ -n "$KAMPODINE_HOST" ]]; then
+  SSH_ARGS=(-o ConnectTimeout=10 -o BatchMode=yes)
+  [[ -n "$SSH_KEY" ]] && SSH_ARGS+=(-i "$SSH_KEY")
+  # $1 is a composed remote command — client-side expansion is the design.
+  # shellcheck disable=SC2029
+  vm() { ssh "${SSH_ARGS[@]}" "$KAMPODINE_HOST" "$1"; }
+  if [[ "$(uname -s)" == "Darwin" ]]; then
+    SSH_AUTH_SOCK="$(launchctl getenv SSH_AUTH_SOCK 2>/dev/null || true)"
+    export SSH_AUTH_SOCK
+  fi
+
+  say
+  say "== VM ($KAMPODINE_HOST) =="
+  df_out="$(vm "df -P $DISK_PATH 2>/dev/null" || true)"
+  pct="$(disk_pct_from_df "$df_out")"
+  case "$(disk_verdict "$pct" "")" in
+    ok) say "disk    : ${pct}% used on $DISK_PATH" ;;
+    warn)
+      say "disk    : ${pct}% used on $DISK_PATH"
+      say "WARNING : VM disk above 90% — old sha-tagged deploy images pile up (~1GB each); reclaim: kampodine deploy prune --dry-run"
+      ;;
+    *) say "disk    : unknown (df unreadable)" ;;
+  esac
+
+  images_raw="$(vm "podman images --format '{{.Tag}}|{{.CreatedAt}}|{{.Size}}' $IMAGE 2>/dev/null" || true)"
+  images="$(printf '%s\n' "$images_raw" | grep -E '^[0-9a-f]{7,40}\|' || true)"
+  running="$(vm "podman ps --format '{{.Image}}' 2>/dev/null" || true)"
+  img_count="$(printf '%s\n' "$images" | grep -c . || true)"
+  reclaim="$(prune_select "$images" "$running" 2 || true)"
+  reclaim_n="0"
+  reclaim_tags="(none)"
+  reclaim_size="~0 MB"
+  if [[ -n "$reclaim" ]]; then
+    reclaim_n="$(printf '%s\n' "$reclaim" | grep -c . || true)"
+    reclaim_tags="$(printf '%s\n' "$reclaim" | cut -d'|' -f1 | paste -sd, -)"
+    reclaim_size="$(sum_sizes_human "$(printf '%s\n' "$reclaim" | cut -d'|' -f2)")"
+  fi
+  say "images  : ${img_count:-0} sha-tagged deploy image(s); prune would remove ${reclaim_n} ${reclaim_tags} (${reclaim_size}): kampodine deploy prune --dry-run"
+
+  # init detection (groundwork for non-Alpine targets): the roll call keeps
+  # the historical shape on Alpine (byte-identical "for s in … rc-service …")
+  # and swaps the probe command on systemd hosts.
+  init_detect vm >/dev/null 2>&1 || true
+  case "$INIT_SYSTEM" in
+    systemd) svc_probe='systemctl status "$s"' ;;
+    *)       svc_probe='rc-service "$s" status' ;;
+  esac
+  say "services:"
+  vm "for s in kampodine-api kamal-proxy walshipper; do if $svc_probe >/dev/null 2>&1; then echo \"\$s: running\"; else echo \"\$s: not-running\"; fi; done" \
+    || say "(service roll call failed)"
+
+  if [[ $VERBOSE -eq 1 ]]; then
+    say
+    say "== metrics snapshot =="
+    if metrics_raw="$(vm "$METRICS_REMOTE_CMD" 2>/dev/null)"; then
+      metrics_render "$metrics_raw"
+      metrics_pct="$(disk_pct_from_df "$M_DISK")"
+      if [[ -n "$metrics_pct" ]] && [[ "$metrics_pct" -gt 90 ]]; then
+        say "WARNING : root disk above 90% — reclaim: kampodine deploy prune --dry-run"
+      fi
+    else
+      say "(metrics snapshot failed — VM unreachable?)"
+    fi
+  fi
+fi
+
 say
 say "== blue/green pair =="
-exec bash "$HERE/bluegreen.sh" status
+if [[ -n "$PROFILE_FLAG" ]]; then
+  exec bash "$HERE/bluegreen.sh" status --profile "$PROFILE_FLAG"
+else
+  exec bash "$HERE/bluegreen.sh" status
+fi