cozyclay 1.0.0

package/tools/ardy/run-on-box.sh

@@ -0,0 +1,568 @@
+#!/usr/bin/env bash
+# run-on-box.sh - generate an ARDY clip on the remote box.
+#
+#   run-on-box.sh [<pose.npz>] --base <motion-id|npz-path> --prompt "<prompt>" \
+#     --duration <seconds> [--dst-frame <N>] [--src-frame <N>] [--seed <S>] \
+#     [--root-2d FRAME X Z HEADING ...] [--output <local.npz>] [--dry-run]
+
+#
+# The pose npz is an ARDY motion npz (it must carry local_rot_mats and
+# posed_joints; the frame at SRC_FRAME holds the pose). It is pushed to a
+# fresh remote temp dir and handed to the generator as
+#   --pose-from <tmp>/pose.npz <src-frame> <dst-frame>
+# which pins the FULL-BODY pose onto <dst-frame> of the new clip. The base
+# motion supplies the unconstrained first pass (--base) and the generated
+# npz is pulled back. Nothing is ever written inside the ARDY checkout: the
+# base npz is read from ~/ardy/outputs/ (or outputs/omb/), the pose and the
+# result live under the remote temp dir, and the temp dir is removed on
+# exit.
+#
+# Three modes, chosen by what is given:
+#   pose       <pose.npz> present (the default): the pose npz is pushed to a
+#              fresh remote temp dir and handed to the constrained generator
+#              as --pose-from <tmp>/pose.npz <src-frame> <dst-frame>, pinning
+#              the FULL-BODY pose onto <dst-frame> of the new clip. Requires
+#              --base and --dst-frame.
+#   waypoints  no pose npz but one or more --root-2d groups: the constrained
+#              generator runs WITHOUT --pose-from, so the clip follows the
+#              path and prompt only, with no full-body pose constraint.
+#              --dst-frame is ignored. --base is optional: given, it is the
+#              unconstrained first pass; omitted, the script free-generates
+#              the base clip itself first (two-pass mode: scripts/generate.py
+#              with the same prompt/duration/seed under the remote temp dir,
+#              then the constrained pass with --base pointing at that npz).
+#   free       no pose npz and no --root-2d: scripts/generate.py on the box,
+#              the unconstrained generator (the constrained script refuses a
+#              run with no target, pose or waypoint). --base and --dst-frame
+#              are ignored.
+#
+# Each --root-2d group pins the root on the X/Z ground plane at one 0-based
+# clip frame (ARDY is Y-up; the horizontal plane is X and Z in meters, Y is
+# not constrained). X and Z must be within -20..20, HEADING is a yaw in
+# radians within -2π..2π or the literal 'none' to leave facing free. The
+# request carries 2..32 sparse authored keys starting at frame 0. Intermediate
+# root positions are left for the model to generate. Values are validated
+# locally and %q-quoted into the remote command like all other args.
+#
+# CPU is forced with CUDA_VISIBLE_DEVICES="" and that is deliberate: the
+# box's RTX 3070 has only 8.2 GB VRAM against 94 GB RAM, and nvidia-smi is
+# broken there (NVML driver/library version mismatch) even though
+# torch.cuda.is_available() still returns True - so without the override the
+# generator's `device = "cuda:0" if torch.cuda.is_available() else "cpu"`
+# line would pick a GPU that cannot hold the model. The preflight probe
+# below mirrors that exact line under the same env and prints the device the
+# generator will actually use.
+#
+# env (names shared with CozyClay scripts/ardy/sync-to-box):
+#   CCLAY_ARDY_HOST  ssh destination for the ARDY host (required)
+#   CCLAY_ARDY_REPO  ARDY checkout on the box       (default $HOME/ardy)
+#   CCLAY_ARDY_VENV  venv python on the box         (default ~/ardy/.venv/bin/python)
+set -euo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+HOST="${CCLAY_ARDY_HOST:-}"
+# REMOTE's default is escaped so the REMOTE shell expands $HOME (the same
+# trick sync-to-box uses); VENV_PY's default keeps a literal ~ so the REMOTE
+# shell tilde-expands it. Both resolve to the same ~/ardy on the box.
+REMOTE="${CCLAY_ARDY_REPO:-\$HOME/ardy}"
+# The box carries two venvs and they are NOT interchangeable: .venv runs the
+# CPU text-encoder service, .venv-cuda is what the motion generator was actually
+# run under (scripts/generate.py in generate_pc_monitor.log, 1060 MiB of VRAM).
+VENV_PY="${CCLAY_ARDY_VENV:-~/ardy/.venv-cuda/bin/python}"
+# Text encoder URL the generator falls back to when TEXT_ENCODER_URL is unset:
+# ardy/model/registry.py:47 DEFAULT_TEXT_ENCODER_URL. Generation dies without it,
+# so the preflight checks it rather than letting the run fail minutes in.
+ENCODER_URL="${CCLAY_ARDY_ENCODER_URL:-http://127.0.0.1:9550/}"
+# Force the motion model onto the CPU as well. Off by default: the encoder --
+# the part that does not fit beside the model in 8.2 GB -- already runs on the
+# CPU as its own service, and the model itself only needs about 1 GB.
+FORCE_CPU=0
+
+# BatchMode: never hang on a password prompt mid-pipeline. ConnectTimeout
+# fails fast on a dead host; ServerAlive* drops a wedged connection instead
+# of leaving the user staring at a silent ssh (30 s interval * 240 = 2 h).
+SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=10 -o ServerAliveInterval=30 -o ServerAliveCountMax=240)
+SCP_OPTS=(-o BatchMode=yes -o ConnectTimeout=10)
+
+usage() {
+  cat >&2 <<'EOF'
+usage: run-on-box.sh [<pose.npz>] --base <motion-id|npz-path> --prompt "<prompt>" \
+       --duration <seconds> [--dst-frame <N>] [--src-frame <N>] [--seed <S>] \
+       [--root-2d FRAME X Z HEADING ...] [--output <local.npz>] [--dry-run]
+
+
+  <pose.npz>    OPTIONAL local ARDY motion npz (local_rot_mats +
+posed_joints) whose frame <src-frame> holds the CozyClay
+                pose to honour. Omit it to generate without a full-body pose
+                constraint: with --root-2d groups the clip is
+                path/prompt-only constrained (with --base, or two-pass: the
+                free generator produces the base clip first); without them
+                it is free generation (scripts/generate.py on the box).
+  --base        base motion on the box: a bare id or bare <name>.npz
+                (looked up in ~/ardy/outputs/<name>.npz, then
+                ~/ardy/outputs/omb/<name>.npz), or a path containing /
+                relative to the ARDY checkout / absolute. Required in pose
+                mode; optional in waypoints mode (omitted = two-pass: the
+                free generator first produces the base clip on the box);
+                ignored in free mode.
+  --prompt      generation prompt for the constrained clip
+  --duration    clip duration in seconds (20 fps, 0.15..1200)
+  --dst-frame   clip frame to pin the pose onto, 0 <= dst < duration*20
+                (pose mode only)
+  --src-frame   frame of <pose.npz> to copy (default 0; pose mode only)
+  --seed        optional seed for reproducible results
+  --output      local destination for the generated npz
+                (default tools/ardy/out/<pose>-constrained.npz, or
+                tools/ardy/out/<epoch>-generated.npz without a pose)
+
+  --dry-run     run the preflight checks, print the exact remote command,
+                and exit without pushing, generating, or pulling
+  --root-2d     repeatable root waypoint: FRAME (0-based, inside the clip),
+                X and Z in meters within -20..20 (ARDY is Y-up; the ground
+                plane is X/Z, Y is not constrained), HEADING a yaw in
+                radians within -2π..2π or the literal 'none' to leave
+                facing free. Supply 2..32 sparse keys beginning at frame 0;
+                ARDY generates all intermediate root motion.
+
+env:
+  CCLAY_ARDY_HOST  ssh destination for the ARDY host (required)
+  CCLAY_ARDY_REPO  ARDY checkout on the box     (default $HOME/ardy)
+  --cpu         also run the motion model on the CPU (slower; the text encoder
+                is on the CPU either way, as its own service)
+
+env:
+  CCLAY_ARDY_VENV          venv python on the box (default ~/ardy/.venv-cuda/bin/python)
+  CCLAY_ARDY_ENCODER_URL   text encoder service   (default http://127.0.0.1:9550/)
+EOF
+  exit 2
+}
+
+POSE_NPZ="" # legacy positional pose; normalized into POSE_ARGS after parsing
+POSE_ARGS=() # repeatable triples: LOCAL_NPZ SRC_FRAME DST_FRAME
+BASE=""
+PROMPT=""
+DURATION=""
+DST_FRAME=""
+SRC_FRAME=0
+SEED=""
+OUTPUT=""
+DRY_RUN=0
+ROOT_2D_ARGS=()
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --base)
+      [[ $# -ge 2 ]] || { echo "run-on-box: --base needs a motion id or npz path" >&2; usage; }
+      BASE="$2"; shift 2 ;;
+    --prompt)
+      [[ $# -ge 2 ]] || { echo "run-on-box: --prompt needs text" >&2; usage; }
+      PROMPT="$2"; shift 2 ;;
+    --duration)
+      [[ $# -ge 2 ]] || { echo "run-on-box: --duration needs seconds" >&2; usage; }
+      [[ "$2" =~ ^[0-9]+([.][0-9]+)?$ ]] || { echo "run-on-box: --duration must be a number, got '$2'" >&2; usage; }
+      DURATION="$2"; shift 2 ;;
+    --dst-frame)
+      [[ $# -ge 2 ]] || { echo "run-on-box: --dst-frame needs a frame number" >&2; usage; }
+      [[ "$2" =~ ^[0-9]+$ ]] || { echo "run-on-box: --dst-frame must be a non-negative integer, got '$2'" >&2; usage; }
+      DST_FRAME="$2"; shift 2 ;;
+    --src-frame)
+      [[ $# -ge 2 ]] || { echo "run-on-box: --src-frame needs a frame number" >&2; usage; }
+      [[ "$2" =~ ^[0-9]+$ ]] || { echo "run-on-box: --src-frame must be a non-negative integer, got '$2'" >&2; usage; }
+      SRC_FRAME="$2"; shift 2 ;;
+    --seed)
+      [[ $# -ge 2 ]] || { echo "run-on-box: --seed needs an integer" >&2; usage; }
+      [[ "$2" =~ ^[0-9]+$ ]] || { echo "run-on-box: --seed must be a non-negative integer, got '$2'" >&2; usage; }
+      SEED="$2"; shift 2 ;;
+    --pose-from)
+      [[ $# -ge 4 ]] || { echo "run-on-box: --pose-from needs LOCAL_NPZ SRC_FRAME DST_FRAME" >&2; usage; }
+      [[ "$3" =~ ^[0-9]+$ && "$4" =~ ^[0-9]+$ ]] || { echo "run-on-box: --pose-from frames must be non-negative integers" >&2; usage; }
+      POSE_ARGS+=("$2" "$3" "$4"); shift 4 ;;
+    --root-2d)
+      [[ $# -ge 5 ]] || { echo "run-on-box: --root-2d needs FRAME X Z HEADING" >&2; usage; }
+      ROOT_2D_ARGS+=("$2" "$3" "$4" "$5"); shift 5 ;;
+    --cpu)
+      FORCE_CPU=1; shift ;;
+    --output)
+      [[ $# -ge 2 ]] || { echo "run-on-box: --output needs a path" >&2; usage; }
+      OUTPUT="$2"; shift 2 ;;
+    --dry-run)
+      DRY_RUN=1; shift ;;
+    -h|--help)
+      usage ;;
+    -*)
+      echo "run-on-box: unknown option '$1'" >&2; usage ;;
+    *)
+      [[ -z "$POSE_NPZ" ]] || { echo "run-on-box: one pose npz only, got '$1'" >&2; usage; }
+      POSE_NPZ="$1"; shift ;;
+  esac
+done
+
+[[ -n "$HOST" ]] || {
+  echo "run-on-box: CCLAY_ARDY_HOST is required (for example: user@ardy-host)" >&2
+  exit 2
+}
+
+# Normalize the legacy positional pose into the repeatable pose list.
+if [[ -n "$POSE_NPZ" ]]; then
+  [[ -n "$DST_FRAME" ]] || usage
+  POSE_ARGS+=("$POSE_NPZ" "$SRC_FRAME" "$DST_FRAME")
+fi
+
+# Mode by what is given: one or more pose npz files pin the full body,
+# waypoints alone constrain the path, and neither is free generation.
+MODE=""
+if [[ ${#POSE_ARGS[@]} -gt 0 ]]; then
+  MODE="pose"
+elif [[ ${#ROOT_2D_ARGS[@]} -gt 0 ]]; then
+  MODE="waypoints"
+else
+  MODE="free"
+fi
+# Waypoints run the constrained generator: ONE model sampling call for the
+# whole clip. ARDY's trained window is 10 s — frames beyond it sit outside
+# the model's temporal horizon and the motion visibly degrades, so the last
+# line of defence refuses what every upstream layer should already have.
+if [[ "$MODE" == "waypoints" ]] && ! awk -v d="$DURATION" 'BEGIN { exit !(d <= 10) }'; then
+  echo "run-on-box: --root-2d requires --duration <= 10 s (one-shot constrained generation; ARDY trained window), got ${DURATION}s" >&2
+  exit 1
+fi
+# Waypoints without --base is two-pass: the free generator first produces
+# the base clip on the box (pass 1), then the constrained pass uses it.
+TWO_PASS=0
+[[ "$MODE" == "waypoints" && -z "$BASE" ]] && TWO_PASS=1
+IMPLICIT_POSE_BASE=0
+
+[[ -n "$PROMPT" && -n "$DURATION" ]] || usage
+if [[ "$MODE" == "pose" && -z "$BASE" ]]; then
+  # Internal parser/reference baseline only. Full-body poses are the actual
+  # constraints; users never select a semantic base motion.
+  BASE="${CCLAY_ARDY_REFERENCE_BASE:-stand-upright-facing-forward-bring-both--0722140757}"
+  IMPLICIT_POSE_BASE=1
+fi
+# --- local validation, mirroring the CozyClay wrapper's bounds -------------
+if [[ "$MODE" == "pose" ]]; then
+  for ((i = 0; i < ${#POSE_ARGS[@]}; i += 3)); do
+    pose_path="${POSE_ARGS[$i]}"
+    [[ -f "$pose_path" ]] || { echo "run-on-box: pose npz not found: $pose_path" >&2; exit 1; }
+  done
+fi
+
+[[ -n "$PROMPT" ]] || { echo "run-on-box: --prompt must not be empty" >&2; exit 1; }
+# The clip is int(duration * 20) frames at ARDY's 20 fps, capped at 1200 s
+# like the wrapper; the remote rejects anything below 3 frames because
+# inter-frame continuity is undefined there.
+awk -v d="$DURATION" 'BEGIN { exit !(d > 0 && d <= 1200) }' || {
+  echo "run-on-box: --duration must be > 0 and <= 1200 seconds, got '$DURATION'" >&2; exit 1; }
+CLIP_FRAMES="$(awk -v d="$DURATION" 'BEGIN { printf "%d", d * 20 }')"
+[[ "$CLIP_FRAMES" -ge 3 ]] || {
+  echo "run-on-box: --duration ${DURATION}s yields ${CLIP_FRAMES} frame(s) at 20 fps; a clip needs at least 3" >&2; exit 1; }
+if [[ "$MODE" == "pose" ]]; then
+  for ((i = 0; i < ${#POSE_ARGS[@]}; i += 3)); do
+    dst="${POSE_ARGS[$((i + 2))]}"
+    awk -v f="$dst" -v m="$CLIP_FRAMES" 'BEGIN { exit !(f < m) }' || {
+      echo "run-on-box: --pose-from destination $dst is outside 0..$((CLIP_FRAMES - 1)) (duration ${DURATION}s at 20 fps)" >&2; exit 1; }
+  done
+fi
+# --- --root-2d validation: each group, then the set as a whole -------------
+# Mirrors the bridge contract: 2..32 sparse keys beginning at frame 0,
+# strictly ascending, X/Z in meters within -20..20, HEADING a finite number
+# of radians within -2π..2π or the literal 'none'.
+FLOAT_RE='^[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?$'
+validate_root_2d() {
+  local n="${#ROOT_2D_ARGS[@]}" i frame x z heading prev=-1 count
+  [[ $((n % 4)) -eq 0 ]] || { echo "run-on-box: internal error: --root-2d group list is misaligned" >&2; exit 1; }
+  count=$((n / 4))
+  if [[ "$count" -ne 0 && ( "$count" -lt 2 || "$count" -gt 32 ) ]]; then
+    echo "run-on-box: --root-2d needs 2..32 sparse waypoints, got $count" >&2
+    exit 1
+  fi
+  for ((i = 0; i < n; i += 4)); do
+    frame="${ROOT_2D_ARGS[$i]}"
+    x="${ROOT_2D_ARGS[$((i + 1))]}"
+    z="${ROOT_2D_ARGS[$((i + 2))]}"
+    heading="${ROOT_2D_ARGS[$((i + 3))]}"
+    [[ "$frame" =~ ^[0-9]+$ ]] || { echo "run-on-box: --root-2d FRAME must be a non-negative integer, got '$frame'" >&2; exit 1; }
+    awk -v f="$frame" -v m="$CLIP_FRAMES" 'BEGIN { exit !(f >= 0 && f < m) }' || {
+      echo "run-on-box: --root-2d frame $frame is outside 0..$((CLIP_FRAMES - 1)) (duration ${DURATION}s at 20 fps)" >&2
+      exit 1
+    }
+    awk -v f="$frame" -v p="$prev" 'BEGIN { exit !(f > p) }' || {
+      echo "run-on-box: --root-2d frames must be strictly ascending; frame $frame repeats or precedes frame $prev" >&2
+      exit 1
+    }
+    if [[ $i -eq 0 && "$frame" -ne 0 ]]; then
+      echo "run-on-box: --root-2d start must be at frame 0, got frame $frame" >&2
+      exit 1
+    fi
+    prev="$frame"
+    [[ "$x" =~ $FLOAT_RE ]] || { echo "run-on-box: --root-2d X must be a number, got '$x'" >&2; exit 1; }
+    awk -v v="$x" 'BEGIN { exit !(v >= -20 && v <= 20) }' || {
+      echo "run-on-box: --root-2d X $x is outside -20..20 meters" >&2
+      exit 1
+    }
+    [[ "$z" =~ $FLOAT_RE ]] || { echo "run-on-box: --root-2d Z must be a number, got '$z'" >&2; exit 1; }
+    awk -v v="$z" 'BEGIN { exit !(v >= -20 && v <= 20) }' || {
+      echo "run-on-box: --root-2d Z $z is outside -20..20 meters" >&2
+      exit 1
+    }
+    if [[ "$heading" != "none" ]]; then
+      [[ "$heading" =~ $FLOAT_RE ]] || {
+        echo "run-on-box: --root-2d HEADING must be a number of radians or the literal 'none', got '$heading'" >&2
+        exit 1
+      }
+      awk -v h="$heading" 'BEGIN { pi = atan2(0, -1); exit !(h >= -2*pi && h <= 2*pi) }' || {
+        echo "run-on-box: --root-2d HEADING $heading rad is outside -2π..2π" >&2
+        exit 1
+      }
+    fi
+  done
+}
+validate_root_2d
+# SRC_FRAME indexes the pose npz; its real bound is only knowable on the
+# box, so like the reference wrapper we accept any non-negative integer here
+# and let the remote range-check it against the npz's frame count.
+if [[ "$MODE" == "pose" ]]; then
+  FIRST_POSE="${POSE_ARGS[0]}"
+  OUTPUT="${OUTPUT:-$HERE/out/$(basename "${FIRST_POSE%.npz}")-constrained.npz}"
+else
+  OUTPUT="${OUTPUT:-$HERE/out/$(date +%s)-generated.npz}"
+fi
+
+
+# --- preflight: fail fast, cheapest first ---------------------------------
+if ! ssh "${SSH_OPTS[@]}" "$HOST" ":" </dev/null; then
+  echo "run-on-box: cannot ssh to ${HOST} (BatchMode). Check network reachability and the SSH key agent." >&2
+  exit 1
+fi
+echo "run-on-box: ssh to ${HOST} ok"
+
+# Free generation runs scripts/generate.py (the constrained script refuses a
+# run with no target, pose or waypoint); every other mode runs the
+# constrained generator. Two-pass waypoints mode needs BOTH scripts:
+# generate.py for the free base pass, then the constrained script.
+GEN_SCRIPT="scripts/cclay_constrained_generate.py"
+[[ "$MODE" == "free" ]] && GEN_SCRIPT="scripts/generate.py"
+GEN_CHECK="test -x ${VENV_PY} && test -f ${REMOTE}/${GEN_SCRIPT}"
+GEN_LABEL="${REMOTE}/${GEN_SCRIPT}"
+if [[ "$TWO_PASS" -eq 1 || "$IMPLICIT_POSE_BASE" -eq 1 ]]; then
+  GEN_CHECK="test -x ${VENV_PY} && test -f ${REMOTE}/scripts/generate.py && test -f ${REMOTE}/scripts/cclay_constrained_generate.py"
+  GEN_LABEL="${REMOTE}/scripts/generate.py and ${REMOTE}/scripts/cclay_constrained_generate.py"
+fi
+# shellcheck disable=SC2029  # REMOTE/VENV_PY expand remotely by design
+if ! ssh "${SSH_OPTS[@]}" "$HOST" "$GEN_CHECK" </dev/null; then
+  echo "run-on-box: ${HOST} is missing ${VENV_PY} or ${GEN_LABEL}; sync the ARDY sources to the box first (CozyClay scripts/ardy/sync-to-box --apply)" >&2
+  exit 1
+fi
+echo "run-on-box: venv python and generator script present on ${HOST}"
+
+
+# Resolve --base on the box: anything containing a slash is a repo-relative
+# or absolute path and passes through; a bare id or bare <name>.npz is
+# looked up under outputs/ then outputs/omb/ (repo-relative, i.e.
+# ~/ardy/outputs*). Free mode takes no base, and waypoints mode without one
+# is two-pass: the pass-1 output is a fresh absolute temp path verified with
+# a plain test -f at the pass-1 step, never through this candidate lookup.
+if [[ -n "$BASE" ]]; then
+  if [[ "$BASE" == */* ]]; then
+    BASE_CANDIDATES="$(printf '%q' "$BASE")"
+    BASE_LOOKED="'${BASE}'"
+  else
+    BASE_NAME="${BASE%.npz}"
+    BASE_CANDIDATES="$(printf '%q' "outputs/${BASE_NAME}.npz") $(printf '%q' "outputs/omb/${BASE_NAME}.npz")"
+    BASE_LOOKED="outputs/${BASE_NAME}.npz or outputs/omb/${BASE_NAME}.npz under ${REMOTE}"
+  fi
+  # shellcheck disable=SC2029  # REMOTE expands remotely by design
+  if ! BASE_RESOLVED="$(ssh "${SSH_OPTS[@]}" "$HOST" "cd ${REMOTE} && for c in ${BASE_CANDIDATES}; do if test -f \"\$c\"; then printf '%s' \"\$c\"; exit 0; fi; done; exit 1" </dev/null)"; then
+    echo "run-on-box: base motion '${BASE}' does not exist on ${HOST} (looked for ${BASE_LOOKED})" >&2
+    exit 1
+  fi
+  echo "run-on-box: base motion on ${HOST}: ${BASE_RESOLVED}"
+fi
+
+# The hidden upright reference is only a convenient baseline when it is long
+# enough. A loaded clip can be longer than that fixture; in that case generate
+# an exact-duration base first instead of letting cclay_constrained_generate.py
+# fail after model load with "base has N frames but requested M".
+if [[ "$IMPLICIT_POSE_BASE" -eq 1 ]]; then
+  # shellcheck disable=SC2029  # resolved path is selected from the box listing
+  if ! BASE_FRAMES="$(ssh "${SSH_OPTS[@]}" "$HOST" \
+    "cd ${REMOTE} && ${VENV_PY} -c 'import numpy as np,sys; print(np.load(sys.argv[1])[\"local_rot_mats\"].shape[0])' $(printf '%q' "$BASE_RESOLVED")" </dev/null)"; then
+    echo "run-on-box: could not inspect implicit pose base ${BASE_RESOLVED}" >&2
+    exit 1
+  fi
+  [[ "$BASE_FRAMES" =~ ^[0-9]+$ ]] || {
+    echo "run-on-box: implicit pose base returned invalid frame count '${BASE_FRAMES}'" >&2
+    exit 1
+  }
+  if [[ "$BASE_FRAMES" -lt "$CLIP_FRAMES" ]]; then
+    echo "run-on-box: implicit pose base has ${BASE_FRAMES} frames; will free-generate ${CLIP_FRAMES}-frame base first"
+    BASE_RESOLVED=""
+    TWO_PASS=1
+  fi
+fi
+
+
+# Probe the device the generator will use, with the SAME env it will run
+# under (CUDA_VISIBLE_DEVICES="" included). The one-liner mirrors the
+# generator's `device = "cuda:0" if torch.cuda.is_available() else "cpu"`
+# exactly, so what is printed is what the generation will pick.
+# shellcheck disable=SC2029  # REMOTE/VENV_PY expand remotely by design
+CUDA_ENV=""
+[[ "$FORCE_CPU" -eq 1 ]] && CUDA_ENV='CUDA_VISIBLE_DEVICES="" '
+
+# The text encoder is a separate CPU service; without it the generator loads the
+# model, then dies. Check it before spending minutes on the rest.
+if ! ENCODER_CODE="$(ssh "${SSH_OPTS[@]}" "$HOST" "curl -s -o /dev/null -w '%{http_code}' -m 10 ${ENCODER_URL}" </dev/null)"; then
+  echo "run-on-box: could not reach ${HOST} to probe the text encoder" >&2
+  exit 1
+fi
+if [[ "$ENCODER_CODE" != "200" ]]; then
+  echo "run-on-box: text encoder at ${ENCODER_URL} answered '${ENCODER_CODE}', not 200." >&2
+  echo "run-on-box: start it with:  ssh ${HOST} 'cd ~/ardy && .venv/bin/python -u scripts/run_text_encoder_server.py --host 127.0.0.1 --port 9550 --device cpu'" >&2
+  exit 1
+fi
+echo "run-on-box: text encoder ${ENCODER_URL} responding (cpu service)"
+
+if ! DEVICE="$(ssh "${SSH_OPTS[@]}" "$HOST" "cd ${REMOTE} && ${CUDA_ENV}${VENV_PY} -c 'import torch; print(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")'" </dev/null)"; then
+  echo "run-on-box: device probe on ${HOST} failed (venv python erroring?)" >&2
+  exit 1
+fi
+[[ -n "$DEVICE" ]] || { echo "run-on-box: device probe on ${HOST} returned nothing" >&2; exit 1; }
+echo "run-on-box: generator will use device '${DEVICE}' on ${HOST}"
+
+# --- the generator invocation ---------------------------------------------
+# Args are shell-quoted with printf %q so prompts and paths survive the
+# remote shell. The output base is absolute (under the remote temp dir), so
+# the generator's _resolve_output_base honors it instead of placing it under
+# outputs/.
+# scripts/generate.py is the unconstrained generator: its prompt is
+# positional and it has no --base / --pose-from / --root-2d. Used for free
+# mode and for the two-pass base pass (pass 1); the npz lands at
+# <out_base>.npz.
+build_free_cmd() {
+  local out_base="$1" cmd
+  cmd="cd ${REMOTE} && ${CUDA_ENV}${VENV_PY} scripts/generate.py $(printf '%q' "$PROMPT")"
+  cmd+=" --duration $(printf '%q' "$DURATION")"
+  cmd+=" --output $(printf '%q' "$out_base")"
+  if [[ -n "$SEED" ]]; then
+    cmd+=" --seed $(printf '%q' "$SEED")"
+  fi
+  printf '%s' "$cmd"
+}
+
+build_remote_cmd() {
+  local tmp_dir="$1" cmd i
+  # Two-pass waypoints mode has no user base: BASE_RESOLVED stays empty and
+  # pass 1's output npz (under the same remote temp dir, cleaned by the EXIT
+  # trap) becomes the --base of the constrained pass.
+  local base_arg="${BASE_RESOLVED:-${tmp_dir}/base.npz}"
+  if [[ "$MODE" == "free" ]]; then
+    cmd="$(build_free_cmd "${tmp_dir}/out")"
+  else
+    cmd="cd ${REMOTE} && ${CUDA_ENV}${VENV_PY} scripts/cclay_constrained_generate.py"
+    cmd+=" --prompt $(printf '%q' "$PROMPT")"
+    cmd+=" --duration $(printf '%q' "$DURATION")"
+    cmd+=" --base $(printf '%q' "$base_arg")"
+    if [[ "$MODE" == "pose" ]]; then
+      for ((i = 0; i < ${#POSE_ARGS[@]}; i += 3)); do
+        pose_index=$((i / 3))
+        cmd+=" --pose-from $(printf '%q' "${tmp_dir}/pose-${pose_index}.npz")"
+        cmd+=" $(printf '%q' "${POSE_ARGS[$((i + 1))]}") $(printf '%q' "${POSE_ARGS[$((i + 2))]}")"
+      done
+    fi
+    # Each validated --root-2d group is %q-quoted value by value, so no
+    # waypoint text can ever reach the remote shell unquoted.
+    for ((i = 0; i < ${#ROOT_2D_ARGS[@]}; i += 4)); do
+      cmd+=" --root-2d $(printf '%q' "${ROOT_2D_ARGS[$i]}")"
+      cmd+=" $(printf '%q' "${ROOT_2D_ARGS[$((i + 1))]}")"
+      cmd+=" $(printf '%q' "${ROOT_2D_ARGS[$((i + 2))]}")"
+      cmd+=" $(printf '%q' "${ROOT_2D_ARGS[$((i + 3))]}")"
+    done
+    cmd+=" --output $(printf '%q' "${tmp_dir}/out")"
+    if [[ -n "$SEED" ]]; then
+      cmd+=" --seed $(printf '%q' "$SEED")"
+    fi
+  fi
+  printf '%s' "$cmd"
+}
+
+
+if [[ $DRY_RUN -eq 1 ]]; then
+  REMOTE_TMP_PLACEHOLDER="<mktemp-dir-on-box>"
+  MODE_LABEL="free generation (generate.py)"
+  [[ "$MODE" == "pose" ]] && MODE_LABEL="full-body pose pin"
+  [[ "$MODE" == "waypoints" ]] && MODE_LABEL="frame-0 root start + prompt (no pose)"
+  [[ "$MODE" == "waypoints" && "$TWO_PASS" -eq 1 ]] && MODE_LABEL="frame-0 root start + prompt (no pose) - two-pass (free base first)"
+  echo "run-on-box: DRY RUN - preflight done, nothing pushed, nothing generated, nothing pulled."
+  echo "run-on-box: mode        ${MODE} - ${MODE_LABEL}"
+  [[ "$MODE" == "pose" ]] || echo "run-on-box: pose npz    <none>"
+  [[ "$MODE" != "pose" ]] || echo "run-on-box: pose pins   $(( ${#POSE_ARGS[@]} / 3 ))"
+  if [[ "$TWO_PASS" -eq 1 ]]; then
+    echo "run-on-box: base motion pass-1 output ${REMOTE_TMP_PLACEHOLDER}/base.npz on ${HOST} (free-generated first)"
+  else
+    [[ -z "${BASE_RESOLVED:-}" ]] || echo "run-on-box: base motion ${BASE_RESOLVED} on ${HOST}"
+  fi
+  echo "run-on-box: device      ${DEVICE}"
+  echo "run-on-box: root-2d    $(( ${#ROOT_2D_ARGS[@]} / 4 )) waypoint(s)"
+  [[ "$MODE" != "pose" ]] || echo "run-on-box: would push  $(( ${#POSE_ARGS[@]} / 3 )) pose npz file(s) to ${HOST}:${REMOTE_TMP_PLACEHOLDER}"
+  if [[ "$TWO_PASS" -eq 1 ]]; then
+    echo "run-on-box: pass 1/2   would run  ssh ${HOST} \"$(build_free_cmd "${REMOTE_TMP_PLACEHOLDER}/base")\""
+    echo "run-on-box: pass 2/2   would run  ssh ${HOST} \"$(build_remote_cmd "$REMOTE_TMP_PLACEHOLDER")\""
+  else
+    echo "run-on-box: would run   ssh ${HOST} \"$(build_remote_cmd "$REMOTE_TMP_PLACEHOLDER")\""
+  fi
+  echo "run-on-box: would pull  scp -q -o BatchMode=yes ${HOST}:${REMOTE_TMP_PLACEHOLDER}/out.npz ${OUTPUT}"
+  echo "run-on-box: would clean ssh ${HOST} \"rm -rf ${REMOTE_TMP_PLACEHOLDER}\""
+  exit 0
+fi
+
+mkdir -p "$(dirname "$OUTPUT")"
+
+# Fresh temp dir on the box; the EXIT trap removes it even on failure, and
+# cleanup failure never masks the real result. Guard the path (must be
+# absolute, as mktemp guarantees) before it can reach the cleanup rm -rf.
+if ! REMOTE_TMP="$(ssh "${SSH_OPTS[@]}" "$HOST" "mktemp -d" </dev/null)"; then
+  echo "run-on-box: could not create a temp dir on ${HOST} (mktemp failed)" >&2
+  exit 1
+fi
+[[ "$REMOTE_TMP" == /* ]] || { echo "run-on-box: remote mktemp returned a non-absolute path '${REMOTE_TMP}'; aborting" >&2; exit 1; }
+trap 'ssh "${SSH_OPTS[@]}" "$HOST" "rm -rf ${REMOTE_TMP}" </dev/null >/dev/null 2>&1 || true' EXIT
+
+if [[ "$TWO_PASS" -eq 1 ]]; then
+  # Pass 1: free-generate the base clip with the same prompt/duration/seed
+  # the constrained pass will use. Its npz lives under REMOTE_TMP, so the
+  # EXIT trap removes it with everything else.
+  PASS1_CMD="$(build_free_cmd "${REMOTE_TMP}/base")"
+  echo "run-on-box: pass 1/2: free-generating base clip on ${HOST} ..."
+  # shellcheck disable=SC2029  # PASS1_CMD is a remote command by design
+  ssh "${SSH_OPTS[@]}" "$HOST" "$PASS1_CMD" </dev/null
+  BASE_RESOLVED="${REMOTE_TMP}/base.npz"
+  # The pass-1 output is a fresh absolute temp path, not a listed base: no
+  # BASE_CANDIDATES lookup, just verify the file appeared.
+  # shellcheck disable=SC2029  # REMOTE_TMP expands remotely by design
+  if ! ssh "${SSH_OPTS[@]}" "$HOST" "test -f ${BASE_RESOLVED}" </dev/null; then
+    echo "run-on-box: pass 1/2 failed: ${BASE_RESOLVED} was not produced on ${HOST}" >&2
+    exit 1
+  fi
+  echo "run-on-box: pass 1/2: base clip at ${HOST}:${BASE_RESOLVED}"
+fi
+
+if [[ "$MODE" == "pose" ]]; then
+  for ((i = 0; i < ${#POSE_ARGS[@]}; i += 3)); do
+    pose_index=$((i / 3))
+    pose_path="${POSE_ARGS[$i]}"
+    echo "run-on-box: pushing pose $((pose_index + 1))/$(( ${#POSE_ARGS[@]} / 3 )) to ${HOST}:${REMOTE_TMP}/pose-${pose_index}.npz ..."
+    scp -q "${SCP_OPTS[@]}" "$pose_path" "$HOST:${REMOTE_TMP}/pose-${pose_index}.npz"
+  done
+fi
+
+REMOTE_CMD="$(build_remote_cmd "$REMOTE_TMP")"
+echo "run-on-box: generating on ${HOST} ..."
+# shellcheck disable=SC2029  # REMOTE_CMD is a remote command by design
+ssh "${SSH_OPTS[@]}" "$HOST" "$REMOTE_CMD" </dev/null
+
+echo "run-on-box: pulling ${HOST}:${REMOTE_TMP}/out.npz -> ${OUTPUT} ..."
+# shellcheck disable=SC2029  # HOST:REMOTE_TMP is a remote path by design
+scp -q "${SCP_OPTS[@]}" "$HOST:${REMOTE_TMP}/out.npz" "$OUTPUT"
+[[ -s "$OUTPUT" ]] || { echo "run-on-box: pulled ${OUTPUT} is empty; the generation failed" >&2; exit 1; }
+echo "run-on-box: done - ${OUTPUT} ($(wc -c < "$OUTPUT" | tr -d ' ') bytes)"