novahiz 0.1.0

package/bundled-skills/expo-skill-eval/scripts/make-workspace.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# Create the eval workspace directory tree for a run.
+#
+# Usage: make-workspace.sh <workspace> <iteration> <num-evals> [configs]
+#   workspace   Run root, e.g. /private/tmp/expo-skill-eval-<skill>
+#   iteration   Iteration folder name, e.g. "iteration-1"
+#   num-evals   Number of eval cases; creates eval-0 .. eval-(N-1)
+#   configs     Space- or comma-separated config names
+#               (default: "with_skill without_skill")
+#
+# Creates <workspace>/trigger-evals/scratch and, per eval x config,
+# <workspace>/<iteration>/eval-<i>/<config>/outputs. Run this once at the
+# start of a run instead of ad-hoc `mkdir` so the call is covered by the
+# skill's `Bash(bash *expo-skill-eval/scripts/*)` rule (the mkdir calls
+# inside run as children of this script and need no permission of their own).
+set -euo pipefail
+
+WORKSPACE="${1:?usage: make-workspace.sh <workspace> <iteration> <num-evals> [configs]}"
+ITERATION="${2:?usage: make-workspace.sh <workspace> <iteration> <num-evals> [configs]}"
+NUM_EVALS="${3:?usage: make-workspace.sh <workspace> <iteration> <num-evals> [configs]}"
+CONFIGS="${4:-with_skill without_skill}"
+CONFIGS="${CONFIGS//,/ }"
+
+mkdir -p "$WORKSPACE/trigger-evals/scratch"
+for ((i = 0; i < NUM_EVALS; i++)); do
+  for cfg in $CONFIGS; do
+    mkdir -p "$WORKSPACE/$ITERATION/eval-$i/$cfg/outputs"
+  done
+done
+
+echo "workspace ready: $WORKSPACE/$ITERATION ($NUM_EVALS evals x [$CONFIGS])"

package/bundled-skills/expo-skill-eval/scripts/snapshot-android.sh

@@ -0,0 +1,284 @@
+#!/usr/bin/env bash
+# Run an Expo app in Expo Go on an Android emulator and capture a screenshot.
+#
+# Usage: snapshot-android.sh <project-path> <output-png> [port] [settle-seconds]
+#
+# Env: EXPO_SKILL_EVAL_RUNNER = expo-go (default) | dev-build — dev-build runs the
+#   app via `expo run:android` (native dev-client compile, slow) instead of Expo Go.
+#
+# If no device is attached, boots the first available AVD and waits for it
+# (the slow path - keep one emulator running across an eval session).
+# Starts Metro with `expo start --android` (auto-installs Expo Go, sets up
+# adb reverse, opens the app), waits for the bundle, settles, screenshots
+# via screencap, and tears Metro down. The Metro log is saved next to the
+# screenshot as <name>.metro.log. On bundle failure it still captures a
+# best-effort screenshot and exits 1.
+set -uo pipefail
+
+PROJECT_PATH="${1:?usage: snapshot-android.sh <project-path> <output-png> [port] [settle-seconds]}"
+OUT="${2:?usage: snapshot-android.sh <project-path> <output-png> [port] [settle-seconds]}"
+PORT="${3:-8082}"
+RUNNER="${EXPO_SKILL_EVAL_RUNNER:-expo-go}"   # expo-go | dev-build
+
+# Step-level logging with timestamp. Every major phase emits a marker so a
+# crash report can be correlated to "which step was running when it died."
+log_step() { echo "[$(date '+%H:%M:%S')] [snapshot-android] STEP: $*" >&2; }
+
+# Track the emulator qemu PID so liveness checks in wait loops can detect a
+# mid-flight crash and emit a clear "emulator crashed at step X" message.
+EMULATOR_PID=""
+
+emulator_alive() {
+  [[ -n "$EMULATOR_PID" ]] && kill -0 "$EMULATOR_PID" 2>/dev/null
+}
+
+log_step "start  runner=$RUNNER port=$PORT project=$PROJECT_PATH"
+
+# GPU mode is fixed to host (Metal on Apple Silicon, hardware-accelerated). The
+# env-var knob was removed on purpose. If host self-aborts the emulator on a
+# given machine (qemu SIGABRT deep in gfxstream/Metal — possible on Apple Silicon
+# under load), change this to "guest" and bump SETTLE to ~25 (software rendering
+# paints slowly). Avoid swiftshader_indirect — it hangs at boot on arm64.
+GPU_MODE="host"
+log_step "gpu_mode=$GPU_MODE"
+
+SETTLE="${4:-8}"
+
+if [[ "$RUNNER" == "dev-build" ]]; then
+  BUNDLE_TIMEOUT="${EXPO_SKILL_EVAL_BUNDLE_TIMEOUT:-900}"   # native compile is slow
+else
+  BUNDLE_TIMEOUT="${EXPO_SKILL_EVAL_BUNDLE_TIMEOUT:-240}"
+fi
+LOG="${OUT%.png}.metro.log"
+
+mkdir -p "$(dirname "$OUT")"
+
+# Free the port up front in case a previous run (or a crash) left Metro on it,
+# so `expo start` binds the port we expect instead of silently incrementing.
+# Runs inside this script, so the eval harness never needs ad-hoc lsof/kill.
+lsof -ti "tcp:$PORT" -sTCP:LISTEN 2>/dev/null | xargs kill -9 2>/dev/null || true
+
+find_sdk_tool() {
+  local tool="$1" subdir="$2"
+  if command -v "$tool" >/dev/null; then
+    command -v "$tool"
+    return
+  fi
+  local sdk
+  for sdk in "${ANDROID_HOME:-}" "${ANDROID_SDK_ROOT:-}" "$HOME/Library/Android/sdk" "$HOME/Android/Sdk"; do
+    if [[ -n "$sdk" && -x "$sdk/$subdir/$tool" ]]; then
+      echo "$sdk/$subdir/$tool"
+      return
+    fi
+  done
+  return 1
+}
+
+ADB="$(find_sdk_tool adb platform-tools)" || { echo "error: adb not found (set ANDROID_HOME)" >&2; exit 1; }
+
+# Recycle a wedged emulator before the boot check. An "offline"/unresponsive
+# emulator isn't in "device" state, so the check below would otherwise reuse it
+# (and hang) or stack a second emulator beside it. Graceful console shutdown
+# first; force-kill only the survivors, then reset adb to clear the stale entry —
+# so the boot check sees a clean slate and cold-boots a fresh (swiftshader) one.
+log_step "checking for wedged/offline emulators"
+STALE="$("$ADB" devices | awk 'NR>1 && $1 ~ /^emulator-/ && $2!="device" {print $1}')"
+if [[ -n "$STALE" ]]; then
+  echo "recycling wedged emulator(s): $STALE" >&2
+  for serial in $STALE; do
+    "$ADB" -s "$serial" emu kill 2>/dev/null || true   # graceful: ask the console to quit
+  done
+  sleep 2
+  # Still wedged after the graceful attempt? Force-kill the qemu process(es) and
+  # reset the adb server to drop the stale "offline" entry. pkill is broad (all
+  # qemu emulators), which is fine here: the eval harness runs a single emulator.
+  if "$ADB" devices | awk 'NR>1 && $1 ~ /^emulator-/ && $2!="device"' | grep -q .; then
+    pkill -9 -f qemu-system-aarch64 2>/dev/null || true
+    "$ADB" kill-server 2>/dev/null || true
+    "$ADB" start-server 2>/dev/null || true
+    sleep 2
+  fi
+fi
+
+# GPU mismatch: if an emulator is already running with a different GPU mode than
+# the one we need, recycle it so the boot path cold-starts with the correct mode.
+# gfxstream (host GPU) uses a gRPC channel that can crash on Metro teardown via
+# a bad_function_call SIGABRT; guest GPU avoids this by keeping rendering in-guest.
+CURRENT_GPU="$(ps aux | grep qemu-system-aarch64 | grep -v grep | grep -o -- '-gpu [a-z_]*' | awk '{print $2}' | head -1)"
+log_step "current_gpu=${CURRENT_GPU:-none}  required=$GPU_MODE"
+if [[ -n "$CURRENT_GPU" && "$CURRENT_GPU" != "$GPU_MODE" ]]; then
+  echo "GPU mismatch: running=$CURRENT_GPU required=$GPU_MODE — recycling emulator" >&2
+  "$ADB" emu kill 2>/dev/null || true
+  sleep 2
+  pkill -9 -f qemu-system-aarch64 2>/dev/null || true
+  "$ADB" kill-server 2>/dev/null || true
+  "$ADB" start-server 2>/dev/null || true
+  sleep 2
+fi
+
+log_step "checking device availability"
+if ! "$ADB" devices | awk 'NR>1 && $2=="device"' | grep -q .; then
+  EMULATOR="$(find_sdk_tool emulator emulator)" || { echo "error: no device attached and emulator binary not found" >&2; exit 1; }
+  AVD="$("$EMULATOR" -list-avds | head -1)"
+  if [[ -z "$AVD" ]]; then
+    echo "error: no AVDs configured" >&2
+    exit 1
+  fi
+  # Reset adb server before cold-booting. A previously crashed emulator leaves
+  # the server in a bad state (still trying to reconnect to dead ports 5554/5555),
+  # which causes adb wait-for-device to hang indefinitely on the next boot.
+  log_step "resetting adb server before cold boot"
+  "$ADB" kill-server 2>/dev/null || true
+  "$ADB" start-server 2>/dev/null || true
+  sleep 1
+  log_step "booting emulator avd=$AVD gpu=$GPU_MODE"
+  # nohup + disown so the emulator outlives this script and is reused by later
+  # eval runs. -no-snapshot forces a clean cold boot (no stale snapshot to replay)
+  # and writes no multi-GB snapshot (disk is often tight).
+  nohup "$EMULATOR" -avd "$AVD" -no-boot-anim \
+    -gpu "$GPU_MODE" -no-snapshot </dev/null >/dev/null 2>&1 &
+  disown
+  EMULATOR_PID="$(pgrep -f "emulator.*-avd $AVD" | head -1)"
+  log_step "emulator booted pid=${EMULATOR_PID:-unknown}"
+  "$ADB" wait-for-device
+  log_step "adb device appeared — waiting for full boot"
+  until [[ "$("$ADB" shell getprop sys.boot_completed 2>/dev/null | tr -d '\r')" == "1" ]]; do
+    if [[ -n "$EMULATOR_PID" ]] && ! kill -0 "$EMULATOR_PID" 2>/dev/null; then
+      echo "error: emulator pid=$EMULATOR_PID crashed during boot (check DiagnosticReports for qemu SIGABRT)" >&2
+      exit 1
+    fi
+    sleep 2
+  done
+  log_step "emulator fully booted"
+else
+  EMULATOR_PID="$(pgrep -f qemu-system-aarch64 | head -1)"
+  log_step "reusing running emulator pid=${EMULATOR_PID:-unknown}"
+fi
+
+# --android makes the CLI install Expo Go on the device when missing. Pin the
+# packager hostname to 127.0.0.1 so every exp:// URL goes through the adb
+# reverse tunnel - the default LAN-IP URL is often unreachable from the
+# emulator (host firewall) and leaves a stuck experience in Expo Go's task
+# stack that hijacks later deep links.
+# dev-build: expo run:android prebuilds + native-compiles a dev client, installs
+# and launches it, and starts Metro — no Expo Go involved.
+if [[ "$RUNNER" == "dev-build" ]]; then
+  log_step "starting expo run:android port=$PORT"
+  # Pass --port so Metro binds to $PORT and the health-check curl matches.
+  (cd "$PROJECT_PATH" && exec env -u CI REACT_NATIVE_PACKAGER_HOSTNAME=127.0.0.1 bunx expo run:android --port "$PORT") </dev/null >"$LOG" 2>&1 &
+else
+  log_step "starting expo start --android port=$PORT"
+  (cd "$PROJECT_PATH" && exec env -u CI REACT_NATIVE_PACKAGER_HOSTNAME=127.0.0.1 bunx expo start --port "$PORT" --android) </dev/null >"$LOG" 2>&1 &
+fi
+METRO_PID=$!
+# APP_PKG is set later (dev-build only) but must be in scope for cleanup.
+APP_PKG=""
+cleanup() {
+  kill "$METRO_PID" 2>/dev/null
+  lsof -ti "tcp:$PORT" -sTCP:LISTEN 2>/dev/null | xargs kill -9 2>/dev/null || true
+  wait "$METRO_PID" 2>/dev/null
+}
+trap cleanup EXIT
+# Resolve the app package name up front so the cleanup trap has it even on
+# early exit. app.json is a fixture source file always present before Metro starts.
+if [[ "$RUNNER" == "dev-build" ]]; then
+  APP_PKG="$(node -e "try{var a=require('$PROJECT_PATH/app.json');console.log((a.expo&&a.expo.android&&a.expo.android.package)||'com.exposkilleval.fixture')}catch(e){console.log('com.exposkilleval.fixture')}" 2>/dev/null || echo 'com.exposkilleval.fixture')"
+  log_step "app package=$APP_PKG"
+fi
+
+DEADLINE=$((SECONDS + BUNDLE_TIMEOUT))
+STATUS=0
+
+log_step "waiting for Metro on port=$PORT"
+until curl -sf "http://localhost:$PORT/status" >/dev/null; do
+  if ! kill -0 "$METRO_PID" 2>/dev/null; then
+    echo "error: Metro (pid=$METRO_PID) exited before coming up. Log tail:" >&2
+    tail -40 "$LOG" >&2
+    exit 1
+  fi
+  if emulator_alive; then : ; elif [[ -n "$EMULATOR_PID" ]]; then
+    echo "error: emulator pid=$EMULATOR_PID crashed while waiting for Metro" >&2
+    tail -40 "$LOG" >&2
+    exit 1
+  fi
+  if ((SECONDS > DEADLINE)); then
+    echo "error: timed out after ${BUNDLE_TIMEOUT}s waiting for Metro. Log tail:" >&2
+    tail -40 "$LOG" >&2
+    exit 1
+  fi
+  sleep 2
+done
+log_step "Metro is up"
+# Expo Go only: wait for it to install (the CLI may still be installing it).
+if [[ "$RUNNER" != "dev-build" ]]; then
+  log_step "waiting for Expo Go install on device"
+  until "$ADB" shell pm list packages 2>/dev/null | grep -q host.exp.exponent; do
+    if ((SECONDS > DEADLINE)); then
+      echo "error: Expo Go was not installed on the device. Log tail:" >&2
+      tail -40 "$LOG" >&2
+      exit 1
+    fi
+    sleep 2
+  done
+fi
+# Both runners: route the bundle over the adb reverse tunnel.
+log_step "setting up adb reverse tunnel port=$PORT"
+"$ADB" reverse "tcp:$PORT" "tcp:$PORT"
+if [[ "$RUNNER" == "dev-build" ]]; then
+  # expo run:android launches the dev client BEFORE Metro is ready, so it shows
+  # "Cannot connect to Expo CLI". Now that Metro + adb-reverse are both up,
+  # force-restart via explicit MainActivity to avoid the "Open in X?" dialog
+  # that an implicit VIEW intent triggers. expo run:android --port embeds the
+  # Metro URL in the build, so the dev client auto-connects on relaunch.
+  if [[ -n "$APP_PKG" ]]; then
+    log_step "launching dev-build app pkg=$APP_PKG"
+    "$ADB" shell am force-stop "$APP_PKG" 2>/dev/null || true
+    sleep 2
+    "$ADB" shell am start -W -n "${APP_PKG}/.MainActivity" >/dev/null 2>&1 || true
+  fi
+else
+  log_step "launching Expo Go exp://127.0.0.1:$PORT"
+  # Force-stop so a stale experience from a previous run can't be resumed, then
+  # give the system a moment to finish task cleanup - launching immediately
+  # after force-stop races it and the new activity gets torn down too.
+  "$ADB" shell am force-stop host.exp.exponent
+  sleep 2
+  "$ADB" shell am start -W -a android.intent.action.VIEW -d "exp://127.0.0.1:$PORT" >/dev/null
+fi
+
+log_step "waiting for JS bundle"
+until grep -q 'Bundled' "$LOG"; do
+  if ! kill -0 "$METRO_PID" 2>/dev/null; then
+    echo "error: Metro (pid=$METRO_PID) exited before bundling. Log tail:" >&2
+    tail -40 "$LOG" >&2
+    STATUS=1
+    break
+  fi
+  if emulator_alive; then : ; elif [[ -n "$EMULATOR_PID" ]]; then
+    echo "error: emulator pid=$EMULATOR_PID crashed while waiting for JS bundle" >&2
+    tail -40 "$LOG" >&2
+    STATUS=1
+    break
+  fi
+  if ((SECONDS > DEADLINE)); then
+    echo "error: timed out after ${BUNDLE_TIMEOUT}s waiting for bundle. Log tail:" >&2
+    tail -40 "$LOG" >&2
+    STATUS=1
+    break
+  fi
+  sleep 2
+done
+
+log_step "bundle done — settling ${SETTLE}s"
+sleep "$SETTLE"
+log_step "taking screenshot"
+"$ADB" exec-out screencap -p >"$OUT" || STATUS=1
+# Raw emulator captures are ~2800px tall and overflow the eval viewer's
+# window. Downscale to a review/grading-friendly size.
+if [[ -s "$OUT" ]]; then
+  sips -Z "${EXPO_SKILL_EVAL_MAX_DIM:-600}" "$OUT" >/dev/null 2>&1 || true
+fi
+log_step "done  status=$STATUS  screenshot=$OUT"
+echo "screenshot: $OUT"
+echo "metro log: $LOG"
+exit "$STATUS"

package/bundled-skills/expo-skill-eval/scripts/snapshot-ios.sh

@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+# Run an Expo app in Expo Go on an iOS simulator and capture a screenshot.
+#
+# Usage: snapshot-ios.sh <project-path> <output-png> [port] [settle-seconds]
+#
+# Env: EXPO_SKILL_EVAL_RUNNER = expo-go (default) | dev-build — dev-build runs the
+#   app via `expo run:ios` (native dev-client compile, slow) instead of Expo Go.
+#
+# Boots the newest available iPhone simulator if none is booted, starts
+# Metro with `expo start --ios` (auto-installs Expo Go on the simulator and
+# opens the app), waits for the bundle to load, settles, screenshots, and
+# tears Metro down. The Metro log is saved next to the screenshot as
+# <name>.metro.log. On bundle failure it still captures a best-effort
+# screenshot (the error screen is grading evidence) and exits 1.
+set -uo pipefail
+
+PROJECT_PATH="${1:?usage: snapshot-ios.sh <project-path> <output-png> [port] [settle-seconds]}"
+OUT="${2:?usage: snapshot-ios.sh <project-path> <output-png> [port] [settle-seconds]}"
+PORT="${3:-8081}"
+SETTLE="${4:-8}"
+RUNNER="${EXPO_SKILL_EVAL_RUNNER:-expo-go}"   # expo-go | dev-build
+if [[ "$RUNNER" == "dev-build" ]]; then
+  BUNDLE_TIMEOUT="${EXPO_SKILL_EVAL_BUNDLE_TIMEOUT:-900}"   # native compile is slow
+else
+  BUNDLE_TIMEOUT="${EXPO_SKILL_EVAL_BUNDLE_TIMEOUT:-240}"
+fi
+LOG="${OUT%.png}.metro.log"
+
+mkdir -p "$(dirname "$OUT")"
+
+# Free the port up front in case a previous run (or a crash) left Metro on it,
+# so `expo start` binds the port we expect instead of silently incrementing.
+# Runs inside this script, so the eval harness never needs ad-hoc lsof/kill.
+lsof -ti "tcp:$PORT" -sTCP:LISTEN 2>/dev/null | xargs kill -9 2>/dev/null || true
+
+if ! xcrun simctl list devices booted | grep -q '(Booted)'; then
+  UDID="$(xcrun simctl list devices available | grep 'iPhone' | tail -1 | grep -oE '[0-9A-F-]{36}')"
+  if [[ -z "$UDID" ]]; then
+    echo "error: no available iPhone simulator found" >&2
+    exit 1
+  fi
+  echo "booting simulator $UDID..." >&2
+  xcrun simctl boot "$UDID"
+  open -a Simulator
+  xcrun simctl bootstatus "$UDID"
+fi
+
+# expo-go: --ios installs Expo Go when missing; its own open is unreliable if a
+# previous experience is foregrounded, so we relaunch deterministically below.
+# dev-build: expo run:ios prebuilds + native-compiles a dev client, installs and
+# launches it on the simulator, and starts Metro — no Expo Go involved.
+if [[ "$RUNNER" == "dev-build" ]]; then
+  # Pass --port so Metro binds to $PORT and the health-check curl matches.
+  (cd "$PROJECT_PATH" && exec env -u CI bunx expo run:ios --port "$PORT") </dev/null >"$LOG" 2>&1 &
+else
+  (cd "$PROJECT_PATH" && exec env -u CI REACT_NATIVE_PACKAGER_HOSTNAME=127.0.0.1 bunx expo start --port "$PORT" --ios) </dev/null >"$LOG" 2>&1 &
+fi
+METRO_PID=$!
+cleanup() {
+  kill "$METRO_PID" 2>/dev/null
+  lsof -ti "tcp:$PORT" -sTCP:LISTEN 2>/dev/null | xargs kill -9 2>/dev/null || true
+  wait "$METRO_PID" 2>/dev/null
+}
+trap cleanup EXIT
+
+DEADLINE=$((SECONDS + BUNDLE_TIMEOUT))
+STATUS=0
+
+until curl -sf "http://localhost:$PORT/status" >/dev/null; do
+  if ! kill -0 "$METRO_PID" 2>/dev/null || ((SECONDS > DEADLINE)); then
+    echo "error: Metro did not come up. Log tail:" >&2
+    tail -40 "$LOG" >&2
+    exit 1
+  fi
+  sleep 2
+done
+# Expo Go only: wait for Expo Go to install, then relaunch the experience
+# deterministically. dev-build skips this — expo run:ios launches the dev client.
+if [[ "$RUNNER" != "dev-build" ]]; then
+  until xcrun simctl get_app_container booted host.exp.Exponent >/dev/null 2>&1; do
+    if ((SECONDS > DEADLINE)); then
+      echo "error: Expo Go was not installed on the simulator. Log tail:" >&2
+      tail -40 "$LOG" >&2
+      exit 1
+    fi
+    sleep 2
+  done
+  # Terminate any running Expo Go so a stale experience can't stay foregrounded,
+  # then open the project URL explicitly.
+  xcrun simctl terminate booted host.exp.Exponent 2>/dev/null || true
+  sleep 2
+  xcrun simctl openurl booted "exp://127.0.0.1:$PORT"
+fi
+
+until grep -q 'Bundled' "$LOG"; do
+  if ! kill -0 "$METRO_PID" 2>/dev/null; then
+    echo "error: Metro exited before bundling. Log tail:" >&2
+    tail -40 "$LOG" >&2
+    STATUS=1
+    break
+  fi
+  if ((SECONDS > DEADLINE)); then
+    echo "error: timed out after ${BUNDLE_TIMEOUT}s waiting for bundle. Log tail:" >&2
+    tail -40 "$LOG" >&2
+    STATUS=1
+    break
+  fi
+  sleep 2
+done
+
+# dev-build: after bundling, relaunch the app so it is reliably in the
+# foreground. Use xcrun simctl launch (not openurl) to avoid the "Open in X?"
+# confirmation dialog that xcrun simctl openurl triggers on the iOS Simulator.
+# expo run:ios embeds the Metro URL in the build, so the dev client auto-connects.
+if [[ "$RUNNER" == "dev-build" ]]; then
+  BUNDLE_ID="$(node -e "try{var a=require('$PROJECT_PATH/app.json');console.log((a.expo&&a.expo.ios&&a.expo.ios.bundleIdentifier)||'com.exposkilleval.fixture')}catch(e){console.log('com.exposkilleval.fixture')}" 2>/dev/null || echo 'com.exposkilleval.fixture')"
+  xcrun simctl terminate booted "$BUNDLE_ID" 2>/dev/null || true
+  sleep 2
+  xcrun simctl launch --terminate-running-process booted "$BUNDLE_ID" >/dev/null 2>&1 || true
+  sleep 3
+fi
+
+sleep "$SETTLE"
+xcrun simctl io booted screenshot "$OUT" || STATUS=1
+# Raw simulator captures are 3x retina (~2600px tall) and overflow the eval
+# viewer's window. Downscale to a review/grading-friendly size.
+if [[ -f "$OUT" ]]; then
+  sips -Z "${EXPO_SKILL_EVAL_MAX_DIM:-600}" "$OUT" >/dev/null 2>&1 || true
+fi
+echo "screenshot: $OUT"
+echo "metro log: $LOG"
+exit "$STATUS"

package/bundled-skills/expo-skill-eval/scripts/snapshot-web.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+# Run an Expo app's web target and capture a browser screenshot. (Phase 4 -
+# only for eval cases whose runtime.platforms includes "web".)
+#
+# Usage: snapshot-web.sh <project-path> <output-png> [port] [settle-seconds]
+#
+# Starts Metro with `expo start --web`, waits for the dev server to respond,
+# then screenshots with Playwright's CLI at an iPhone-ish viewport. The first
+# run downloads Chromium via `bunx playwright install chromium`.
+set -uo pipefail
+
+PROJECT_PATH="${1:?usage: snapshot-web.sh <project-path> <output-png> [port] [settle-seconds]}"
+OUT="${2:?usage: snapshot-web.sh <project-path> <output-png> [port] [settle-seconds]}"
+PORT="${3:-8081}"
+SETTLE="${4:-8}"
+SERVER_TIMEOUT="${EXPO_SKILL_EVAL_BUNDLE_TIMEOUT:-240}"
+LOG="${OUT%.png}.metro.log"
+
+mkdir -p "$(dirname "$OUT")"
+
+# Free the port up front in case a previous run (or a crash) left Metro on it,
+# so `expo start` binds the port we expect instead of silently incrementing.
+# Runs inside this script, so the eval harness never needs ad-hoc lsof/kill.
+lsof -ti "tcp:$PORT" -sTCP:LISTEN 2>/dev/null | xargs kill -9 2>/dev/null || true
+
+bunx playwright install chromium >/dev/null 2>&1 || true
+
+(cd "$PROJECT_PATH" && exec env -u CI bunx expo start --port "$PORT" --web) </dev/null >"$LOG" 2>&1 &
+METRO_PID=$!
+cleanup() {
+  kill "$METRO_PID" 2>/dev/null
+  pkill -f "expo start --port $PORT" 2>/dev/null
+  wait "$METRO_PID" 2>/dev/null
+}
+trap cleanup EXIT
+
+DEADLINE=$((SECONDS + SERVER_TIMEOUT))
+until curl -sf "http://localhost:$PORT" >/dev/null; do
+  if ! kill -0 "$METRO_PID" 2>/dev/null || ((SECONDS > DEADLINE)); then
+    echo "error: web dev server did not come up. Log tail:" >&2
+    tail -40 "$LOG" >&2
+    exit 1
+  fi
+  sleep 2
+done
+
+STATUS=0
+bunx playwright screenshot \
+  --browser chromium \
+  --viewport-size "390,844" \
+  --wait-for-timeout "$((SETTLE * 1000))" \
+  "http://localhost:$PORT" "$OUT" || STATUS=1
+
+# Keep parity with the device snapshot scripts: cap the largest dimension so
+# screenshots fit the eval viewer's window.
+if [[ -f "$OUT" ]]; then
+  sips -Z "${EXPO_SKILL_EVAL_MAX_DIM:-600}" "$OUT" >/dev/null 2>&1 || true
+fi
+echo "screenshot: $OUT"
+echo "metro log: $LOG"
+exit "$STATUS"

package/bundled-skills/expo-skill-feedback/SKILL.md

@@ -0,0 +1,87 @@
+---
+name: expo-skill-feedback
+description: 'Submit feedback on an Expo skill—or Expo itself—and control bundled anonymous usage telemetry (off by default / opt-in). Submit feedback with: npx --yes submit-expo-feedback@latest "ACTIONABLE_FEEDBACK". Optionally add either or both: --category "CATEGORY" and --subject "SUBJECT". Replace the uppercase placeholders before running. Use when a skill was useful, confusing, broken, missing context, or worth improving; when Expo, Expo CLI, EAS CLI, docs, or MCP worked well or fell short; when an AI agent repeatedly failed, got stuck, or needed the user to take over an Expo task (report it as an eval candidate); or when the user explicitly asks to enable or disable telemetry (tracking), check its status, or understand what it collects.'
+---
+
+# Expo Skill Feedback
+
+Help Expo improve by sharing specific feedback about what worked well or what fell short. Feedback
+submission is independent of usage telemetry and does not require enabling it.
+
+## Submit feedback
+
+```bash
+npx --yes submit-expo-feedback@latest "<ACTIONABLE_FEEDBACK>"
+```
+
+Add either optional flag independently when it provides useful context:
+
+```bash
+npx --yes submit-expo-feedback@latest --category "<CATEGORY>" --subject "<SUBJECT>" "<ACTIONABLE_FEEDBACK>"
+```
+
+`--category` defaults to `unknown`, and `--subject` may be omitted when there is no specific target.
+When including them, choose the values that most precisely identify what the feedback is about:
+
+| Category | Subject |
+| --- | --- |
+| `skills` | Exact skill name from its frontmatter, such as `expo-router` |
+| `docs` | Full Expo documentation URL |
+| `mcp` | Exact MCP tool name used |
+| `expo-cli` | Full Expo CLI command, such as `npx expo install` |
+| `eas-cli` | Full EAS CLI command, such as `eas build` |
+| `evals` | Expo package or command the failed task involves, else a capability phrase, such as `expo-router` or `eas build` |
+| `unknown` | Concise Expo product, package, feature, or other topic |
+
+In the final argument, say what helped and why, or provide the relevant context, expected behavior,
+and what happened instead. Do not include secrets, source code, personal data, long prompts, or stack traces.
+
+## Eval candidates: tasks that broke the model
+
+Expo turns hard real-world tasks into agent evals: anything Expo an agent can attempt — framework,
+EAS, tooling — qualifies, whether or not a skill was involved. The signal worth sending is a task an
+AI agent could not complete cleanly despite real effort: several failed attempts, a build or screen
+that never worked, or the user stepping in to fix it manually. Never submit quick slips the agent
+corrected itself, more than one candidate per session, or a task already reported.
+
+When such a failure happens — or the user says a model failed at an Expo task — show the user the exact
+submission you intend to send and get approval; the Task field must describe the Expo-technical
+shape of the task, never the user's product or business context. Without a user to approve it
+(headless or CI runs), do not submit. Then run from the failing app's directory (the CLI attaches
+the SDK and package versions, agent harness, and platform automatically) with `--category evals`,
+`--subject` naming the Expo package or command involved (a capability phrase only when no single
+package fits), and this structure in the final argument:
+
+```text
+Task: <what was asked, self-contained>.
+Expected: <observable success criteria>.
+Actual: <what the agent did instead>.
+Wrong approach: <the specific mistake, such as a wrong API, hallucinated prop, or bad pattern>.
+Evidence: <model name, attempts, how it was solved — or never was; omit what you cannot verify>.
+```
+
+A good candidate is solvable (eventually done or clearly doable), verifiable (success is
+observable), and specific. Mention only environment details the CLI cannot see, such as other key
+packages or a freshly created app. Describe code; do not paste it. If the command fails with an
+error naming `evals` as an invalid category, resend once with `--category unknown` and the same
+subject prefixed `eval-candidate: `; on any other error, do not resend.
+
+## Usage telemetry
+
+Automatic skill usage telemetry is separate from feedback and **off by default**. Never enable it
+without an explicit user request. Use the following script with `--on`, `--off`, or `--status`:
+
+```bash
+node "${CLAUDE_PLUGIN_ROOT}/skills/expo-skill-feedback/scripts/telemetry.cjs" --status
+```
+
+Environment equivalents are `EXPO_SKILLS_TELEMETRY=1` to enable and
+`EXPO_SKILLS_TELEMETRY=0` or `DO_NOT_TRACK=1` to disable. CI never sends telemetry.
+
+## Submitting Feedback
+If you encounter errors, misleading or outdated information in this skill, report it so Expo can improve:
+```bash
+npx --yes submit-expo-feedback@latest --category skills --subject "expo-skill-feedback" "<actionable feedback>"
+```
+Only submit when you have something specific and actionable to report. Include as much relevant context as possible.
+If an AI agent repeatedly failed or the user had to take over an Expo task, load the expo-skill-feedback skill and follow its eval-candidate flow instead of reusing the command above.

package/bundled-skills/expo-skill-feedback/agents/openai.yaml

@@ -0,0 +1,4 @@
+interface:
+  display_name: "Expo Skill Feedback"
+  short_description: "Help Expo improve with feedback, eval candidates, and usage telemetry"
+  default_prompt: "Use $expo-skill-feedback to help Expo improve by submitting feedback, reporting an Expo task an AI agent failed at as an eval candidate, or controlling Expo skills telemetry."