@aarwitz/tapp 0.17.0-rc.9 → 0.17.1

package/scripts/quick-capture.sh

@@ -23,9 +23,9 @@ TAPP_RUNTIME_HOME="${TAPP_HOME:-}"
 if [[ -n "${TAPP_CAPTURE_DIR:-}" ]]; then
   CAPTURE_DIR="$TAPP_CAPTURE_DIR"
 elif [[ -n "$TAPP_RUNTIME_HOME" ]]; then
-  CAPTURE_DIR="$TAPP_RUNTIME_HOME/captures/$(date +%Y%m%d-%H%M%S)"
+  CAPTURE_DIR="$TAPP_RUNTIME_HOME/captures/ios-$(date +%Y%m%d-%H%M%S)"
 else
-  CAPTURE_DIR="$PROJECT_ROOT/captures/$(date +%Y%m%d-%H%M%S)"
+  CAPTURE_DIR="$PROJECT_ROOT/captures/ios-$(date +%Y%m%d-%H%M%S)"
 fi
 if [[ -n "$TAPP_RUNTIME_HOME" ]]; then
   HARNESS_DERIVED="$TAPP_RUNTIME_HOME/harness-derived"
@@ -163,13 +163,28 @@ run_harness_test() {
   \"OCQA_PR_TARGET\": ${OCQA_PR_TARGET_JSON}"
   fi
 
+  # Visual evidence must begin at the first settled target-app frame, never while Xcode is
+  # installing/launching the test runner or SpringBoard is selecting the app. The harness writes
+  # the ready file after the target is foregrounded + stable, then briefly waits for the host's
+  # recording-start acknowledgement so the first autonomous action cannot race ahead of video.
+  local visual_ready_line=""
+  if [[ -n "${OCQA_VISUAL_READY_PATH:-}" ]]; then
+    visual_ready_line=",
+  \"OCQA_VISUAL_READY_PATH\": \"${OCQA_VISUAL_READY_PATH}\""
+  fi
+  local recording_started_line=""
+  if [[ -n "${OCQA_RECORDING_STARTED_PATH:-}" ]]; then
+    recording_started_line=",
+  \"OCQA_RECORDING_STARTED_PATH\": \"${OCQA_RECORDING_STARTED_PATH}\""
+  fi
+
   cat > /tmp/ocqa-run-config.json << CONF
 {
   "OCQA_BUNDLE_ID": "$bundle_id",
   "OCQA_MAX_ACTIONS": "$max_actions",
   "OCQA_TIMEOUT_SECONDS": "$timeout_secs",
   "OCQA_TEST_EMAIL": "${OCQA_TEST_EMAIL:-qa@example.com}",
-  "OCQA_TEST_PASSWORD": "${OCQA_TEST_PASSWORD:-Tapp123!}"$interactive_line$overrides_line$launch_args_line$launch_env_line$login_steps_line$pr_target_line
+  "OCQA_TEST_PASSWORD": "${OCQA_TEST_PASSWORD:-Tapp123!}"$interactive_line$overrides_line$launch_args_line$launch_env_line$login_steps_line$pr_target_line$visual_ready_line$recording_started_line
 }
 CONF
 
@@ -290,23 +305,38 @@ OCQA_COMPLETE:{\"actions\":0,\"states\":0,\"issues\":1,\"screens\":\"\",\"outcom
       echo "WARNING: Target process exited during launch preflight; recorded a crash instead of waiting for the exploration timeout." >&2
     else
 
-      # Start video recording in background
+      # Prepare the foreground/settled handshake before starting the harness. Recording begins
+      # only after that handshake; this excludes build/install/SpringBoard footage without using a
+      # brittle fixed trim duration. A caller such as VS Code may provide its own ready path to
+      # reveal a preview at the same authoritative boundary.
       cleanup_stale_recorders "$UDID"
-      if xcrun simctl io "$UDID" recordVideo --codec=h264 "$CAPTURE_DIR/exploration.mov" & then
-        RECORD_PID=$!
-      fi
-      sleep 0.5
-      if ! kill -0 "$RECORD_PID" 2>/dev/null; then
-        echo "WARNING: Could not start simulator video recording. Continuing without video." >&2
-        RECORD_PID=""
-      fi
+      VISUAL_READY_PATH="${OCQA_VISUAL_READY_PATH:-$CAPTURE_DIR/visual-ready}"
+      RECORDING_STARTED_PATH="$CAPTURE_DIR/recording-started"
+      rm -f "$VISUAL_READY_PATH" "$RECORDING_STARTED_PATH"
+      export OCQA_VISUAL_READY_PATH="$VISUAL_READY_PATH"
+      export OCQA_RECORDING_STARTED_PATH="$RECORDING_STARTED_PATH"
 
       # Run exploration with watchdog timeout to avoid silent hangs.
       run_harness_test "testAutonomousExploration" "$SIM_NAME" "$APP_BUNDLE" "$MAX_ACTIONS" "$EXPLORE_TIMEOUT" > "$local_output_file" 2>&1 &
       HARNESS_PID=$!
 
       START_TS=$(date +%s)
+      RECORDING_ATTEMPTED=0
       while kill -0 "$HARNESS_PID" 2>/dev/null; do
+        if [[ "$RECORDING_ATTEMPTED" -eq 0 && -f "$VISUAL_READY_PATH" ]]; then
+          RECORDING_ATTEMPTED=1
+          if xcrun simctl io "$UDID" recordVideo --codec=h264 "$CAPTURE_DIR/exploration.mov" & then
+            RECORD_PID=$!
+          fi
+          sleep 0.25
+          if [[ -z "$RECORD_PID" ]] || ! kill -0 "$RECORD_PID" 2>/dev/null; then
+            echo "WARNING: Could not start simulator video recording. Continuing without video." >&2
+            RECORD_PID=""
+          fi
+          # Always release the bounded harness wait. The missing video remains explicit in the
+          # report; exploration itself must not hang merely because recording was unavailable.
+          : > "$RECORDING_STARTED_PATH"
+        fi
         NOW_TS=$(date +%s)
         ELAPSED=$((NOW_TS - START_TS))
         if [[ "$ELAPSED" -ge "$EXPLORE_TIMEOUT" ]]; then
@@ -316,7 +346,7 @@ OCQA_COMPLETE:{\"actions\":0,\"states\":0,\"issues\":1,\"screens\":\"\",\"outcom
           kill -KILL "$HARNESS_PID" 2>/dev/null || true
           break
         fi
-        sleep 2
+        if [[ "$RECORDING_ATTEMPTED" -eq 0 ]]; then sleep 0.1; else sleep 1; fi
       done
 
       wait "$HARNESS_PID" 2>/dev/null || true
@@ -327,6 +357,7 @@ OCQA_COMPLETE:{\"actions\":0,\"states\":0,\"issues\":1,\"screens\":\"\",\"outcom
         kill -INT "$RECORD_PID" 2>/dev/null || true
         wait "$RECORD_PID" 2>/dev/null || true
       fi
+      rm -f "$RECORDING_STARTED_PATH"
       sleep 1
     fi
 

package/scripts/run-flow.sh

@@ -40,6 +40,14 @@ TOKEN="$(date +%s)"
 CFG="/tmp/ocqa-flow-$TOKEN.json"
 AI_RESP="/tmp/ocqa-flow-ai-$TOKEN.json"
 AI_DIR="/tmp/ocqa-flow-ai-$TOKEN"
+EVIDENCE_DIR="${TAPP_FLOW_EVIDENCE_DIR:-/tmp/tapp-flow-ios-$TOKEN}"
+RESULT_BUNDLE="$EVIDENCE_DIR/result.xcresult"
+mkdir -p "$EVIDENCE_DIR"
+case "$FLOW" in
+  *.json) FLOW_EVIDENCE_SOURCE="$EVIDENCE_DIR/flow-source.json" ;;
+  *) FLOW_EVIDENCE_SOURCE="$EVIDENCE_DIR/flow-source.yml" ;;
+esac
+cp "$FLOW" "$FLOW_EVIDENCE_SOURCE"
 python3 - "$CFG" "$APP" "$FLOW_JSON" "$AI_RESP" "$AI_DIR" <<'PY'
 import json, os, sys
 cfg, app, flow_json, ai_resp, ai_dir = sys.argv[1:6]
@@ -82,9 +90,12 @@ fi
 
 TEST_RUNNER_OCQA_CONFIG_PATH="$CFG" xcodebuild test-without-building \
   -xctestrun "$XCTR" -destination "platform=iOS Simulator,id=$UDID" \
-  -only-testing:"OCQAHarnessUITests/ExplorerTests/testReplayFlow" > "$LOG" 2>&1
+  -only-testing:"OCQAHarnessUITests/ExplorerTests/testReplayFlow" \
+  -resultBundlePath "$RESULT_BUNDLE" > "$LOG" 2>&1
 [ -n "$RESPONDER_PID" ] && { kill "$RESPONDER_PID" 2>/dev/null; wait "$RESPONDER_PID" 2>/dev/null; }
 
+cp "$LOG" "$EVIDENCE_DIR/flow.log"
+python3 "$ROOT/scripts/flow_lib.py" report --json "$LOG" > "$EVIDENCE_DIR/flow-report.json"
 echo ""
 python3 "$ROOT/scripts/flow_lib.py" report "$LOG"
 exit $?

package/skills/tapp/SKILL.md

@@ -0,0 +1,95 @@
+---
+name: tapp
+description: Use Tapp to see, drive, explore, and verify real application surfaces on iOS simulators, Android emulators/devices, or the web. Use when a user asks an agent to test an app or UI change, find bugs, inspect or screenshot a screen, exercise a journey, create a replayable flow, gather release evidence, or run the deterministic Tapp gate. Also use when the user mentions Tapp, @aarwitz/tapp, tapp_* tools, .tapp artifacts, or asks whether agent-authored UI actually works.
+---
+
+# Tapp
+
+Use Tapp as the app's hands and eyes. Work on the real UI surface and show evidence; do not claim a
+screen or journey works from source inspection alone.
+
+## Choose the smallest operation
+
+| Intent | Operation |
+|---|---|
+| See or screenshot one screen | `open` / `tapp_open_app` |
+| Inspect controls on the current screen | `tree` / `tapp_ui_tree` |
+| Reach a named screen/control | `focus` / `tapp_focus` (source + observed UI Map fast path) |
+| Drive a specific journey | MCP session start → focus or act → end |
+| Find bugs autonomously | `explore` / `tapp_explore` |
+| Preserve a journey | record and save a Flow; replay it deterministically |
+| Decide whether a merge passes policy | `ci`; exploration never decides this |
+
+Prefer connected `tapp_*` MCP tools when available: they keep interactive sessions alive and return
+screenshots inline. Otherwise run `npx -y @aarwitz/tapp@latest` from the app repository. Do not require MCP,
+an account, an API key, or a global install for the core workflow.
+
+## Start source-connected
+
+When the user asks for a general first test of a repository:
+
+1. If `.tapp/application-model.json` exists, run `npx -y @aarwitz/tapp@latest explore`.
+2. Otherwise run `npx -y @aarwitz/tapp@latest init . --explore` or call `tapp_init` with
+   `{operation:"explore", projectDir:"."}`.
+3. If Tapp returns `target-selection-required`, present its actual choices and ask the user to pick.
+   Never guess among multiple targets. Re-run with the selected platform/target exactly as Tapp
+   instructs.
+4. If a prerequisite is missing, call `tapp_health` when MCP is connected or run
+   `npx -y @aarwitz/tapp@latest doctor`, apply only the stated remediation that is in scope, and
+   retry once.
+
+For a focused request, use the requested target directly rather than forcing repository onboarding.
+Targets may be a repository path, Xcode container, `.app`, iOS bundle id, APK plus Android app id,
+or owned HTTP(S) URL. Never explore a third-party web property without authorization: exploration
+clicks and types.
+
+## Navigate like a source-connected expert
+
+When the user names a screen, control, or UI condition, do not discover the app one screenshot at a
+time. Start from the repository source, then use Tapp's observed navigation evidence:
+
+1. With MCP, pass the exact request as `focus` to `tapp_session_start`, or call `tapp_focus` in an
+   active session. Without MCP, run `npx -y @aarwitz/tapp@latest focus "<exact request>" [target]`.
+2. Tapp searches owned source, reconciles the likely surface with `.tapp/ui-map.json`, and executes
+   the shortest runtime-observed route in one call. Read its final tree before visual assertions.
+3. If Tapp returns source evidence but no replayable route, inspect the cited file/line and relevant
+   router/navigation source. Do not wander blindly or invent a path; ground the map or drive only a
+   route supported by that source evidence.
+
+A fresh repository needs one grounding exploration before `focus` can replay a route. Source can
+locate an unobserved surface, but it never authorizes unobserved taps.
+
+Source establishes intent and location; the real UI establishes behavior. A URL-only target has no
+source advantage and correctly falls back to runtime observation.
+
+## Observe honestly
+
+Exploration returns findings, coverage, evidence, and `inconclusive`; it does not return a score or
+ship verdict. Report:
+
+- target and platform;
+- screens/actions and whether coverage was conclusive;
+- deterministic versus advisory finding counts;
+- each important finding and its evidence/report path;
+- what Tapp explicitly did not check.
+
+If `inconclusive: true`, explain the blocker. A login wall or missing test data is not a pass. Ask for
+credentials or launch configuration instead of rerunning blindly. Do not infer content accuracy,
+privacy, brand consistency, or business guarantees from a generic crawl; those require a reviewed
+Flow, Scenario, contract, verifier, or human review.
+
+When a screenshot path is printed, open it with the client's image-reading tool before describing
+the screen. For web, use `--watch` when the human wants to follow Tapp's controlled browser. For iOS,
+point the human to the report's exploration recording when available.
+
+## Drive safely
+
+After the focused fast path, read returned `elements[]` before any remaining action, target accessibility
+ids or visible labels, check `hittable`, tap a field before typing, and wait for navigation or async
+content. Use coordinates only as a last resort. End the session when finished.
+
+Do not edit the app merely because testing found a defect unless the user also asked for a fix. State
+what the evidence proves and what remains untested.
+
+Read [references/commands.md](references/commands.md) only when exact CLI/MCP syntax, Flow replay,
+credentials, or platform prerequisites are needed.

package/skills/tapp/agents/openai.yaml

@@ -0,0 +1,4 @@
+interface:
+  display_name: "Tapp"
+  short_description: "Test real iOS, Android, and web apps"
+  default_prompt: "Use $tapp to test this app on its real UI surface and report evidence-backed findings."

package/skills/tapp/references/commands.md

@@ -0,0 +1,105 @@
+# Tapp command reference
+
+## Plain CLI
+
+Run from the application repository. `[target]` is optional when Tapp can read the repository model
+or detect one unambiguous target.
+
+```bash
+npx -y @aarwitz/tapp@latest init . --explore
+npx -y @aarwitz/tapp@latest explore [target]
+npx -y @aarwitz/tapp@latest focus "Save storefront settings visible above keyboard" [target]
+npx -y @aarwitz/tapp@latest open [target]
+npx -y @aarwitz/tapp@latest tree [target] --json
+npx -y @aarwitz/tapp@latest shot
+npx -y @aarwitz/tapp@latest report latest
+npx -y @aarwitz/tapp@latest doctor
+```
+
+Platform examples:
+
+```bash
+# Web: Tapp may build/start/stop a source target; --watch is human-visible.
+npx -y @aarwitz/tapp@latest explore --platform web --target website --watch
+npx -y @aarwitz/tapp@latest explore https://staging.example.com
+
+# iOS: source repo, .app, or bundle id.
+npx -y @aarwitz/tapp@latest explore MyApp.xcodeproj --platform ios
+npx -y @aarwitz/tapp@latest open com.example.MyApp --platform ios
+
+# Android: app id is required; APK is optional if already installed.
+npx -y @aarwitz/tapp@latest explore app-debug.apk --platform android --app-id com.example.app
+```
+
+Focused web evidence can perform one semantic interaction and wait for async content:
+
+```bash
+npx -y @aarwitz/tapp@latest open https://example.com --tap "Not now" --wait-for "Dashboard"
+```
+
+## MCP mapping
+
+- `tapp_init`: inspect or initialize a source repository; `operation:"explore"` prepares and explores.
+- `tapp_build`: build and install an iOS app without needing its bundle id first.
+- `tapp_open_app`: launch and return a screen summary plus inline screenshot.
+- `tapp_focus`: source-locate a named screen/control and execute the shortest observed route in the active session.
+- `tapp_ui_tree` / `tapp_screenshot`: inspect the current real surface.
+- `tapp_session_start` → `tapp_session_act` → `tapp_session_end`: drive one persistent journey.
+- `tapp_explore`: autonomous iOS, Android, or web exploration; observation only.
+- `tapp_flow_save` / `tapp_flow_run`: save a driven journey and replay it deterministically.
+- `tapp_release_contract`: validate, compile, or run a reviewed business guarantee.
+- `tapp_ci_setup`: create a target-scoped baseline or reviewable CI installation.
+
+An iOS no-bundle-id MCP path is `tapp_build {projectDir:"."}` followed by `tapp_explore` with the
+returned `bundleId`. Android uses `androidAppId` and optional `apkPath`; web uses `url` or
+source-connected `tapp_init`.
+
+## Interactive session loop
+
+```text
+tapp_session_start {appBundleId:"com.example.app", focus:"Save storefront settings visible above keyboard", projectDir:"."}
+tapp_focus         {query:"Save storefront settings visible above keyboard"}
+tapp_session_act   {action:"tap", id:"Email"}
+tapp_session_act   {action:"type", text:"qa@example.com"}
+tapp_session_act   {action:"tap", id:"Password"}
+tapp_session_act   {action:"type", text:"..."}
+tapp_session_act   {action:"tap", id:"Sign In"}
+tapp_session_act   {action:"wait", text:"Home", timeoutMs:10000}
+tapp_screenshot
+tapp_session_end
+```
+
+Android sessions use `androidAppId`, optional `apkPath`, and the same action loop.
+
+## Credentials and test configuration
+
+For autonomous exploration, pass test-only values when authorized:
+
+- CLI: `--email`, `--password`, repeated `--launch-arg`, and JSON `--launch-env`.
+- MCP: `testEmail`, `testPassword`, `inputOverrides`, `appLaunchArgs`, `appLaunchEnv`, or explicit
+  `loginSteps`.
+
+Do not persist secrets in `.tapp/`. If the result reports input fields and no values were supplied,
+ask the user rather than pretending the explored surface was complete.
+
+## Replay and gating
+
+Flow YAML belongs under `.tapp/flows/` and can replay without a model or API key:
+
+```bash
+npx -y @aarwitz/tapp@latest flow run .tapp/flows/smoke.yml
+npx -y @aarwitz/tapp@latest ci
+```
+
+`tapp explore` observes. `tapp ci` applies versioned deterministic policy to evidence, selected
+Flows/Scenarios/contracts, coverage, and any target-scoped baseline. Its outcomes are `pass`, `fail`,
+or `inconclusive`; both `fail` and `inconclusive` block a merge.
+
+## Platform prerequisites
+
+- iOS: macOS, Xcode, and a booted simulator. First use builds a cached harness under `~/.tapp`.
+- Android: `adb` and a connected authorized emulator/device.
+- Web: Playwright and Chromium. If Tapp reports the browser missing, run
+  `npx playwright install chromium` and retry.
+
+Captures and screenshots are stored under `~/.tapp/captures/` and `~/.tapp/shots/`.