@aarwitz/tapp 0.16.5 → 0.17.0-rc.10

package/scripts/ci-gate.sh

@@ -3,13 +3,15 @@
 #
 # Boots a simulator if needed, installs the app build under test, runs the autonomous exploration
 # harness, replays every committed Flow (deterministic E2E tests), diffs the findings against a
-# stored baseline, writes a GitHub Actions step summary, and exits non-zero when the gate fails
-# (new high/critical findings vs. baseline, or any failed Flow). Wrapped by ../action.yml for
-# GitHub Actions; equally usable from any other CI or locally.
+# stored baseline, writes a GitHub Actions step summary, and exits with the public outcome contract:
+# pass 0, deterministic fail 1, infrastructure/usage error 2, inconclusive evidence 3. Absolute
+# blockers and reviewed Flow/Scenario/Contract failures are enforced even without a baseline;
+# baseline comparisons additionally catch new high/critical regressions. Wrapped by ../action.yml
+# for GitHub Actions; equally usable from any other CI or locally.
 #
 # Usage:
 #   scripts/ci-gate.sh [--platform ios] --app <path/to/App.app> [--bundle-id <com.example.app>]
-#   scripts/ci-gate.sh --platform android --apk <path/to/app.apk> --app-id <com.example.app>
+#   scripts/ci-gate.sh --platform android --apk <path/to/app.apk> --app-id <com.example.app> [--serial <adb-serial>]
 #   scripts/ci-gate.sh --platform web [--url <http(s)://owned-app>]
 #                      # omit --url with --project-dir to detect/build/start/stop one owned web target
 #                      # bundle id is detected from the .app when omitted
@@ -25,17 +27,18 @@
 #                      [--pr-plan-out <file.json>] # persist the reviewable selection plan
 #                      [--baseline <file.json>]   # prior report to diff against (skipped if absent)
 #                      [--target-key <stable-id>] # isolates target-specific baselines in monorepos
-#                      [--fail-on gate|blocked|any]  # gate policy (default gate; see ci-report.js)
+#                      [--fail-on gate|absolute|any]  # gate policy (default gate; see ci-report.js)
 #                      [--json-out <file.json>]   # write the full report (use as the next baseline)
 #                      [--md-out <file.md>]       # write the rendered markdown report (for a PR comment)
 #                      [--device <name>]          # simulator device to boot if none is (default "iPhone 16 Pro")
+#                      [--serial <adb-serial>]    # Android emulator/device (default: first connected device)
 #
 # The app must be a SIMULATOR build (xcodebuild ... -destination 'generic/platform=iOS Simulator').
 set -uo pipefail
 ROOT="$(cd "$(dirname "$0")/.." && pwd)"
 
 usage() {
-  sed -n '2,30p' "$0" | sed 's/^# \{0,1\}//'
+  sed -n '2,/^set -uo pipefail$/p' "$0" | sed '$d; s/^# \{0,1\}//'
 }
 
 PLATFORM="ios" APP_PATH="" BUNDLE_ID="" APK_PATH="" APP_ID="" URL="" WEB_TARGET="" TARGET_KEY="" SERIAL="" ACTIONS=40 TIMEOUT=600 FLOWS="" SCENARIOS="" CONTRACTS="" PROJECT_DIR="" BASELINE="" FAIL_ON="gate" JSON_OUT="" MD_OUT="" DEVICE="iPhone 16 Pro" PR_BASE="" PR_HEAD="HEAD" CHANGED_FILES_FILE="" PR_PLAN_OUT=""
@@ -74,7 +77,7 @@ done
 [[ "$PLATFORM" == "ios" || "$PLATFORM" == "android" || "$PLATFORM" == "web" ]] || { echo "❌ --platform must be ios|android|web" >&2; exit 2; }
 [[ "$ACTIONS" =~ ^[1-9][0-9]*$ ]] || { echo "❌ --actions must be a positive integer" >&2; exit 2; }
 [[ "$TIMEOUT" =~ ^[1-9][0-9]*$ ]] || { echo "❌ --timeout must be a positive integer" >&2; exit 2; }
-[[ "$FAIL_ON" == "gate" || "$FAIL_ON" == "blocked" || "$FAIL_ON" == "any" ]] || { echo "❌ --fail-on must be gate|blocked|any" >&2; exit 2; }
+[[ "$FAIL_ON" == "gate" || "$FAIL_ON" == "absolute" || "$FAIL_ON" == "any" ]] || { echo "❌ --fail-on must be gate|absolute|any" >&2; exit 2; }
 if [[ -n "$PROJECT_DIR" ]]; then
   [[ -d "$PROJECT_DIR" ]] || { echo "❌ Project directory not found: $PROJECT_DIR" >&2; exit 2; }
   PROJECT_DIR="$(cd "$PROJECT_DIR" && pwd)"
@@ -82,7 +85,6 @@ fi
 TAPP_PROJECT_ARTIFACTS=""
 if [[ -n "$PROJECT_DIR" ]]; then
   [[ -d "$PROJECT_DIR/.tapp" ]] && TAPP_PROJECT_ARTIFACTS="$PROJECT_DIR/.tapp"
-  [[ -z "$TAPP_PROJECT_ARTIFACTS" && -d "$PROJECT_DIR/.autotap" ]] && TAPP_PROJECT_ARTIFACTS="$PROJECT_DIR/.autotap"
 fi
 [[ -z "$FLOWS" && -n "$TAPP_PROJECT_ARTIFACTS" && -d "$TAPP_PROJECT_ARTIFACTS/flows" ]] && FLOWS="$TAPP_PROJECT_ARTIFACTS/flows/*.yml"
 [[ "$PLATFORM" == "web" && -z "$SCENARIOS" && -n "$TAPP_PROJECT_ARTIFACTS" && -d "$TAPP_PROJECT_ARTIFACTS/scenarios" ]] && SCENARIOS="$TAPP_PROJECT_ARTIFACTS/scenarios/*.yml"
@@ -274,7 +276,7 @@ xcrun simctl install "$UDID" "$APP_PATH" || { echo "❌ simctl install failed
 # ── Autonomous exploration (quick-capture builds the harness itself if needed).
 step "Explore ($ACTIONS actions, ${TIMEOUT}s watchdog)"
 set +e
-CAPTURE_ROOT="${TAPP_HOME:-${AUTOTAP_HOME:-$ROOT}}/captures"
+CAPTURE_ROOT="${TAPP_HOME:-$ROOT}/captures"
 mkdir -p "$CAPTURE_ROOT"
 CAPTURE_DIR="$(mktemp -d "$CAPTURE_ROOT/ci.XXXXXX")"
 TAPP_CAPTURE_DIR="$CAPTURE_DIR" OCQA_PR_TARGET_JSON="$IOS_PR_TARGET_JSON" "$ROOT/scripts/quick-capture.sh" explore "$BUNDLE_ID" --actions "$ACTIONS" --timeout "$TIMEOUT"

package/scripts/flow_ai_judge.py

@@ -26,7 +26,7 @@ def _ctx():
 
 
 SSL_CTX = _ctx()
-MODEL = os.environ.get("TAPP_VISION_MODEL") or os.environ.get("AUTOTAP_VISION_MODEL", "claude-haiku-4-5-20251001")
+MODEL = os.environ.get("TAPP_VISION_MODEL", "claude-haiku-4-5-20251001")
 SYSTEM = (
     "You are a QA test oracle. You are shown one iOS app screenshot and a CLAIM the test asserts about "
     "it. Decide if the claim is TRUE of what is actually visible. Be strict and literal — only pass if "

package/scripts/platform-gate.js

@@ -37,7 +37,10 @@ for (let i = 2; i < process.argv.length; i += 1) {
   else if (key === "--json-out") args.jsonOut = value;
   else if (key === "--md-out") args.mdOut = value;
   else if (key === "--pr-plan") args.prPlan = value;
-  else throw new Error(`Unknown argument: ${key}`);
+  else {
+    console.error(`Unknown argument: ${key}`);
+    process.exit(2);
+  }
 }
 if (!["web", "android"].includes(args.platform)) throw new Error("--platform must be web|android");
 if (args.platform === "web" && !args.url && !args.projectDir) throw new Error("Web gate requires --url or --project-dir for managed build/start");
@@ -94,7 +97,7 @@ if (args.prPlan) {
 }
 
 let managedRuntime = null;
-let exitCode = 1;
+let exitCode = 2;
 try {
   if (args.platform === "web" && !args.url) {
     const started = await startManagedWebTarget({
@@ -106,7 +109,7 @@ try {
     if (started.error) {
       console.error(`❌ ${started.error}`);
       if (started.details?.remediation) console.error(`   ${started.details.remediation}`);
-      process.exitCode = 1;
+      exitCode = 2;
     } else {
       managedRuntime = started;
       args.url = started.url;
@@ -114,7 +117,7 @@ try {
     }
   }
   if (args.platform === "web" && !args.url) {
-    exitCode = 1;
+    exitCode = 2;
   } else {
     const qa = args.platform === "web"
       ? await runQaWeb({ url: args.url, maxActions: args.actions, timeout: args.timeout, testEmail: process.env.OCQA_TEST_EMAIL, testPassword: process.env.OCQA_TEST_PASSWORD, seedTargets: prExplorationTargets })
@@ -122,7 +125,7 @@ try {
           testEmail: process.env.OCQA_TEST_EMAIL, testPassword: process.env.OCQA_TEST_PASSWORD, seedTargets: prExplorationTargets });
     if (qa.error) {
       console.error(`❌ ${qa.error}`);
-      exitCode = 1;
+      exitCode = 2;
     } else {
       const captureDir = qa.structured.capture.path;
       const markers = path.join(captureDir, "ocqa-markers.txt");
@@ -177,6 +180,9 @@ try {
       exitCode = report.status ?? 1;
     }
   }
+} catch (error) {
+  console.error(`❌ ${error?.message || String(error)}`);
+  exitCode = 2;
 } finally {
   if (managedRuntime) {
     await stopManagedWebTarget(managedRuntime);

package/scripts/quick-capture.sh

@@ -19,7 +19,7 @@ PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
 # accidentally reuse stale markers). TAPP_HOME redirects all writable output otherwise —
 # set by the `tapp` CLI when running as an installed npm package, where the package dir
 # must stay read-only. Unset (repo dev flow), everything lands in the repo as before.
-TAPP_RUNTIME_HOME="${TAPP_HOME:-${AUTOTAP_HOME:-}}"
+TAPP_RUNTIME_HOME="${TAPP_HOME:-}"
 if [[ -n "${TAPP_CAPTURE_DIR:-}" ]]; then
   CAPTURE_DIR="$TAPP_CAPTURE_DIR"
 elif [[ -n "$TAPP_RUNTIME_HOME" ]]; then
@@ -262,58 +262,92 @@ case "$MODE" in
     echo "Running autonomous exploration ($MAX_ACTIONS actions, timeout: ${EXPLORE_TIMEOUT}s, app: $APP_BUNDLE)..."
     ensure_harness_built "$SIM_NAME"
 
-    # Start video recording in background
-    cleanup_stale_recorders "$UDID"
-    RECORD_PID=""
-    if xcrun simctl io "$UDID" recordVideo --codec=h264 "$CAPTURE_DIR/exploration.mov" & then
-      RECORD_PID=$!
-    fi
-    sleep 0.5
-    if ! kill -0 "$RECORD_PID" 2>/dev/null; then
-      echo "WARNING: Could not start simulator video recording. Continuing without video." >&2
-      RECORD_PID=""
+    # XCUITest can block inside `app.launch()` until the outer watchdog when the target dies in its
+    # initializer, producing a misleading timeout after minutes. For an ordinary unconfigured
+    # launch, probe the target directly first and verify that the returned PID survives a short
+    # settle window. (Configured launch args/env skip this probe because simctl would not reproduce
+    # that launch contract.) The harness still performs its own fresh launch for healthy apps.
+    PREFLIGHT_CRASH=0
+    PREFLIGHT_OUTPUT=""
+    if [[ -z "${OCQA_APP_LAUNCH_ARGS_JSON:-}" && -z "${OCQA_APP_LAUNCH_ENV_JSON:-}" ]]; then
+      xcrun simctl terminate "$UDID" "$APP_BUNDLE" >/dev/null 2>&1 || true
+      PREFLIGHT_OUTPUT="$(xcrun simctl launch "$UDID" "$APP_BUNDLE" 2>&1 || true)"
+      PREFLIGHT_PID="$(echo "$PREFLIGHT_OUTPUT" | sed -n 's/.*: \([0-9][0-9]*\)$/\1/p' | tail -1)"
+      if [[ -n "$PREFLIGHT_PID" ]]; then
+        sleep 2
+        if ! kill -0 "$PREFLIGHT_PID" 2>/dev/null; then PREFLIGHT_CRASH=1; fi
+      fi
+      xcrun simctl terminate "$UDID" "$APP_BUNDLE" >/dev/null 2>&1 || true
     fi
 
-    # Run exploration with watchdog timeout to avoid silent hangs.
     local_output_file="$CAPTURE_DIR/harness-output.txt"
-    run_harness_test "testAutonomousExploration" "$SIM_NAME" "$APP_BUNDLE" "$MAX_ACTIONS" "$EXPLORE_TIMEOUT" > "$local_output_file" 2>&1 &
-    HARNESS_PID=$!
-
-    START_TS=$(date +%s)
     TIMED_OUT=0
-    while kill -0 "$HARNESS_PID" 2>/dev/null; do
-      NOW_TS=$(date +%s)
-      ELAPSED=$((NOW_TS - START_TS))
-      if [[ "$ELAPSED" -ge "$EXPLORE_TIMEOUT" ]]; then
-        TIMED_OUT=1
-        kill -TERM "$HARNESS_PID" 2>/dev/null || true
-        sleep 2
-        kill -KILL "$HARNESS_PID" 2>/dev/null || true
-        break
+    RECORD_PID=""
+    if [[ "$PREFLIGHT_CRASH" -eq 1 ]]; then
+      OUTPUT="OCQA_ISSUE:{\"type\":\"crash\",\"severity\":\"critical\",\"title\":\"App crashed during launch preflight\",\"screen\":\"Launch\",\"step\":0}
+OCQA_COMPLETE:{\"actions\":0,\"states\":0,\"issues\":1,\"screens\":\"\",\"outcome\":\"launch_crash\"}"
+      printf '%s\n%s\n' "$PREFLIGHT_OUTPUT" "$OUTPUT" > "$local_output_file"
+      echo "WARNING: Target process exited during launch preflight; recorded a crash instead of waiting for the exploration timeout." >&2
+    else
+
+      # Start video recording in background
+      cleanup_stale_recorders "$UDID"
+      if xcrun simctl io "$UDID" recordVideo --codec=h264 "$CAPTURE_DIR/exploration.mov" & then
+        RECORD_PID=$!
+      fi
+      sleep 0.5
+      if ! kill -0 "$RECORD_PID" 2>/dev/null; then
+        echo "WARNING: Could not start simulator video recording. Continuing without video." >&2
+        RECORD_PID=""
       fi
-      sleep 2
-    done
 
-    wait "$HARNESS_PID" 2>/dev/null || true
-    OUTPUT="$(cat "$local_output_file" 2>/dev/null || true)"
+      # Run exploration with watchdog timeout to avoid silent hangs.
+      run_harness_test "testAutonomousExploration" "$SIM_NAME" "$APP_BUNDLE" "$MAX_ACTIONS" "$EXPLORE_TIMEOUT" > "$local_output_file" 2>&1 &
+      HARNESS_PID=$!
+
+      START_TS=$(date +%s)
+      while kill -0 "$HARNESS_PID" 2>/dev/null; do
+        NOW_TS=$(date +%s)
+        ELAPSED=$((NOW_TS - START_TS))
+        if [[ "$ELAPSED" -ge "$EXPLORE_TIMEOUT" ]]; then
+          TIMED_OUT=1
+          kill -TERM "$HARNESS_PID" 2>/dev/null || true
+          sleep 2
+          kill -KILL "$HARNESS_PID" 2>/dev/null || true
+          break
+        fi
+        sleep 2
+      done
+
+      wait "$HARNESS_PID" 2>/dev/null || true
+      OUTPUT="$(cat "$local_output_file" 2>/dev/null || true)"
 
-    # Stop recording
-    if [[ -n "$RECORD_PID" ]]; then
-      kill -INT "$RECORD_PID" 2>/dev/null || true
-      wait "$RECORD_PID" 2>/dev/null || true
+      # Stop recording
+      if [[ -n "$RECORD_PID" ]]; then
+        kill -INT "$RECORD_PID" 2>/dev/null || true
+        wait "$RECORD_PID" 2>/dev/null || true
+      fi
+      sleep 1
     fi
-    sleep 1
 
     # Parse OCQA_ markers
     echo "$OUTPUT" | grep "^OCQA_" > "$CAPTURE_DIR/ocqa-markers.txt" || true
     if [[ "$TIMED_OUT" -eq 1 ]]; then
-      echo "OCQA_ISSUE:{\"type\":\"explore_timeout\",\"severity\":\"high\",\"title\":\"Exploration timed out\",\"timeoutSeconds\":$EXPLORE_TIMEOUT}" >> "$CAPTURE_DIR/ocqa-markers.txt"
-      echo "OCQA_COMPLETE:{\"actions\":0,\"states\":0,\"issues\":1,\"screens\":\"\",\"timedOut\":true,\"timeoutSeconds\":$EXPLORE_TIMEOUT}" >> "$CAPTURE_DIR/ocqa-markers.txt"
-      echo "WARNING: Exploration hit timeout after ${EXPLORE_TIMEOUT}s" >&2
+      # The caller's wall-clock budget expiring means the evidence is partial; it is not proof that
+      # the app itself is slow or hung. Preserve observed coverage in the terminal marker and let
+      # report.js classify the run as inconclusive instead of inventing an app finding.
+      TIMED_ACTIONS="$(echo "$OUTPUT" | sed -n 's/^OCQA_PROGRESS:{"action":\([0-9][0-9]*\).*/\1/p' | tail -1)"
+      TIMED_STATES="$(echo "$OUTPUT" | sed -n 's/^OCQA_PROGRESS:.*"states":\([0-9][0-9]*\).*/\1/p' | tail -1)"
+      TIMED_ISSUES="$(echo "$OUTPUT" | grep -c '^OCQA_ISSUE:' || true)"
+      TIMED_ACTIONS="${TIMED_ACTIONS:-0}"
+      TIMED_STATES="${TIMED_STATES:-0}"
+      TIMED_ISSUES="${TIMED_ISSUES:-0}"
+      echo "OCQA_COMPLETE:{\"actions\":$TIMED_ACTIONS,\"states\":$TIMED_STATES,\"issues\":$TIMED_ISSUES,\"screens\":\"\",\"timedOut\":true,\"timeoutSeconds\":$EXPLORE_TIMEOUT}" >> "$CAPTURE_DIR/ocqa-markers.txt"
+      echo "WARNING: Exploration reached its ${EXPLORE_TIMEOUT}s time budget; evidence is partial" >&2
     fi
     echo "$OUTPUT" > "$CAPTURE_DIR/full-output.txt"
 
-    COMPLETE_LINE=$(echo "$OUTPUT" | grep "OCQA_COMPLETE" | tail -1)
+    COMPLETE_LINE=$(grep "OCQA_COMPLETE" "$CAPTURE_DIR/ocqa-markers.txt" | tail -1 || true)
     if [[ -n "$COMPLETE_LINE" ]]; then
       echo ""
       echo "Exploration complete: $COMPLETE_LINE"

package/scripts/run-flow.sh

@@ -26,7 +26,7 @@ UDID="$(xcrun simctl list devices booted -j 2>/dev/null | python3 -c 'import sys
 [ -z "$UDID" ] && { echo "❌ No booted simulator."; exit 2; }
 # When run through the installed `tapp` CLI, the harness cache lives under TAPP_HOME.
 XCTR=""
-TAPP_RUNTIME_HOME="${TAPP_HOME:-${AUTOTAP_HOME:-}}"
+TAPP_RUNTIME_HOME="${TAPP_HOME:-}"
 [ -n "$TAPP_RUNTIME_HOME" ] && XCTR="$(find "$TAPP_RUNTIME_HOME/harness-derived/Build/Products" -name '*.xctestrun' 2>/dev/null | head -1)"
 [ -z "$XCTR" ] && XCTR="$(find "$HOME/Library/Developer/Xcode/DerivedData/OCQAHarness-"*/Build/Products -name '*.xctestrun' 2>/dev/null | head -1)"
 [ -z "$XCTR" ] && XCTR="$(find /tmp/tapp-harness-derived/Build/Products -name '*.xctestrun' 2>/dev/null | head -1)"