@aarwitz/tapp 0.15.0

package/scripts/validation-matrix.sh

@@ -0,0 +1,146 @@
+#!/usr/bin/env bash
+# AutoTap harness accuracy benchmark.
+#
+# Runs the exploration harness against DemoApp (the fixture corpus) and asserts expected outcomes:
+#   - the run completes (no crash/abort),
+#   - a minimum number of distinct screens are reached (breadth),
+#   - each detection fixture, WHEN its screen is reached, produces its expected finding (accuracy),
+#   - DemoApp never produces a `crash` finding (no false alarms).
+# Then, if RestaurantDemo is installed, a freeze regression: with interactive input ENABLED it must
+# NOT pause for input on its (credential-free) item screens — guarding the multi-minute stall fix.
+#
+# Exploration is non-deterministic, so fixture assertions are conditional on the screen being
+# reached. Use --actions to raise the budget for fuller coverage. Exits non-zero on any failure.
+#
+# Prereqs: a booted simulator, the harness built (scripts/deploy-and-build.sh --harness), and
+# DemoApp installed (ruby generate-demoapp-xcodeproj.rb + xcodebuild + simctl install).
+set -uo pipefail
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+ACTIONS="${1:-70}"
+BUNDLE="com.autotap.demoapp"
+
+UDID="$(xcrun simctl list devices booted -j 2>/dev/null | python3 -c 'import sys,json; d=json.load(sys.stdin); print(next((x["udid"] for v in d["devices"].values() for x in v if x.get("state")=="Booted"), ""))')"
+[ -z "$UDID" ] && { echo "❌ No booted simulator. Boot one first."; exit 2; }
+
+XCTR="$(find "$HOME/Library/Developer/Xcode/DerivedData/OCQAHarness-"*/Build/Products -name '*.xctestrun' 2>/dev/null | head -1)"
+# Fallback: harness built via deploy-and-build.sh --harness lands here
+[ -z "$XCTR" ] && XCTR="$(find /tmp/autotap-harness-derived/Build/Products -name '*.xctestrun' 2>/dev/null | head -1)"
+[ -z "$XCTR" ] && { echo "❌ Harness not built. Run: scripts/deploy-and-build.sh --harness"; exit 2; }
+
+xcrun simctl get_app_container "$UDID" "$BUNDLE" >/dev/null 2>&1 || { echo "❌ DemoApp not installed on $UDID."; exit 2; }
+
+CFG="/tmp/ocqa-validation-matrix.json"
+cat > "$CFG" <<JSON
+{ "OCQA_BUNDLE_ID": "$BUNDLE", "OCQA_MAX_ACTIONS": "$ACTIONS", "OCQA_TIMEOUT_SECONDS": "600" }
+JSON
+
+LOG="/tmp/ocqa-validation-matrix.log"
+echo "Running harness vs DemoApp ($ACTIONS actions, sim $UDID)…"
+TEST_RUNNER_OCQA_CONFIG_PATH="$CFG" xcodebuild test-without-building \
+  -xctestrun "$XCTR" -destination "platform=iOS Simulator,id=$UDID" \
+  -only-testing:"OCQAHarnessUITests/ExplorerTests/testAutonomousExploration" > "$LOG" 2>&1
+
+python3 - "$LOG" <<'PY'
+import sys, re, json
+log = open(sys.argv[1]).read()
+screens, issues, complete = set(), {}, None
+for line in log.splitlines():
+    m = re.search(r'OCQA_STATE:(\{.*\})', line)
+    if m:
+        try: screens.add(json.loads(m.group(1)).get("screen",""))
+        except: pass
+    m = re.search(r'OCQA_ISSUE:(\{.*\})', line)
+    if m:
+        try:
+            d = json.loads(m.group(1)); issues.setdefault(d.get("screen",""), set()).add(d.get("type"))
+        except: pass
+    m = re.search(r'OCQA_COMPLETE:(\{.*\})', line)
+    if m:
+        try: complete = json.loads(m.group(1))
+        except: pass
+screens.discard("")
+
+# fixture screen -> expected finding type (asserted only if the screen was reached)
+FIXTURES = [
+    ("System Status",   "error_surface"),        # always-on error text
+    ("System Status",   "unresponsive_element"),  # dead Retry button on error screen
+    ("Live Feed",       "app_hang"),              # perpetual spinner
+    ("Dashboard",       "unresponsive_element"),  # dead Sync Now button
+    ("Changelog",       "error_surface"),         # error message below the fold
+    ("Update Profile",  "error_surface"),         # inline validation error after form submit
+    ("Saved Reports",   "unresponsive_element"),  # dead Add Report button on empty state
+    ("Queued Tasks",    "unresponsive_element"),  # swipe-only rows invisible to tap-based exploration
+]
+MIN_SCREENS = 12
+
+rows, ok = [], True
+def check(name, passed, detail=""):
+    global ok
+    ok = ok and passed
+    rows.append((("PASS ✅" if passed else "FAIL ❌"), name, detail))
+
+check("run completed (no abort/crash)", complete is not None, "" if complete else "no OCQA_COMPLETE")
+check(f">= {MIN_SCREENS} distinct screens reached", len(screens) >= MIN_SCREENS, f"reached {len(screens)}")
+crashed = any("crash" in t for t in issues.values())
+check("no spurious 'crash' finding", not crashed)
+for screen, expected in FIXTURES:
+    if screen in screens:
+        check(f"[{screen}] → {expected}", expected in issues.get(screen, set()), f"found {sorted(issues.get(screen,set()))}")
+    else:
+        rows.append(("SKIP ⏭ ", f"[{screen}] → {expected}", "screen not reached this run"))
+
+print("\n=== AutoTap Validation Matrix ===")
+for status, name, detail in rows:
+    print(f"  {status}  {name}" + (f"   ({detail})" if detail else ""))
+print(f"\nScreens: {', '.join(sorted(screens))}")
+sys.exit(0 if ok else 1)
+PY
+RESULT=$?
+
+# ===== RestaurantDemo: interactive-input freeze regression =====
+# RestaurantDemo has form fields (e.g. "Special Instructions") but NO credential screens, so with
+# interactive input ENABLED it must NOT pause (OCQA_AWAIT_INPUT) — that pause once froze runs for
+# ~180s on item detail screens. Guards the credential-only gate + short fallback.
+RD_BUNDLE="com.autotap.restaurantdemo"
+RD_RESULT=0
+if xcrun simctl get_app_container "$UDID" "$RD_BUNDLE" >/dev/null 2>&1; then
+  RD_CFG="/tmp/ocqa-restaurant-regression.json"
+  rm -f /tmp/ocqa-rd-noresp.json
+  cat > "$RD_CFG" <<JSON
+{ "OCQA_BUNDLE_ID": "$RD_BUNDLE", "OCQA_MAX_ACTIONS": "20", "OCQA_TIMEOUT_SECONDS": "240", "OCQA_INTERACTIVE_INPUT": "1", "OCQA_INPUT_RESPONSE_PATH": "/tmp/ocqa-rd-noresp.json", "OCQA_INPUT_WAIT_TIMEOUT": "30" }
+JSON
+  RD_LOG="/tmp/ocqa-restaurant-regression.log"
+  echo ""; echo "Running RestaurantDemo freeze-regression (interactive input ON, $UDID)…"
+  TEST_RUNNER_OCQA_CONFIG_PATH="$RD_CFG" xcodebuild test-without-building \
+    -xctestrun "$XCTR" -destination "platform=iOS Simulator,id=$UDID" \
+    -only-testing:"OCQAHarnessUITests/ExplorerTests/testAutonomousExploration" > "$RD_LOG" 2>&1
+  python3 - "$RD_LOG" <<'PY'
+import sys, re
+log = open(sys.argv[1]).read()
+pauses = len(re.findall(r'OCQA_AWAIT_INPUT', log))
+screens = set(re.findall(r'OCQA_STATE:\{"screen":"([^"]*)"', log))
+completed = "OCQA_COMPLETE" in log
+ok = True
+def check(name, passed, detail=""):
+    global ok; ok = ok and passed
+    print(f"  {'PASS ✅' if passed else 'FAIL ❌'}  {name}" + (f"   ({detail})" if detail else ""))
+print("\n=== RestaurantDemo freeze regression ===")
+check("no interactive-input pause (0 OCQA_AWAIT_INPUT)", pauses == 0, f"{pauses} pause(s)")
+check("reached Menu", "Menu" in screens)
+check("reached a menu-item detail", any(s in screens for s in ("Bruschetta","Tiramisu","Charcuterie Board")), f"screens={sorted(screens)}")
+check("run completed", completed)
+sys.exit(0 if ok else 1)
+PY
+  RD_RESULT=$?
+else
+  echo ""; echo "⏭  RestaurantDemo not installed — skipping freeze regression"
+  echo "    (ruby generate-demo-named.rb RestaurantDemo + xcodebuild + simctl install to enable)"
+fi
+
+echo ""
+if [ $RESULT -eq 0 ] && [ "$RD_RESULT" -eq 0 ]; then
+  echo "✅ Validation matrix PASSED"; exit 0
+else
+  echo "❌ Validation matrix FAILED (DemoApp log: $LOG)"; exit 1
+fi

package/scripts/vision-fp-eval.sh

@@ -0,0 +1,206 @@
+#!/usr/bin/env bash
+# Vision false-positive eval — measure how often AutoTap's vision pass invents a defect on a clean
+# screen, BEFORE we default it on.
+#
+# The vision reviewer (VisionInspector.swift) is disabled by default. This runs the SAME prompt/model
+# over real captured screens (via scripts/vision_fp_probe.py) and tallies the visual findings. Our
+# corpus apps are standard SwiftUI (visually clean — their fixtures are logical/interaction bugs, not
+# visual ones), so on that corpus every finding is a candidate FALSE POSITIVE. A low flag rate here
+# is the evidence needed to default vision on.
+#
+# Usage:
+#   ANTHROPIC_API_KEY=... scripts/vision-fp-eval.sh                 # capture + review corpus apps
+#   ANTHROPIC_API_KEY=... scripts/vision-fp-eval.sh com.acme.app    # specific installed app(s)
+#   ANTHROPIC_API_KEY=... scripts/vision-fp-eval.sh --dir path/to/pngs   # review existing PNGs
+#   ACTIONS=40 scripts/vision-fp-eval.sh                            # exploration budget per app
+#
+# Prereqs (capture mode): a booted sim, the harness built (scripts/deploy-and-build.sh --harness),
+# apps installed. Writes captures/vision-fp-<timestamp>.json.
+set -uo pipefail
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+ACTIONS="${ACTIONS:-40}"
+TIMEOUT="${TIMEOUT:-420}"
+MAX_SCREENS="${MAX_SCREENS:-25}"   # mirrors ExplorationService.visionScreenBudget
+DEFAULT_APPS=(com.autotap.demoapp com.autotap.logindemo com.autotap.wizarddemo com.autotap.shopdemo com.autotap.restaurantdemo)
+
+[ -z "${ANTHROPIC_API_KEY:-}" ] && { echo "❌ ANTHROPIC_API_KEY not set."; exit 2; }
+
+TS="$(date +%Y%m%d-%H%M%S)"
+REPORT="$ROOT/captures/vision-fp-$TS.json"
+mkdir -p "$ROOT/captures"
+
+# ---- Direct-directory mode: review a folder of PNGs, no capture ----
+if [ "${1:-}" = "--dir" ]; then
+  DIR="${2:?--dir needs a path}"
+  PNGS=()  # macOS ships bash 3.2 (no mapfile) — read into the array portably.
+  while IFS= read -r p; do [ -n "$p" ] && PNGS+=("$p"); done < <(find "$DIR" -iname '*.png' | sort)
+  [ ${#PNGS[@]} -eq 0 ] && { echo "❌ No PNGs under $DIR"; exit 2; }
+  echo "Reviewing ${#PNGS[@]} screenshot(s) from $DIR ..."
+  python3 "$ROOT/scripts/vision_fp_probe.py" "${PNGS[@]}" > "$REPORT"
+  python3 -c '
+import sys, json
+d = json.load(open(sys.argv[1]))
+if d.get("errors"): print("⚠️  %d/%d calls FAILED (e.g. %s) — not clean screens." % (d["errors"], d["reviewed"], (d.get("first_error") or "")[:80]))
+print("reviewed=%s succeeded=%s flagged=%s findings=%s by_sev=%s fp_rate=%s" % (d["reviewed"], d["succeeded"], d["flagged"], d["findings_total"], d["by_severity"], d["fp_rate_screens"]))
+' "$REPORT"
+  echo "Report: $REPORT"
+  exit 0
+fi
+
+# ---- Capture mode: drive corpus apps through the harness, extract distinct screens, review ----
+APPS=("$@"); [ ${#APPS[@]} -eq 0 ] && APPS=("${DEFAULT_APPS[@]}")
+
+UDID="$(xcrun simctl list devices booted -j 2>/dev/null | python3 -c 'import sys,json; d=json.load(sys.stdin); print(next((x["udid"] for v in d["devices"].values() for x in v if x.get("state")=="Booted"), ""))')"
+[ -z "$UDID" ] && { echo "❌ No booted simulator. Boot one first."; exit 2; }
+XCTR="$(find "$HOME/Library/Developer/Xcode/DerivedData/OCQAHarness-"*/Build/Products -name '*.xctestrun' 2>/dev/null | head -1)"
+[ -z "$XCTR" ] && XCTR="$(find /tmp/autotap-harness-derived/Build/Products -name '*.xctestrun' 2>/dev/null | head -1)"
+[ -z "$XCTR" ] && XCTR="$(find /tmp/harness-build/Build/Products -name '*.xctestrun' 2>/dev/null | head -1)"
+[ -z "$XCTR" ] && { echo "❌ Harness not built. Run: scripts/deploy-and-build.sh --harness"; exit 2; }
+
+echo "Vision FP eval — $ACTIONS actions/app, sim $UDID"
+echo "Apps: ${APPS[*]}"
+echo ""
+
+ALL_PNGS=()
+TITLE_MAP="/tmp/vision-fp-titlemap-$TS.json"
+SUMMARY_MAP="/tmp/vision-fp-summarymap-$TS.json"   # basename -> a11y summary (mirrors runVisionPass)
+echo "{}" > "$TITLE_MAP"; echo "{}" > "$SUMMARY_MAP"
+
+for APP in "${APPS[@]}"; do
+  if ! xcrun simctl get_app_container "$UDID" "$APP" >/dev/null 2>&1; then
+    printf "  %-32s ⏭  not installed — skipping\n" "$APP"; continue
+  fi
+  CFG="/tmp/ocqa-visfp-$APP.json"
+  cat > "$CFG" <<JSON
+{ "OCQA_BUNDLE_ID": "$APP", "OCQA_MAX_ACTIONS": "$ACTIONS", "OCQA_TIMEOUT_SECONDS": "$TIMEOUT" }
+JSON
+  XCRESULT="/tmp/ocqa-visfp-$APP.xcresult"; rm -rf "$XCRESULT"
+  OUTDIR="/tmp/ocqa-visfp-shots-$APP"; rm -rf "$OUTDIR"; mkdir -p "$OUTDIR"
+  LOG="/tmp/ocqa-visfp-$APP.log"   # harness stdout; carries OCQA_STATE settled flags per action
+  printf "  %-32s exploring...\n" "$APP"
+  TEST_RUNNER_OCQA_CONFIG_PATH="$CFG" xcodebuild test-without-building \
+    -xctestrun "$XCTR" -destination "platform=iOS Simulator,id=$UDID" \
+    -resultBundlePath "$XCRESULT" \
+    -only-testing:"OCQAHarnessUITests/ExplorerTests/testAutonomousExploration" > "$LOG" 2>&1
+
+  xcrun xcresulttool export attachments --path "$XCRESULT" --output-path "$OUTDIR" >/dev/null 2>&1 || true
+
+  # Dedup to distinct SCREENS and pick ONE representative per title, PREFERRING a settled capture —
+  # mirrors ExplorationService.runVisionPass. The harness names shots "state_<action>_<Title>_<n>_
+  # <uuid>.png"; the action count joins to the "settled" flag on that step from the harness log
+  # (OCQA_STATE). Bounded by MAX_SCREENS. Kept apostrophe-free (heredoc nested in $()).
+  SELECTED="$(APP="$APP" OUTDIR="$OUTDIR" MAX_SCREENS="$MAX_SCREENS" TITLE_MAP="$TITLE_MAP" SUMMARY_MAP="$SUMMARY_MAP" LOG="$LOG" SETTLED_MODE="${SETTLED_MODE:-prefer}" python3 - <<'PY'
+import os, re, json, glob
+outdir = os.environ["OUTDIR"]; app = os.environ["APP"]
+maxs = int(os.environ["MAX_SCREENS"]); tmpath = os.environ["TITLE_MAP"]; smpath = os.environ["SUMMARY_MAP"]
+logpath = os.environ.get("LOG", ""); mode = os.environ.get("SETTLED_MODE", "prefer")
+
+# action -> settled(bool) and action -> a11y context (summary + full text inventory), parsed from
+# the harness log OCQA_STATE lines. Context mirrors ExplorationService.visionContext.
+settled_by_action = {}
+summary_by_action = {}
+if logpath and os.path.exists(logpath):
+    for line in open(logpath, errors="ignore"):
+        if line.startswith("OCQA_STATE:{"):
+            try:
+                o = json.loads(line[len("OCQA_STATE:"):])
+                if "action" not in o:
+                    continue
+                act = int(o["action"])
+                if "settled" in o:
+                    settled_by_action[act] = bool(o["settled"])
+                ctx = o.get("summary", "")
+                texts = [t for t in (o.get("atext") or []) if isinstance(t, str)]
+                if texts:
+                    ctx += ("\n" if ctx else "") + "Full visible text (from accessibility): " + " | ".join(texts)
+                if ctx:
+                    summary_by_action[act] = ctx
+            except Exception:
+                pass
+
+def parse_name(nm):
+    m = re.match(r"(?:final_)?state_(\d+)_(.+?)_\d+_[0-9A-Fa-f-]{8,}\.png$", nm)
+    if m:
+        return int(m.group(1)), m.group(2).replace("_", " ").strip()
+    if nm.startswith("final_state"):
+        return 10**9, "final state"
+    return -1, os.path.splitext(nm)[0]
+
+entries = []  # (path, title, action, settled)
+manifest = os.path.join(outdir, "manifest.json")
+if os.path.exists(manifest):
+    try:
+        data = json.load(open(manifest))
+        for test in data if isinstance(data, list) else []:
+            for att in test.get("attachments", []):
+                fn = att.get("exportedFileName"); nm = att.get("suggestedHumanReadableName") or fn or ""
+                if not (fn and fn.lower().endswith(".png")):
+                    continue
+                action, title = parse_name(nm)
+                st = settled_by_action.get(action, True)
+                entries.append((os.path.join(outdir, fn), title, action, st))
+    except Exception:
+        pass
+if not entries:
+    entries = [(p, "Screen %d" % (i + 1), -1, True)
+               for i, p in enumerate(sorted(glob.glob(os.path.join(outdir, "*.png"))))]
+
+# One representative per title. mode=prefer upgrades an unsettled pick to a settled one.
+best = {}  # title -> [path, settled, order, action]
+order = 0
+for path, title, action, st in sorted(entries, key=lambda e: e[2] if e[2] >= 0 else 10**9):
+    if title.lower() in ("unknown", ""):
+        continue
+    order += 1
+    if title in best:
+        if mode == "prefer" and (not best[title][1]) and st:
+            best[title][0] = path; best[title][1] = True; best[title][3] = action
+    else:
+        best[title] = [path, st, order, action]
+
+chosen = sorted(best.items(), key=lambda kv: kv[1][2])[:maxs]
+tmap = json.load(open(tmpath)) if os.path.exists(tmpath) else {}
+smap = json.load(open(smpath)) if os.path.exists(smpath) else {}
+paths = []
+for title, (path, st, _o, action) in chosen:
+    paths.append(path)
+    tmap[os.path.basename(path)] = ("%s: %s" % (app, title)) + ("" if st else " [unsettled]")
+    smap[os.path.basename(path)] = summary_by_action.get(action, "")
+json.dump(tmap, open(tmpath, "w"))
+json.dump(smap, open(smpath, "w"))
+print("\n".join(paths))
+PY
+)"
+  CNT=0
+  while IFS= read -r line; do [ -n "$line" ] && { ALL_PNGS+=("$line"); CNT=$((CNT+1)); }; done <<< "$SELECTED"
+  printf "  %-32s %s distinct screen(s) captured\n" "$APP" "$CNT"
+done
+
+[ ${#ALL_PNGS[@]} -eq 0 ] && { echo "❌ No screenshots captured — is the harness built and are apps installed?"; exit 2; }
+
+echo ""
+echo "Reviewing ${#ALL_PNGS[@]} distinct screen(s) with the vision model..."
+python3 "$ROOT/scripts/vision_fp_probe.py" --title-map "$TITLE_MAP" --summary-map "$SUMMARY_MAP" "${ALL_PNGS[@]}" > "$REPORT"
+
+python3 -c '
+import sys, json
+d = json.load(open(sys.argv[1]))
+print("")
+print("  reviewed screens : %d" % d["reviewed"])
+print("  succeeded calls  : %d" % d["succeeded"])
+if d.get("errors"):
+    print("  ⚠️  FAILED calls  : %d  (e.g. %s)" % (d["errors"], (d.get("first_error") or "")[:70]))
+    print("      Failed calls are NOT clean screens — fix connectivity and re-run before trusting the rate.")
+print("  flagged screens  : %d" % d["flagged"])
+print("  total findings   : %d  (by severity: %s)" % (d["findings_total"], d["by_severity"]))
+rate = d["fp_rate_screens"]
+print("  screen flag rate : %s  (over succeeded calls; on a visually-clean corpus this ~= false-positive rate)" % ("n/a" if rate is None else "%.1f%%" % (100*rate)))
+if d["flagged"]:
+    print("\n  flagged (inspect each — real defect or false positive?):")
+    for img in d["images"]:
+        for f in img["findings"]:
+            print("    - [%s/%s] %s — %s :: %s" % (f["severity"], f["category"], img["title"], f["title"], f["detail"][:80]))
+' "$REPORT"
+echo ""
+echo "Report: $REPORT"

package/scripts/vision_escalation_responder.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+"""Host-side responder for in-loop vision escalation — lets script runs (coverage-eval.sh) exercise
+OCQA_VISION_QUERY without the AutoTap desktop app.
+
+Tails the harness log for OCQA_VISION_QUERY markers, sends the screenshot to the vision model with
+the SAME next-move prompt AnthropicVisionInspector.decideNextAction uses (kept in sync with
+VisionInspector.swift), and writes the {requestId, action, x, y} decision to the response file the
+harness polls. On any failure it writes action="none" so the harness falls back to its bail path
+promptly instead of blocking to its timeout.
+
+Usage (run alongside xcodebuild, kill when the run ends):
+    ANTHROPIC_API_KEY=... python3 vision_escalation_responder.py <harness.log> <response_path>
+"""
+import base64
+import json
+import os
+import ssl
+import sys
+import time
+import urllib.request
+
+
+def _ssl_context():
+    try:
+        import certifi
+        return ssl.create_default_context(cafile=certifi.where())
+    except Exception:
+        return ssl.create_default_context()
+
+
+_SSL_CTX = _ssl_context()
+MODEL = os.environ.get("AUTOTAP_VISION_MODEL", "claude-haiku-4-5-20251001")
+
+# Mirror of AnthropicVisionInspector.navPrompt (VisionInspector.swift).
+NAV_PROMPT = (
+    "You are guiding an automated iOS UI explorer that is STUCK on the screen shown — either its "
+    "accessibility tree is empty (custom-drawn/canvas/WebView UI) or it has been looping without "
+    "reaching anything new. Look at the screenshot and choose the SINGLE next action a real user "
+    "would take to make forward progress (reveal new content or navigate onward). Respond with ONLY "
+    "JSON: {\"action\":\"tap|swipe_up|swipe_down|back|none\",\"x\":<0..1>,\"y\":<0..1>,\"reason\":"
+    "\"<brief>\"}. For \"tap\", x and y are NORMALIZED coordinates (0=left/top, 1=right/bottom) of "
+    "the point to tap — aim at the center of a primary button or an unexplored item. Use "
+    "\"swipe_up\" to scroll for more, \"back\" to leave a dead-end, and \"none\" only if truly "
+    "nothing is actionable."
+)
+
+
+def parse_action(text):
+    """Mirror of parseVisionAction — tolerant, clamps coords, tap-without-coords -> none."""
+    start, end = text.find("{"), text.rfind("}")
+    if start < 0 or end <= start:
+        return None
+    try:
+        obj = json.loads(text[start:end + 1])
+    except Exception:
+        return None
+    raw = str(obj.get("action", "none")).lower().strip()
+    aliases = {"swipeup": "swipe_up", "swipe up": "swipe_up", "scroll_up": "swipe_up", "scroll up": "swipe_up",
+               "swipedown": "swipe_down", "swipe down": "swipe_down", "scroll_down": "swipe_down", "scroll down": "swipe_down"}
+    kind = raw if raw in ("tap", "swipe_up", "swipe_down", "back", "none") else aliases.get(raw, "none")
+
+    def num(v):
+        try:
+            return float(v)
+        except Exception:
+            return None
+    x, y = num(obj.get("x")), num(obj.get("y"))
+    if kind == "tap" and (x is None or y is None):
+        kind = "none"
+    return {"action": kind,
+            "x": min(max(x if x is not None else 0.5, 0.0), 1.0),
+            "y": min(max(y if y is not None else 0.5, 0.0), 1.0),
+            "reason": str(obj.get("reason", ""))[:120]}
+
+
+def decide(image_path, screen, reason, key):
+    with open(image_path, "rb") as f:
+        b64 = base64.b64encode(f.read()).decode()
+    body = {
+        "model": MODEL, "max_tokens": 300, "system": NAV_PROMPT,
+        "messages": [{"role": "user", "content": [
+            {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": b64}},
+            {"type": "text", "text": f"Screen title: {screen}. The explorer got stuck ({reason}). Choose the single best next action as the JSON object."},
+        ]}],
+    }
+    req = urllib.request.Request("https://api.anthropic.com/v1/messages", data=json.dumps(body).encode(),
+                                 method="POST", headers={"x-api-key": key, "anthropic-version": "2023-06-01",
+                                                         "content-type": "application/json"})
+    with urllib.request.urlopen(req, timeout=45, context=_SSL_CTX) as resp:
+        data = json.load(resp)
+    text = "\n".join(b.get("text", "") for b in data.get("content", []) if b.get("type") == "text")
+    return parse_action(text)
+
+
+def main():
+    if len(sys.argv) < 3:
+        print("usage: vision_escalation_responder.py <harness.log> <response_path>", file=sys.stderr)
+        return 2
+    log_path, response_path = sys.argv[1], sys.argv[2]
+    key = os.environ.get("ANTHROPIC_API_KEY", "")
+    if not key:
+        print("ANTHROPIC_API_KEY not set", file=sys.stderr)
+        return 2
+
+    handled = set()
+    pos = 0
+    while True:
+        time.sleep(0.4)
+        try:
+            with open(log_path, errors="ignore") as f:
+                f.seek(pos)
+                chunk = f.read()
+                pos = f.tell()
+        except FileNotFoundError:
+            continue
+        for line in chunk.splitlines():
+            idx = line.find("OCQA_VISION_QUERY:{")
+            if idx < 0:
+                continue
+            try:
+                q = json.loads(line[idx + len("OCQA_VISION_QUERY:"):])
+            except Exception:
+                continue
+            rid = q.get("requestId", "")
+            if not rid or rid in handled:
+                continue
+            handled.add(rid)
+            decision = None
+            try:
+                decision = decide(q.get("image", ""), q.get("screen", "?"), q.get("reason", "stuck"), key)
+            except Exception as e:
+                print(f"responder: decide failed for {rid}: {e}", file=sys.stderr, flush=True)
+            out = {"requestId": rid,
+                   "action": (decision or {}).get("action", "none"),
+                   "x": (decision or {}).get("x", 0.5),
+                   "y": (decision or {}).get("y", 0.5)}
+            tmp = response_path + ".tmp"
+            with open(tmp, "w") as f:
+                json.dump(out, f)
+            os.replace(tmp, response_path)  # atomic — harness never sees a partial file
+            print(f"responder: {rid[:8]} on '{q.get('screen','?')}' ({q.get('reason','')}) -> "
+                  f"{out['action']} ({out['x']:.2f},{out['y']:.2f}) {(decision or {}).get('reason','')}",
+                  flush=True)
+
+
+if __name__ == "__main__":
+    sys.exit(main())