@aarwitz/tapp 0.15.0 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/AGENTS.md +4 -4
  2. package/Harness/OCQAHarness/AppDelegate.swift +1 -1
  3. package/Harness/OCQAHarness.xcodeproj/project.pbxproj +2 -2
  4. package/Harness/OCQAHarnessUITests/ExplorerTests.swift +2 -2
  5. package/Harness/generate-harness-xcodeproj.rb +3 -3
  6. package/README.md +17 -15
  7. package/bin/tapp.js +28 -28
  8. package/browser/app.js +2 -2
  9. package/browser/index.html +1 -1
  10. package/docs/BROWSER-PRODUCT.md +2 -2
  11. package/docs/PRODUCT-ENGINE.md +7 -6
  12. package/docs/application-model.md +20 -25
  13. package/docs/scenarios.md +7 -7
  14. package/mcp-server/src/application-model.js +32 -26
  15. package/mcp-server/src/browser-product.js +1 -1
  16. package/mcp-server/src/ci-report.js +5 -4
  17. package/mcp-server/src/ci-setup.js +16 -7
  18. package/mcp-server/src/enrich.js +1 -1
  19. package/mcp-server/src/index.js +63 -62
  20. package/mcp-server/src/maintenance-proposal.js +4 -4
  21. package/mcp-server/src/pr-selection.js +9 -8
  22. package/mcp-server/src/product-execution.js +1 -1
  23. package/mcp-server/src/product-operations.js +20 -18
  24. package/mcp-server/src/project-config.js +8 -5
  25. package/mcp-server/src/project-paths.js +32 -0
  26. package/mcp-server/src/report.js +3 -3
  27. package/mcp-server/src/task-runtime.js +14 -10
  28. package/package.json +15 -3
  29. package/scripts/ci-gate.sh +15 -10
  30. package/scripts/flow_ai_judge.py +1 -1
  31. package/scripts/flow_lib.py +1 -1
  32. package/scripts/quick-capture.sh +15 -14
  33. package/scripts/run-flow.sh +6 -6
  34. package/scripts/android-corpus-e2e.sh +0 -30
  35. package/scripts/cleanup-xcode.sh +0 -157
  36. package/scripts/corpus-apps.txt +0 -9
  37. package/scripts/corpus-sweep.sh +0 -121
  38. package/scripts/coverage-eval.sh +0 -92
  39. package/scripts/coverage_eval_parse.py +0 -95
  40. package/scripts/deploy-and-build.sh +0 -99
  41. package/scripts/mutation-recall-desktop.sh +0 -186
  42. package/scripts/mutation-recall.sh +0 -121
  43. package/scripts/mutation_lib.py +0 -128
  44. package/scripts/mutation_operators.py +0 -144
  45. package/scripts/validation-matrix.sh +0 -146
  46. package/scripts/vision-fp-eval.sh +0 -206
  47. package/scripts/vision_escalation_responder.py +0 -147
  48. package/scripts/vision_fp_probe.py +0 -221
@@ -1,157 +0,0 @@
1
- #!/usr/bin/env bash
2
- # Reclaim disk space consumed by Xcode build artifacts.
3
- #
4
- # Default run (no flags): deletes unavailable simulators and stale DerivedData
5
- # from unrelated projects. Flags unlock more aggressive passes.
6
- #
7
- # Usage:
8
- # scripts/cleanup-xcode.sh # safe defaults
9
- # scripts/cleanup-xcode.sh --dry-run # preview only, nothing deleted
10
- # scripts/cleanup-xcode.sh --device-support # also wipe iOS DeviceSupport caches
11
- # scripts/cleanup-xcode.sh --all # everything above + Xcode caches
12
- #
13
- # AutoTap-related DerivedData (OCQAHarness, DemoApp, LoginDemo, etc.) is always
14
- # preserved. The local build/ directory in this repo is never touched.
15
- set -uo pipefail
16
-
17
- DRY_RUN=0
18
- CLEAN_DEVICE_SUPPORT=0
19
- CLEAN_ALL=0
20
-
21
- for arg in "$@"; do
22
- case "$arg" in
23
- --dry-run) DRY_RUN=1 ;;
24
- --device-support) CLEAN_DEVICE_SUPPORT=1 ;;
25
- --all) CLEAN_DEVICE_SUPPORT=1; CLEAN_ALL=1 ;;
26
- -h|--help)
27
- sed -n '/^# /s/^# //p' "$0" | head -15
28
- exit 0 ;;
29
- *) echo "Unknown flag: $arg (try --help)"; exit 1 ;;
30
- esac
31
- done
32
-
33
- DERIVED="$HOME/Library/Developer/Xcode/DerivedData"
34
- SIMS="$HOME/Library/Developer/CoreSimulator/Devices"
35
- DEVICE_SUPPORT="$HOME/Library/Developer/Xcode/iOS DeviceSupport"
36
- XCODE_CACHES="$HOME/Library/Caches/com.apple.dt.Xcode"
37
-
38
- # Projects to preserve in DerivedData (all AutoTap-related build products).
39
- KEEP_PREFIXES=("AutoTap" "OCQAHarness" "DemoApp" "LoginDemo" "WizardDemo"
40
- "ShopDemo" "RestaurantDemo" "SymbolCache")
41
-
42
- hr() { printf '%0.s─' {1..60}; echo; }
43
- human() { du -sh "$1" 2>/dev/null | cut -f1; }
44
-
45
- run() {
46
- if [[ $DRY_RUN -eq 1 ]]; then
47
- echo " [dry-run] $*"
48
- else
49
- eval "$@"
50
- fi
51
- }
52
-
53
- echo ""
54
- hr
55
- echo " Xcode Cleanup $(date '+%Y-%m-%d %H:%M')"
56
- [[ $DRY_RUN -eq 1 ]] && echo " DRY RUN — nothing will be deleted"
57
- hr
58
-
59
- # ── Before snapshot ──────────────────────────────────────────────────────────
60
- echo ""
61
- echo "Before:"
62
- printf " %-38s %s\n" "CoreSimulator/Devices" "$(human "$SIMS")"
63
- printf " %-38s %s\n" "Xcode/DerivedData" "$(human "$DERIVED")"
64
- printf " %-38s %s\n" "Xcode/iOS DeviceSupport" "$(human "$DEVICE_SUPPORT")"
65
- printf " %-38s %s\n" "Xcode caches" "$(human "$XCODE_CACHES")"
66
- echo ""
67
-
68
- # ── 1. Unavailable simulators ─────────────────────────────────────────────────
69
- echo "1/3 Removing unavailable simulators …"
70
- UNAVAIL=$(xcrun simctl list devices --json 2>/dev/null | \
71
- python3 -c '
72
- import sys, json
73
- d = json.load(sys.stdin)
74
- n = sum(1 for v in d["devices"].values() for x in v if not x.get("isAvailable", True))
75
- print(n)
76
- ')
77
- echo " Found $UNAVAIL unavailable device(s)."
78
- if [[ "$UNAVAIL" -gt 0 ]]; then
79
- run "xcrun simctl delete unavailable"
80
- fi
81
-
82
- # ── 2. Unrelated DerivedData ──────────────────────────────────────────────────
83
- echo ""
84
- echo "2/3 Scanning DerivedData for unrelated projects …"
85
- STALE=()
86
- if [[ -d "$DERIVED" ]]; then
87
- while IFS= read -r -d '' entry; do
88
- base="$(basename "$entry")"
89
- keep=0
90
- for prefix in "${KEEP_PREFIXES[@]}"; do
91
- if [[ "$base" == "$prefix"* ]]; then
92
- keep=1; break
93
- fi
94
- done
95
- [[ $keep -eq 0 ]] && STALE+=("$entry")
96
- done < <(find "$DERIVED" -maxdepth 1 -mindepth 1 -type d -print0)
97
- fi
98
-
99
- if [[ ${#STALE[@]} -eq 0 ]]; then
100
- echo " Nothing to remove."
101
- else
102
- echo " Will remove ${#STALE[@]} unrelated build folder(s):"
103
- for entry in "${STALE[@]}"; do
104
- printf " %-8s %s\n" "$(human "$entry")" "$(basename "$entry")"
105
- done
106
- for entry in "${STALE[@]}"; do
107
- run "rm -rf \"$entry\""
108
- done
109
- fi
110
-
111
- # ── 3. iOS DeviceSupport (optional) ──────────────────────────────────────────
112
- echo ""
113
- echo "3/3 iOS DeviceSupport …"
114
- if [[ $CLEAN_DEVICE_SUPPORT -eq 1 ]]; then
115
- if [[ -d "$DEVICE_SUPPORT" ]]; then
116
- echo " Removing $(human "$DEVICE_SUPPORT") of device symbol caches …"
117
- echo " (Xcode re-downloads these automatically when a device is connected.)"
118
- run "rm -rf \"$DEVICE_SUPPORT\""
119
- else
120
- echo " Directory not found — nothing to do."
121
- fi
122
- else
123
- SIZE=$(human "$DEVICE_SUPPORT")
124
- echo " Skipped ($SIZE on disk). Pass --device-support to remove."
125
- fi
126
-
127
- # ── 4. Xcode caches (--all only) ─────────────────────────────────────────────
128
- if [[ $CLEAN_ALL -eq 1 ]]; then
129
- echo ""
130
- echo "4/4 Xcode caches …"
131
- if [[ -d "$XCODE_CACHES" ]]; then
132
- echo " Removing $(human "$XCODE_CACHES") …"
133
- run "rm -rf \"$XCODE_CACHES\""
134
- fi
135
- fi
136
-
137
- # ── Simulator data cache (always safe) ───────────────────────────────────────
138
- SIM_CACHES="$HOME/Library/Developer/CoreSimulator/Caches"
139
- if [[ -d "$SIM_CACHES" ]] && [[ "$(du -sk "$SIM_CACHES" 2>/dev/null | cut -f1)" -gt 0 ]]; then
140
- echo ""
141
- echo " Removing simulator caches ($(human "$SIM_CACHES")) …"
142
- run "rm -rf \"$SIM_CACHES\""
143
- fi
144
-
145
- # ── After snapshot ─────────────────────────────────────────────────────────────
146
- echo ""
147
- if [[ $DRY_RUN -eq 0 ]]; then
148
- hr
149
- echo "After:"
150
- printf " %-38s %s\n" "CoreSimulator/Devices" "$(human "$SIMS")"
151
- printf " %-38s %s\n" "Xcode/DerivedData" "$(human "$DERIVED")"
152
- printf " %-38s %s\n" "Xcode/iOS DeviceSupport" "$(human "$DEVICE_SUPPORT")"
153
- printf " %-38s %s\n" "Xcode caches" "$(human "$XCODE_CACHES")"
154
- echo ""
155
- fi
156
-
157
- echo "Done."
@@ -1,9 +0,0 @@
1
- # Corpus sweep list — curated from dkhamsing/open-source-ios-apps across backend classes.
2
- # Class coverage: self-hostable-service client (login wall), public API (no key),
3
- # public API (keyed), no-backend/local-first, heavyweight real-world.
4
- https://github.com/Dimillian/IceCubesApp
5
- https://github.com/weiran/Hackers
6
- https://github.com/Ranchero-Software/NetNewsWire
7
- https://github.com/yattee/yattee
8
- https://github.com/wikimedia/wikipedia-ios
9
- https://github.com/mastodon/mastodon-ios
@@ -1,121 +0,0 @@
1
- #!/usr/bin/env bash
2
- # Corpus sweep — run a list of real open-source iOS apps through the tapp pipeline and
3
- # produce a friction scoreboard. Failures ARE the data: every app gets a row recording
4
- # how far it got (clone → detect → build → install → explore) and why it stopped.
5
- #
6
- # Usage: scripts/corpus-sweep.sh <urls-file> [--actions N] [--out <dir>]
7
- # urls-file: one git URL per line (# comments ok)
8
- #
9
- # Needs a booted simulator. Rows land in <out>/scoreboard.tsv; per-app logs in <out>/<app>/.
10
- set -uo pipefail
11
- ROOT="$(cd "$(dirname "$0")/.." && pwd)"
12
-
13
- URLS_FILE="${1:?usage: corpus-sweep.sh <urls-file> [--actions N] [--out dir]}"; shift
14
- ACTIONS=25
15
- OUT="/tmp/tapp-corpus"
16
- while [[ $# -gt 0 ]]; do
17
- case "$1" in
18
- --actions) ACTIONS="$2"; shift 2 ;;
19
- --out) OUT="$2"; shift 2 ;;
20
- *) echo "unknown arg $1" >&2; exit 2 ;;
21
- esac
22
- done
23
- mkdir -p "$OUT"
24
- BOARD="$OUT/scoreboard.tsv"
25
- [[ -f "$BOARD" ]] || echo -e "app\tstage\tbackend_signals\tbundle\tscreens\tactions\tfindings\tverdict\tnote" > "$BOARD"
26
-
27
- UDID="$(xcrun simctl list devices booted -j | python3 -c 'import sys,json; d=json.load(sys.stdin); print(next((x["udid"] for v in d["devices"].values() for x in v if x.get("state")=="Booted"), ""))')"
28
- [[ -n "$UDID" ]] || { echo "❌ boot a simulator first" >&2; exit 1; }
29
-
30
- row() { echo -e "$1\t$2\t$3\t$4\t$5\t$6\t$7\t$8\t$9" >> "$BOARD"; }
31
-
32
- # Cheap backend-class signals from the repo tree (feeds the classifier's rules).
33
- backend_signals() {
34
- local dir="$1" sig=()
35
- [[ -n "$(find "$dir" -maxdepth 4 -name 'GoogleService-Info.plist' -print -quit 2>/dev/null)" ]] && sig+=("firebase-plist")
36
- grep -rqli "firebase" "$dir"/Podfile "$dir"/*/Package.resolved "$dir"/Package.resolved 2>/dev/null && sig+=("firebase-dep")
37
- grep -rqli "supabase" "$dir"/Podfile "$dir"/*/Package.resolved "$dir"/Package.resolved 2>/dev/null && sig+=("supabase-dep")
38
- [[ -n "$(find "$dir" -maxdepth 2 -name 'docker-compose.y*ml' -print -quit 2>/dev/null)" ]] && sig+=("in-repo-backend")
39
- grep -rqli "mastodon" "$dir"/Package.resolved 2>/dev/null && sig+=("mastodon-client")
40
- grep -rql "YOUR_API_KEY\|INSERT_API_KEY\|<API_KEY>" --include="*.xcconfig" --include="*.plist" --include="*.swift" "$dir" 2>/dev/null | head -1 | grep -q . && sig+=("key-placeholder")
41
- local IFS=","; echo "${sig[*]:-none}"
42
- }
43
-
44
- # fd 3 for the URL list — xcodebuild/git inside the loop consume stdin and would
45
- # silently swallow the remaining lines (this truncated the first sweep run).
46
- while IFS= read -r url <&3; do
47
- [[ -z "$url" || "$url" == \#* ]] && continue
48
- name="$(basename "$url" .git)"
49
- dir="$OUT/$name"
50
- log="$OUT/$name.log"
51
- echo "━━━ $name"
52
-
53
- # 1. clone
54
- if [[ ! -d "$dir" ]]; then
55
- git clone --depth 1 --recurse-submodules --shallow-submodules "$url" "$dir" >"$log" 2>&1 \
56
- || { row "$name" clone none - - - - - "git clone failed"; continue; }
57
- fi
58
- signals="$(backend_signals "$dir")"
59
-
60
- # 2. detect workspace/project + scheme (workspace wins; ignore Pods)
61
- target="$(find "$dir" -maxdepth 3 -name '*.xcworkspace' -not -path '*/Pods/*' -not -path '*/.*' | head -1)"
62
- flag="-workspace"
63
- if [[ -z "$target" ]]; then
64
- target="$(find "$dir" -maxdepth 3 -name '*.xcodeproj' -not -path '*/Pods/*' | head -1)"
65
- flag="-project"
66
- fi
67
- [[ -n "$target" ]] || { row "$name" detect "$signals" - - - - - "no xcworkspace/xcodeproj found"; continue; }
68
- schemes="$(xcodebuild -list -json $flag "$target" 2>>"$log" | python3 -c 'import sys,json
69
- try:
70
- d=json.load(sys.stdin); o=d.get("workspace") or d.get("project") or {}
71
- print("\n".join(o.get("schemes") or []))
72
- except Exception: pass')"
73
- # Prefer an explicitly-iOS scheme (multi-platform repos like NetNewsWire name-match the
74
- # macOS scheme first, which can't build for an iOS Simulator destination).
75
- scheme="$(echo "$schemes" | grep -i "ios" | grep -iv "test\|widget\|extension\|notification\|clip" | head -1)"
76
- [[ -z "$scheme" ]] && scheme="$(echo "$schemes" | grep -ix "$name" | head -1)"
77
- [[ -z "$scheme" ]] && scheme="$(echo "$schemes" | grep -iv "test\|uitests\|widget\|extension\|notification\|clip\|watch\|mac" | head -1)"
78
- [[ -z "$scheme" ]] && scheme="$(echo "$schemes" | head -1)"
79
- [[ -n "$scheme" ]] || { row "$name" detect "$signals" - - - - - "no schemes listed"; continue; }
80
- echo " scheme: $scheme"
81
-
82
- # 3. build for simulator (20 min cap; perl alarm — stock macOS has no `timeout`)
83
- dd="$OUT/dd-$name"
84
- if ! perl -e 'alarm shift; exec @ARGV' 1200 xcodebuild $flag "$target" -scheme "$scheme" \
85
- -destination 'generic/platform=iOS Simulator' -derivedDataPath "$dd" \
86
- CODE_SIGN_IDENTITY="" CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO \
87
- build >>"$log" 2>&1; then
88
- reason="$(grep -m1 -E "error: |xcodebuild: error" "$log" | tail -c 120 || true)"
89
- row "$name" build "$signals" - - - - - "build failed: ${reason:-see log}"
90
- continue
91
- fi
92
- app="$(find "$dd/Build/Products" -maxdepth 2 -name '*.app' -path '*iphonesimulator*' | head -1)"
93
- [[ -n "$app" ]] || { row "$name" build "$signals" - - - - - "built but no simulator .app (macOS-only scheme?)"; continue; }
94
- bundle="$(/usr/libexec/PlistBuddy -c 'Print CFBundleIdentifier' "$app/Info.plist" 2>/dev/null || echo "?")"
95
-
96
- # 4. install
97
- xcrun simctl install "$UDID" "$app" >>"$log" 2>&1 \
98
- || { row "$name" install "$signals" "$bundle" - - - - "simctl install failed"; continue; }
99
-
100
- # 5. explore + report
101
- "$ROOT/scripts/quick-capture.sh" explore "$bundle" --actions "$ACTIONS" --timeout 420 >>"$log" 2>&1
102
- cap="$(ls -td "$ROOT"/captures/*/ 2>/dev/null | head -1)"
103
- if [[ ! -f "$cap/ocqa-markers.txt" ]]; then
104
- row "$name" explore "$signals" "$bundle" - - - - "exploration produced no markers"
105
- continue
106
- fi
107
- json="$OUT/$name-report.json"
108
- node "$ROOT/mcp-server/src/ci-report.js" --markers "$cap/ocqa-markers.txt" --json-out "$json" >>"$log" 2>&1 || true
109
- read -r screens acted findings verdict <<< "$(python3 -c "
110
- import json,sys
111
- try:
112
- r=json.load(open('$json'))
113
- print(r.get('screensExplored','?'), r.get('actionsPerformed','?'), r.get('findingCounts',{}).get('total','?'), r.get('verdict','?'))
114
- except Exception: print('? ? ? ?')")"
115
- row "$name" explored "$signals" "$bundle" "$screens" "$acted" "$findings" "$verdict" "capture: $(basename "$cap")"
116
- echo " ✅ $verdict — $screens screens"
117
- done 3< "$URLS_FILE"
118
-
119
- echo ""
120
- echo "═══ scoreboard: $BOARD ═══"
121
- column -t -s $'\t' "$BOARD"
@@ -1,92 +0,0 @@
1
- #!/usr/bin/env bash
2
- # AutoTap penetration / coverage eval — the "real-app scoreboard".
3
- #
4
- # Unlike validation-matrix.sh (which asserts *expected findings* against DemoApp, i.e. accuracy on a
5
- # known fixture), this measures how far AutoTap actually PENETRATES an arbitrary app: did it get past
6
- # launch, did it hit a login wall, how many distinct screens it reached, how much of its budget went
7
- # to productive exploration vs. stuck/recovery flailing, and what it found. There is no ground truth —
8
- # this is a diagnostic you point at real apps to see where they stall, so coverage work is measurable.
9
- #
10
- # Usage:
11
- # scripts/coverage-eval.sh [bundleId ...] # defaults to the local corpus apps
12
- # scripts/coverage-eval.sh com.acme.app # a real app (must be installed on the booted sim)
13
- # ACTIONS=120 scripts/coverage-eval.sh com.acme.app
14
- # VISION_ESCALATION=1 ANTHROPIC_API_KEY=... scripts/coverage-eval.sh com.acme.app
15
- # # opt-in: when the a11y tree goes blank or exploration gets stuck, a sidecar
16
- # # (vision_escalation_responder.py) answers the harness's OCQA_VISION_QUERY with a
17
- # # model-chosen next move — same channel the desktop app serves.
18
- #
19
- # Prereqs: a booted simulator, the harness built (scripts/deploy-and-build.sh --harness), and each
20
- # app installed on the sim. Writes a JSON report to captures/coverage-eval-<timestamp>.json.
21
- #
22
- # NOTE: auth sessions stored in the keychain (Firebase et al) SURVIVE app reinstall — a "fresh"
23
- # run may silently resume the previous account. For a genuinely fresh login:
24
- # xcrun simctl keychain <udid> reset
25
- set -uo pipefail
26
-
27
- ROOT="$(cd "$(dirname "$0")/.." && pwd)"
28
- ACTIONS="${ACTIONS:-70}"
29
- TIMEOUT="${TIMEOUT:-600}"
30
- DEFAULT_APPS=(com.autotap.demoapp com.autotap.logindemo com.autotap.wizarddemo com.autotap.shopdemo com.autotap.restaurantdemo)
31
- APPS=("$@"); [ ${#APPS[@]} -eq 0 ] && APPS=("${DEFAULT_APPS[@]}")
32
-
33
- UDID="$(xcrun simctl list devices booted -j 2>/dev/null | python3 -c 'import sys,json; d=json.load(sys.stdin); print(next((x["udid"] for v in d["devices"].values() for x in v if x.get("state")=="Booted"), ""))')"
34
- [ -z "$UDID" ] && { echo "❌ No booted simulator. Boot one first."; exit 2; }
35
-
36
- XCTR="$(find "$HOME/Library/Developer/Xcode/DerivedData/OCQAHarness-"*/Build/Products -name '*.xctestrun' 2>/dev/null | head -1)"
37
- [ -z "$XCTR" ] && XCTR="$(find /tmp/autotap-harness-derived/Build/Products -name '*.xctestrun' 2>/dev/null | head -1)"
38
- [ -z "$XCTR" ] && XCTR="$(find /tmp/harness-build/Build/Products -name '*.xctestrun' 2>/dev/null | head -1)"
39
- [ -z "$XCTR" ] && { echo "❌ Harness not built. Run: scripts/deploy-and-build.sh --harness"; exit 2; }
40
-
41
- TS="$(date +%Y%m%d-%H%M%S)"
42
- REPORT="$ROOT/captures/coverage-eval-$TS.json"
43
- mkdir -p "$ROOT/captures"
44
- echo "AutoTap coverage eval — $ACTIONS actions/app, sim $UDID"
45
- echo "Apps: ${APPS[*]}"
46
- echo ""
47
-
48
- RESULTS_JSON="["
49
- FIRST=1
50
- for APP in "${APPS[@]}"; do
51
- if ! xcrun simctl get_app_container "$UDID" "$APP" >/dev/null 2>&1; then
52
- printf " %-32s ⏭ not installed — skipping\n" "$APP"
53
- continue
54
- fi
55
- CFG="/tmp/ocqa-coverage-$APP.json"
56
- LOG="/tmp/ocqa-coverage-$APP.log"
57
- RESPONDER_PID=""
58
- # Real test credentials (OCQA_TEST_EMAIL / OCQA_TEST_PASSWORD env) let the heuristic login
59
- # preamble get PAST a real auth wall — the single biggest coverage unlock on gated apps.
60
- CREDS_LINE=""
61
- [ -n "${OCQA_TEST_EMAIL:-}" ] && CREDS_LINE=",
62
- \"OCQA_TEST_EMAIL\": \"$OCQA_TEST_EMAIL\", \"OCQA_TEST_PASSWORD\": \"${OCQA_TEST_PASSWORD:-}\""
63
- if [ "${VISION_ESCALATION:-0}" = "1" ] && [ -n "${ANTHROPIC_API_KEY:-}" ]; then
64
- VRESP="/tmp/ocqa-vision-response-eval-$APP.json"
65
- VDIR="/tmp/ocqa-vision-eval-$APP"; mkdir -p "$VDIR"
66
- cat > "$CFG" <<JSON
67
- { "OCQA_BUNDLE_ID": "$APP", "OCQA_MAX_ACTIONS": "$ACTIONS", "OCQA_TIMEOUT_SECONDS": "$TIMEOUT",
68
- "OCQA_VISION_ESCALATION": "1", "OCQA_VISION_RESPONSE_PATH": "$VRESP",
69
- "OCQA_VISION_IMAGE_DIR": "$VDIR", "OCQA_VISION_BUDGET": "4", "OCQA_VISION_WAIT_TIMEOUT": "60"$CREDS_LINE }
70
- JSON
71
- : > "$LOG"
72
- python3 "$ROOT/scripts/vision_escalation_responder.py" "$LOG" "$VRESP" > "/tmp/ocqa-vision-responder-$APP.log" 2>&1 &
73
- RESPONDER_PID=$!
74
- else
75
- cat > "$CFG" <<JSON
76
- { "OCQA_BUNDLE_ID": "$APP", "OCQA_MAX_ACTIONS": "$ACTIONS", "OCQA_TIMEOUT_SECONDS": "$TIMEOUT"$CREDS_LINE }
77
- JSON
78
- fi
79
- TEST_RUNNER_OCQA_CONFIG_PATH="$CFG" xcodebuild test-without-building \
80
- -xctestrun "$XCTR" -destination "platform=iOS Simulator,id=$UDID" \
81
- -only-testing:"OCQAHarnessUITests/ExplorerTests/testAutonomousExploration" > "$LOG" 2>&1
82
- if [ -n "$RESPONDER_PID" ]; then kill "$RESPONDER_PID" 2>/dev/null; wait "$RESPONDER_PID" 2>/dev/null; fi
83
-
84
- ROW="$(python3 "$ROOT/scripts/coverage_eval_parse.py" "$LOG" "$APP")"
85
- echo "$ROW" | python3 -c 'import sys,json; d=json.load(sys.stdin); ca=d.get("crash_action") or {}; print(" %-32s screens=%-3s launch=%-3s login_wall=%-3s stuck=%-4s findings=%s%s" % (d["app"], d["screens"], "ok" if d["launch_ok"] else "NO", "YES" if d["login_wall"] else "no", d["stuck_ratio"], d["findings_total"], (" ⚠️ CRASHED on %s %s" % (ca.get("type",""), ca.get("screen","")) if d.get("crashed") else "")))'
86
- [ $FIRST -eq 0 ] && RESULTS_JSON+=","
87
- RESULTS_JSON+="$ROW"; FIRST=0
88
- done
89
- RESULTS_JSON+="]"
90
- echo "$RESULTS_JSON" | python3 -m json.tool > "$REPORT" 2>/dev/null || echo "$RESULTS_JSON" > "$REPORT"
91
- echo ""
92
- echo "Report: $REPORT"
@@ -1,95 +0,0 @@
1
- #!/usr/bin/env python3
2
- """Parse a harness exploration log into penetration/coverage diagnostics for the coverage eval.
3
- Emits one JSON object describing how far AutoTap got into the app. No ground truth — diagnostics only."""
4
- import sys, re, json
5
-
6
- log = open(sys.argv[1], encoding="utf-8", errors="replace").read()
7
- app = sys.argv[2] if len(sys.argv) > 2 else "?"
8
-
9
- # Reasons / action types that mean "flailing to escape / not making forward progress".
10
- RECOVERY_REASONS = ("escape", "exhausted", "blind", "stuck", "discover", "drawer",
11
- "carousel", "probe", "dismiss", "recovery", "back_failed", "trap", "rotation")
12
- RECOVERY_TYPES = {"back", "swipe_dismiss", "tab_rotation", "forced_escape", "swipe_back", "swipe"}
13
-
14
- screens, findings, actions = set(), {}, 0
15
- recovery_actions = 0
16
- saw_secure_field = False
17
- completed = False
18
- test_failed = False
19
- last_action = None # (type, screen, step) of the most recent action
20
-
21
- for line in log.splitlines():
22
- m = re.search(r"OCQA_STATE:(\{.*\})", line)
23
- if m:
24
- try:
25
- d = json.loads(m.group(1))
26
- s = (d.get("screen") or "").strip()
27
- if s and s not in ("Unknown", "pending"):
28
- screens.add(s)
29
- for inp in d.get("inputs", []) or []:
30
- if inp.get("secure"):
31
- saw_secure_field = True
32
- except Exception:
33
- pass
34
- continue
35
- m = re.search(r"OCQA_ACTION:(\{.*\})", line)
36
- if m:
37
- actions += 1
38
- try:
39
- d = json.loads(m.group(1))
40
- reason = str(d.get("reason", "")).lower()
41
- atype = str(d.get("type", "")).lower()
42
- last_action = (atype, (d.get("screen") or "").strip(), d.get("step"))
43
- if atype in RECOVERY_TYPES or any(k in reason for k in RECOVERY_REASONS):
44
- recovery_actions += 1
45
- except Exception:
46
- pass
47
- continue
48
- m = re.search(r"OCQA_ISSUE:(\{.*\})", line)
49
- if m:
50
- try:
51
- t = json.loads(m.group(1)).get("type", "unknown")
52
- findings[t] = findings.get(t, 0) + 1
53
- except Exception:
54
- pass
55
- continue
56
- if line.startswith("OCQA_COMPLETE"):
57
- completed = True
58
- elif "TEST EXECUTE FAILED" in line or re.search(r"Executed 1 test, with [1-9]\d* failure", line):
59
- test_failed = True
60
-
61
- # ---- Host-side crash detection ----
62
- # A crash DURING an action kills the test's connection to the app and aborts the harness before
63
- # any in-harness detector can run (found on Wikipedia: WMFCaptchaViewController.refreshImage
64
- # assertionFailure crashes the app on login). The signal is unambiguous: the test FAILED and the
65
- # harness never emitted OCQA_COMPLETE, with actions performed — i.e. it died mid-run. Attribute the
66
- # crash to the last action and surface it as a critical finding (the whole point of the tool).
67
- crashed = test_failed and not completed and actions > 0
68
- if crashed and "crash" not in findings:
69
- findings["crash"] = findings.get("crash", 0) + 1
70
-
71
- n_screens = len(screens)
72
- launch_ok = n_screens >= 2
73
- # Login wall: a password field was present but the explorer never got past a handful of screens.
74
- login_present = saw_secure_field
75
- login_wall = login_present and n_screens <= 3
76
- stuck_ratio = round(recovery_actions / actions, 2) if actions else 0.0
77
- findings_total = sum(findings.values())
78
-
79
- print(json.dumps({
80
- "app": app,
81
- "screens": n_screens,
82
- "screen_names": sorted(screens),
83
- "launch_ok": launch_ok,
84
- "login_present": login_present,
85
- "login_wall": login_wall,
86
- "actions": actions,
87
- "recovery_actions": recovery_actions,
88
- "stuck_ratio": stuck_ratio, # 0=all productive, 1=all flailing
89
- "findings_total": findings_total,
90
- "findings_by_type": findings,
91
- "completed": completed,
92
- "crashed": crashed,
93
- "crash_action": ({"type": last_action[0], "screen": last_action[1], "step": last_action[2]}
94
- if crashed and last_action else None),
95
- }))
@@ -1,99 +0,0 @@
1
- #!/usr/bin/env bash
2
- set -euo pipefail
3
-
4
- # Build AutoTap locally on this Mac
5
- # Usage: ./scripts/deploy-and-build.sh [--harness] [--clean]
6
- # --harness Also build the OCQAHarness XCUITest bundle
7
- # --clean Clean derived data before building
8
-
9
- PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
10
- BUILD_HARNESS=0
11
- DO_CLEAN=0
12
-
13
- for arg in "$@"; do
14
- case "$arg" in
15
- --harness) BUILD_HARNESS=1 ;;
16
- --clean) DO_CLEAN=1 ;;
17
- esac
18
- done
19
-
20
- echo "=== AutoTap Local Build ==="
21
- echo "Project: $PROJECT_ROOT"
22
- echo ""
23
-
24
- if [[ $DO_CLEAN -eq 1 ]]; then
25
- echo "Cleaning derived data..."
26
- rm -rf "$PROJECT_ROOT/build/DerivedData"
27
- rm -rf /tmp/autotap-harness-derived
28
- echo ""
29
- fi
30
-
31
- # Step 1: Generate Xcode project if needed
32
- if [[ ! -f "$PROJECT_ROOT/AutoTap.xcodeproj/project.pbxproj" ]] || \
33
- [[ "$PROJECT_ROOT/generate-xcodeproj.rb" -nt "$PROJECT_ROOT/AutoTap.xcodeproj/project.pbxproj" ]]; then
34
- echo "Generating Xcode project..."
35
- cd "$PROJECT_ROOT" && ruby generate-xcodeproj.rb
36
- echo ""
37
- fi
38
-
39
- # Step 2: Build main app
40
- echo "Building AutoTap..."
41
- xcodebuild \
42
- -project "$PROJECT_ROOT/AutoTap.xcodeproj" \
43
- -scheme AutoTap \
44
- -configuration Debug \
45
- -derivedDataPath "$PROJECT_ROOT/build/DerivedData" \
46
- CODE_SIGN_IDENTITY=- \
47
- CODE_SIGNING_REQUIRED=NO \
48
- CODE_SIGNING_ALLOWED=NO \
49
- 2>&1 | tail -20
50
-
51
- echo ""
52
-
53
- # Step 3: Build harness if requested
54
- if [[ $BUILD_HARNESS -eq 1 ]]; then
55
- echo "Building OCQAHarness..."
56
-
57
- # Generate harness xcodeproj if needed
58
- if [[ ! -f "$PROJECT_ROOT/Harness/OCQAHarness.xcodeproj/project.pbxproj" ]] || \
59
- [[ "$PROJECT_ROOT/Harness/generate-harness-xcodeproj.rb" -nt "$PROJECT_ROOT/Harness/OCQAHarness.xcodeproj/project.pbxproj" ]]; then
60
- cd "$PROJECT_ROOT/Harness" && ruby generate-harness-xcodeproj.rb
61
- fi
62
-
63
- # Prefer a booted simulator by UDID to avoid ambiguity when the same device
64
- # name exists across multiple OS runtimes (e.g. "iPhone 16 Pro" on iOS 18 + 26).
65
- SIM_UDID=$(xcrun simctl list devices booted -j 2>/dev/null | \
66
- python3 -c 'import sys,json; d=json.load(sys.stdin); print(next((x["udid"] for v in d["devices"].values() for x in v if x.get("state")=="Booted"), ""))' 2>/dev/null || true)
67
-
68
- if [[ -n "$SIM_UDID" ]]; then
69
- SIM_DEST="platform=iOS Simulator,id=$SIM_UDID"
70
- echo " Target simulator: $SIM_UDID (booted)"
71
- else
72
- SIM_NAME=$(xcrun simctl list devices available | grep -E 'iPhone.*(Booted|Shutdown)' | head -1 | sed 's/ (.*//' | xargs)
73
- SIM_NAME=${SIM_NAME:-"iPhone 16 Pro"}
74
- SIM_DEST="platform=iOS Simulator,name=$SIM_NAME"
75
- echo " Target simulator: $SIM_NAME"
76
- fi
77
-
78
- xcodebuild build-for-testing \
79
- -project "$PROJECT_ROOT/Harness/OCQAHarness.xcodeproj" \
80
- -scheme OCQAHarnessUITests \
81
- -destination "$SIM_DEST" \
82
- -derivedDataPath "/tmp/autotap-harness-derived" \
83
- 2>&1 | tail -5
84
-
85
- echo ""
86
- echo "Harness built. Quick capture:"
87
- echo " ./scripts/quick-capture.sh explore com.autotap.demoapp --actions 25"
88
- echo " ./scripts/quick-capture.sh screenshot"
89
- echo " ./scripts/quick-capture.sh tree com.autotap.demoapp"
90
- fi
91
-
92
- echo ""
93
- APP_PATH="$PROJECT_ROOT/build/DerivedData/Build/Products/Debug/AutoTap.app"
94
- if [[ -d "$APP_PATH" ]]; then
95
- echo "Build artifact: $APP_PATH"
96
- echo "Run: open $APP_PATH"
97
- else
98
- echo "(check build output above for errors)"
99
- fi