@aarwitz/tapp 0.15.0

package/scripts/flow_lib.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+"""Flow helpers for run-flow.sh: convert a .yml/.json Flow to the harness's OCQA_FLOW_JSON, and
+parse the harness's OCQA_FLOW_STEP / OCQA_FLOW_RESULT markers into a scannable pass/fail report.
+
+Usage:
+  flow_lib.py to-json  <flow.yml|flow.json>          # prints {steps:[...], name, vars, ...} JSON
+  flow_lib.py raw-json <flow.yml|flow.json>          # prints the complete repository spec as JSON
+  flow_lib.py to-yaml  '<flow-json-string>'          # prints tidy YAML (for saving a recorded flow)
+  flow_lib.py report   <harness.log>                 # prints a human-readable pass/fail report
+  flow_lib.py report --json <harness.log>            # prints machine JSON {passed,total,failed,steps}
+"""
+import json
+import re
+import sys
+
+
+def load_flow(path):
+    raw = open(path, encoding="utf-8").read()
+    if path.endswith(".json"):
+        return json.loads(raw)
+    import yaml
+    return yaml.safe_load(raw)
+
+
+def to_yaml(json_str):
+    """Emit a recorded/inline Flow (JSON string) as tidy YAML for saving to .autotap/flows/*.yml."""
+    import yaml
+    flow = json.loads(json_str)
+    # Order keys for readability. `platform` + `url` make the same repository-native
+    # Flow format portable across the XCUITest, Playwright, and Android drivers.
+    ordered = {}
+    for k in ("name", "platform", "app", "url", "vars", "reset"):
+        if flow.get(k):
+            ordered[k] = flow[k]
+    ordered["steps"] = flow.get("steps", [])
+    print(yaml.safe_dump(ordered, sort_keys=False, default_flow_style=False, allow_unicode=True, width=100).rstrip())
+
+
+def to_json(path):
+    flow = load_flow(path) or {}
+    # Steps pass through as-is — the harness normalizes both sugar ({tap: X}) and explicit
+    # ({action: tap, target: X}) forms, so no rewriting is needed here.
+    out = {
+        "name": flow.get("name", "flow"),
+        "kind": flow.get("kind", "flow"),
+        "platform": flow.get("platform", ""),
+        "app": flow.get("app", ""),
+        "url": flow.get("url", ""),
+        "steps": flow.get("steps", []),
+        "vars": flow.get("vars", {}),
+        "reset": flow.get("reset", "launch"),
+        "continueOnFailure": bool(flow.get("continueOnFailure", False)),
+    }
+    # Multi-actor Scenarios deliberately reuse the Flow step language and marker
+    # protocol, while preserving actor/session and lifecycle orchestration.
+    for key in ("actors", "setup", "teardown", "timeoutMs", "taskPlan", "releaseContract"):
+        if key in flow:
+            out[key] = flow[key]
+    print(json.dumps(out))
+
+
+def raw_json(path):
+    print(json.dumps(load_flow(path) or {}))
+
+
+def report(path, as_json=False):
+    log = open(path, encoding="utf-8", errors="replace").read()
+    steps = []
+    result = None
+    name = "flow"
+    kind = "flow"
+    for line in log.splitlines():
+        line = line.strip()
+        if line.startswith("OCQA_FLOW_STEP:"):
+            try:
+                steps.append(json.loads(line[len("OCQA_FLOW_STEP:"):]))
+            except Exception:
+                pass
+        elif line.startswith("OCQA_FLOW_RESULT:started"):
+            m = re.search(r"name=(.*)$", line)
+            if m:
+                name = m.group(1).split(" kind=", 1)[0]
+            km = re.search(r"\bkind=([^ ]+)", line)
+            if km:
+                kind = km.group(1)
+        elif line.startswith("OCQA_FLOW_RESULT:{"):
+            try:
+                result = json.loads(line[len("OCQA_FLOW_RESULT:"):])
+                kind = result.get("kind", kind)
+            except Exception:
+                pass
+
+    total = (result or {}).get("total", len(steps))
+    failed = (result or {}).get("failed", sum(1 for s in steps if s.get("status") == "fail"))
+    passed = (result or {}).get("passed", failed == 0 and bool(steps))
+
+    executed = (result or {}).get("executed", len(steps))
+    passed_steps = sum(1 for s in steps if s.get("status") == "pass")
+
+    if as_json:
+        print(json.dumps({"name": name, "kind": kind, "passed": passed, "total": total, "executed": executed, "failed": failed, "steps": steps}))
+        return 0 if passed else 1
+
+    icon = {"pass": "✅", "fail": "❌", "skip": "⚪️"}
+    verb = {"tap": "👆 tap", "type": "⌨️  type", "swipe": "↔️  swipe", "back": "◀️  back",
+            "wait": "⏳ wait", "wait_for": "⏳ wait for", "assert_screen": "🔎 screen is",
+            "assert_exists": "🔎 exists", "assert_absent": "🔎 absent", "assert_text": "🔎 text",
+            "assert_ai": "🤖 ai"}
+    label = "RELEASE CONTRACT" if kind == "release-contract" else ("SCENARIO" if kind == "scenario" else "FLOW")
+    progress = f"{passed_steps}/{total} steps" if passed else f"{passed_steps} passed · {failed} failed · {executed}/{total} executed"
+    head = f"### {'🟢 ' + label + ' PASSED' if passed else '🔴 ' + label + ' FAILED'} — {name}  ·  {progress}"
+    print(head)
+    print("")
+    for s in steps:
+        st = s.get("status", "?")
+        label = verb.get(s.get("action", ""), s.get("action", ""))
+        tgt = s.get("target", "")
+        actor = s.get("actor", "")
+        line = f"{icon.get(st, '•')}" + (f" **{actor}**" if actor else "") + f" {label}" + (f" `{tgt}`" if tgt else "")
+        if st != "pass" and s.get("detail"):
+            line += f" — {s['detail']}"
+        print(line)
+    if not passed:
+        print("")
+        bad = next((s for s in steps if s.get("status") == "fail"), None)
+        if bad:
+            print(f"**First failure:** step {bad.get('index')} ({bad.get('action')}) — {bad.get('detail','')}")
+    return 0 if passed else 1
+
+
+def main():
+    if len(sys.argv) < 3:
+        print(__doc__)
+        return 2
+    cmd = sys.argv[1]
+    if cmd == "to-json":
+        to_json(sys.argv[2])
+        return 0
+    if cmd == "raw-json":
+        raw_json(sys.argv[2])
+        return 0
+    if cmd == "to-yaml":
+        to_yaml(sys.argv[2])
+        return 0
+    if cmd == "report":
+        as_json = "--json" in sys.argv
+        path = [a for a in sys.argv[2:] if not a.startswith("--")][0]
+        return report(path, as_json=as_json)
+    print(__doc__)
+    return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/mutation-recall-desktop.sh

@@ -0,0 +1,186 @@
+#!/usr/bin/env bash
+# Mutation-recall benchmark — DESKTOP APP EDITION.
+#
+# Same idea as mutation-recall.sh (seeded faults, differential catch, coverage-vs-detection
+# decomposition) but the system under test is the REAL desktop pipeline: each run goes through
+# `AutoTap.app --verify-run` (OrchestratorService.executeReleaseCheck — the exact code the Run
+# button calls: build → install → explore → host-side crash cross-check → vision/enrichment →
+# verdict). That matters because the Swift interpretation layer is a SEPARATE implementation
+# from the CLI/MCP marker parsing (see CLAUDE.md) — a recall number measured only via
+# quick-capture logs says nothing about what the product actually reports.
+#
+# Scoring is on the verify-run findings JSON: `type` is FindingCategory.rawValue (so harness
+# issue types are mapped, e.g. error_surface -> network_error_surface), `screen` names come from
+# the run's screensVisited, and each catching finding records its flow so advisory
+# ("Vision review") catches are reported separately from deterministic ones.
+#
+# K repeats (RUNS, default 2): exploration is nondeterministic, so the baseline is the UNION of
+# K clean runs (kills false-differentials — a flaky low-severity finding that happens to appear
+# only in a mutant run would otherwise read as a catch) and a mutant scores reached/caught if
+# ANY of its K runs did. Runs keep their artifacts (--keep-artifacts) so any surprising row can
+# be verified against the actual screenshots afterwards — never trust the marker stream alone.
+#
+# Prereqs: booted simulator; Release AutoTap.app at build/Build/Products/Release/AutoTap.app.
+# The curated mutant set below was compile-smoked; a build failure mid-run is recorded, not fatal.
+# Ends by rebuilding + reinstalling CLEAN DemoApp so the simulator isn't left with a mutant.
+set -uo pipefail
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+APP_BIN="$ROOT/build/Build/Products/Release/AutoTap.app"
+ACTIONS="${ACTIONS:-45}"
+RUNS="${RUNS:-2}"
+RUN_TIMEOUT="${RUN_TIMEOUT:-1080}"   # seconds per verify-run (build + explore + AI passes)
+
+[[ -d "$APP_BIN" ]] || { echo "❌ build AutoTap first ($APP_BIN missing)"; exit 2; }
+UDID="$(xcrun simctl list devices booted -j 2>/dev/null | python3 -c 'import sys,json;d=json.load(sys.stdin);print(next((x["udid"] for v in d["devices"].values() for x in v if x.get("state")=="Booted"),""))')"
+[[ -z "$UDID" ]] && { echo "❌ boot a simulator first"; exit 2; }
+
+WORK="${WORK_DIR:-$(mktemp -d /tmp/mutation-recall-desktop.XXXXXX)}"
+echo "▶ work dir: $WORK  (runs per variant: $RUNS, actions: $ACTIONS)"
+
+# Curated mutants: file<TAB>op<TAB>screen<TAB>expected_harness_type_or_-<TAB>in_taxonomy
+# (screen = the harness-visible nav title of the mutated site, from mutation_operators.py list)
+MUTANTS="$WORK/mutants.tsv"
+cat > "$MUTANTS" <<'EOF'
+DemoApp/Sources/CounterView.swift	dead_button	Counter	unresponsive_element	1
+DemoApp/Sources/DashboardHomeView.swift	infinite_spinner	Dashboard	app_hang	1
+DemoApp/Sources/DashboardHomeView.swift	crash_on_appear	Dashboard	crash	1
+DemoApp/Sources/SettingsView.swift	inject_error	Settings	error_surface	1
+DemoApp/Sources/TodoView.swift	off_by_one	Todo List	-	0
+DemoApp/Sources/CounterView.swift	mislabel_button	Counter	-	0
+EOF
+
+# One verify-run of the CURRENT DemoApp source tree; waits for the output JSON.
+run_verify() {  # $1 = out json path
+  local out="$1"
+  rm -f "$out"
+  pkill -f -- "--verify-run" 2>/dev/null && sleep 3
+  open -n "$APP_BIN" --args --verify-run "$ROOT/DemoApp" com.autotap.demoapp \
+    --scheme DemoApp --project "$ROOT/DemoApp/DemoApp.xcodeproj" \
+    --actions "$ACTIONS" --keep-artifacts --out "$out"
+  local waited=0
+  while [[ ! -s "$out" && $waited -lt $RUN_TIMEOUT ]]; do sleep 15; waited=$((waited+15)); done
+  if [[ ! -s "$out" ]]; then
+    echo '{"error":"timeout"}' > "$out"
+    pkill -f -- "--verify-run" 2>/dev/null
+  fi
+  sleep 5   # let the instance fully exit before the next launch
+}
+
+run_k() {  # $1 = tag; produces $WORK/<tag>_r{1..RUNS}.json
+  for r in $(seq 1 "$RUNS"); do
+    SECONDS=0
+    run_verify "$WORK/${1}_r${r}.json"
+    echo "    ${1} run $r/$RUNS done in ${SECONDS}s"
+  done
+}
+
+echo "▶ baseline: $RUNS verify-run(s) of clean DemoApp…"
+run_k baseline
+
+i=0
+while IFS=$'\t' read -r FILE OP SCREEN EXPECTED INTAX; do
+  i=$((i+1))
+  echo "▶ mutant $i: $OP @ $(basename "$FILE") [$SCREEN]"
+  cp "$ROOT/$FILE" "$ROOT/$FILE.bak"
+  if python3 "$ROOT/scripts/mutation_operators.py" apply "$ROOT/$FILE" "$OP"; then
+    run_k "mut_${i}_${OP}"
+  else
+    echo "  (operator no-op — skipped)"
+    for r in $(seq 1 "$RUNS"); do echo '{"error":"noop"}' > "$WORK/mut_${i}_${OP}_r${r}.json"; done
+  fi
+  mv "$ROOT/$FILE.bak" "$ROOT/$FILE"    # ALWAYS restore
+done < "$MUTANTS"
+
+# Leave the simulator holding a CLEAN build, not the last mutant.
+echo "▶ restoring clean DemoApp on the simulator…"
+xcodebuild -project "$ROOT/DemoApp/DemoApp.xcodeproj" -scheme DemoApp -configuration Debug \
+  -destination "platform=iOS Simulator,id=$UDID" -derivedDataPath "$WORK/clean-build" build >/dev/null 2>&1 \
+  && xcrun simctl install "$UDID" \
+       "$(find "$WORK/clean-build/Build/Products/Debug-iphonesimulator" -maxdepth 1 -name '*.app' | head -1)" \
+  && echo "  clean DemoApp reinstalled" || echo "  ⚠️ clean reinstall failed — run generate/build manually"
+
+echo
+python3 - "$WORK" "$MUTANTS" "$RUNS" <<'PY'
+import sys, json, os, glob
+
+work, mutants_path, K = sys.argv[1], sys.argv[2], int(sys.argv[3])
+# Harness issue type -> FindingCategory.rawValue (ExplorationService's OCQA_ISSUE switch).
+CAT = {"unresponsive_element": "unresponsive_element", "app_hang": "app_hang",
+       "error_surface": "network_error_surface", "blank_screen": "blank_screen", "crash": "crash"}
+
+def load(p):
+    d = json.load(open(p))
+    finds = [(f["type"], f.get("screen") or "", f.get("flow") or "", f["title"], f["severity"])
+             for f in d.get("findings", [])]
+    return d, set(d.get("screens", [])), finds
+
+# Baseline = union across K clean runs.
+base_keys, base_screens, base_ok = set(), set(), 0
+for r in range(1, K + 1):
+    d, screens, finds = load(os.path.join(work, f"baseline_r{r}.json"))
+    if "error" in d:
+        print(f"⚠️ baseline run {r} failed: {d['error']}")
+        continue
+    base_ok += 1
+    base_screens |= screens
+    base_keys |= {(t, s) for t, s, *_ in finds}
+if base_ok == 0:
+    print("❌ no successful baseline run — cannot score"); sys.exit(1)
+
+rows = []
+for i, line in enumerate(open(mutants_path).read().splitlines(), 1):
+    file, op, screen, expected, intax = line.split("\t")
+    per_run, reached_any, det_any, vis_any, catchers_all, dirs = [], False, False, False, [], []
+    verdicts = []
+    for r in range(1, K + 1):
+        p = os.path.join(work, f"mut_{i}_{op}_r{r}.json")
+        d, screens, finds = load(p)
+        if "error" in d:
+            per_run.append({"run": r, "error": d["error"]}); continue
+        reached = screen in screens
+        if expected != "-":
+            want = CAT[expected]
+            if want == "crash":   # crash screen attribution is fuzzy — category match anywhere
+                catchers = [f for f in finds if f[0] == "crash" and ("crash", f[1]) not in base_keys]
+            else:
+                catchers = [f for f in finds if f[0] == want and f[1] == screen
+                            and (f[0], f[1]) not in base_keys]
+        else:                     # out-of-taxonomy: ANY new finding at the injected screen
+            catchers = [f for f in finds if f[1] == screen and (f[0], f[1]) not in base_keys]
+        det = [c for c in catchers if c[2] != "Vision review"]
+        vis = [c for c in catchers if c[2] == "Vision review"]
+        reached_any |= reached; det_any |= bool(det); vis_any |= bool(vis)
+        catchers_all += [c for c in catchers if c not in catchers_all]
+        verdicts.append(f"{d.get('verdict')}({d.get('confidence')})")
+        if d.get("supportDir"): dirs.append(d["supportDir"])
+        per_run.append({"run": r, "reached": reached, "det": bool(det), "vis": bool(vis)})
+    rows.append({"op": op, "screen": screen, "in_tax": intax == "1",
+                 "reached": reached_any, "caught_det": det_any, "caught_vis": vis_any,
+                 "verdicts": verdicts, "per_run": per_run, "artifact_dirs": dirs,
+                 "catchers": [{"type": c[0], "screen": c[1], "flow": c[2], "title": c[3][:80],
+                               "severity": c[4]} for c in catchers_all]})
+
+print("=" * 76)
+print(f"MUTATION-RECALL — DESKTOP APP (--verify-run), K={K}, union baseline ({base_ok} ok)")
+print("=" * 76)
+print(f"baseline screens: {sorted(base_screens)}")
+print(f"baseline finding keys: {sorted(base_keys)}")
+for r in rows:
+    tag = "IN " if r["in_tax"] else "OUT"
+    catch = ("DET" if r["caught_det"] else ("VIS" if r["caught_vis"] else
+             ("miss(detect)" if r["reached"] else "miss(coverage)")))
+    runs_s = " ".join(("E" if "error" in p else f"{'R' if p['reached'] else '-'}{'D' if p['det'] else ''}{'V' if p['vis'] else ''}") for p in r["per_run"])
+    print(f"  [{tag}] {r['op']:18s} {r['screen']:12s} catch={catch:14s} runs[{runs_s}] verdicts={','.join(r['verdicts'])}")
+    for c in r["catchers"]:
+        print(f"        └─ {c['severity']}/{c['type']} [{c['flow']}] {c['title']}")
+
+def rate(xs): return f"{sum(xs)}/{len(xs)}" if xs else "n/a"
+ok = rows
+it = [r for r in ok if r["in_tax"]]; ot = [r for r in ok if not r["in_tax"]]
+print(f"\nIN-TAXONOMY   deterministic recall: {rate([r['caught_det'] for r in it])}   "
+      f"reach: {rate([r['reached'] for r in it])}")
+print(f"OUT-OF-TAXONOMY deterministic recall: {rate([r['caught_det'] for r in ot])}   "
+      f"(+vision-advisory: {rate([r['caught_vis'] for r in ot])})")
+json.dump(rows, open(os.path.join(work, "records.json"), "w"), indent=2)
+print(f"\nrecords: {work}/records.json   (artifact dirs preserved per run for visual verification)")
+PY

package/scripts/mutation-recall.sh

@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+# Mutation-recall benchmark — the HONEST recall scoreboard (docs/COMPETITIVE-MAP.md §0).
+#
+# Unlike validation-matrix.sh (which only tests bug classes we built detectors for, so it scores
+# near-100% and flatters us), this INJECTS seeded faults — including out-of-taxonomy ones we have
+# no detector for — and measures what fraction the harness actually catches, decomposed into
+# coverage (did we reach the screen?) vs detection (did we flag it once reached?).
+#
+# Loop per mutant:  back up source -> apply ONE mutation -> regenerate xcodeproj -> build ->
+#                   install -> run `explore` K times -> parse markers -> restore source.
+# Then mutation_lib.py rolls the records into the recall table.
+#
+# Cost note: one xcodebuild per mutant (~30-90s). Start SMALL: --app DemoApp --per-op 1 --runs 2.
+# Mutants are independent -> the full version fans out across simulators; this sketch is 1 sim.
+#
+# SKETCH / WIP: the source backup/restore + per-app regenerate seams are marked TODO where a
+# corpus app needs its own generator. Prereqs mirror validation-matrix.sh (booted sim + built
+# harness). Exits 0 always (it's a measurement, not a gate) unless setup is missing.
+set -uo pipefail
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+
+APP="DemoApp"; RUNS=2; PER_OP=1; ACTIONS=70; OPS_FILTER=""
+while [[ $# -gt 0 ]]; do case "$1" in
+  --app) APP="$2"; shift 2;;
+  --runs) RUNS="$2"; shift 2;;          # K repeats per mutant (non-determinism)
+  --per-op) PER_OP="$2"; shift 2;;      # sites per operator (sketch uses first match => 1)
+  --actions) ACTIONS="$2"; shift 2;;
+  --ops) OPS_FILTER="$2"; shift 2;;     # comma list, e.g. dead_button,off_by_one
+  *) echo "unknown arg $1"; exit 2;;
+esac; done
+
+SRC_DIR="$ROOT/$APP/Sources"
+BUNDLE="com.autotap.$(echo "$APP" | tr '[:upper:]' '[:lower:]')"
+[[ -d "$SRC_DIR" ]] || { echo "❌ no $SRC_DIR"; exit 2; }
+
+UDID="$(xcrun simctl list devices booted -j 2>/dev/null | python3 -c 'import sys,json;d=json.load(sys.stdin);print(next((x["udid"] for v in d["devices"].values() for x in v if x.get("state")=="Booted"),""))')"
+[[ -z "$UDID" ]] && { echo "❌ boot a simulator first"; exit 2; }
+
+WORK="$(mktemp -d)"; RECORDS="$WORK/records.json"; echo "[]" > "$RECORDS"
+trap 'rm -rf "$WORK"' EXIT
+
+# --- regenerate + build + install the CURRENT source of $APP, return 0 on success ------------
+build_install() {
+  case "$APP" in
+    DemoApp) ( cd "$ROOT" && ruby generate-demoapp-xcodeproj.rb ) >/dev/null 2>&1 ;;
+    *)       ( cd "$ROOT" && ruby generate-demo-named.rb "$APP" ) >/dev/null 2>&1 ;;  # LoginDemo/WizardDemo
+    # TODO: ShopDemo/RestaurantDemo have bespoke generators — add cases as needed.
+  esac
+  local proj="$ROOT/$APP/$APP.xcodeproj"
+  xcodebuild -project "$proj" -scheme "$APP" -configuration Debug \
+    -destination "platform=iOS Simulator,id=$UDID" -derivedDataPath "$WORK/build" build >/dev/null 2>&1 || return 1
+  local appdir; appdir="$(find "$WORK/build/Build/Products/Debug-iphonesimulator" -maxdepth 1 -name '*.app' | head -1)"
+  [[ -n "$appdir" ]] && xcrun simctl install "$UDID" "$appdir" >/dev/null 2>&1
+}
+
+# --- run explore K times against the installed build, echo the K log paths -------------------
+explore_k() {
+  local tag="$1" logs=()
+  for i in $(seq 1 "$RUNS"); do
+    local log="$WORK/${tag}_run$i.log"
+    "$ROOT/scripts/quick-capture.sh" explore "$BUNDLE" --actions "$ACTIONS" >/dev/null 2>&1
+    cp "$(ls -td "$ROOT"/captures/*/harness-output.txt 2>/dev/null | head -1)" "$log" 2>/dev/null || echo "" > "$log"
+    logs+=("$log")
+  done
+  echo "${logs[@]}"
+}
+
+# 1) BASELINE — pristine build, K runs, union of findings = "pre-existing" (never counts as caught)
+echo "▶ baseline: build + $RUNS explore run(s) of clean $APP…"
+build_install || { echo "❌ baseline build failed"; exit 2; }
+BASE_LOGS=($(explore_k baseline))
+BASE_JSON="$WORK/baseline.json"
+python3 "$ROOT/scripts/mutation_lib.py" parse "${BASE_LOGS[0]}" > "$BASE_JSON"   # sketch: 1st run; full: union K
+
+# 2) enumerate applicable mutation sites
+SITES="$WORK/sites.json"
+python3 "$ROOT/scripts/mutation_operators.py" list "$SRC_DIR" > "$SITES"
+COUNT="$(python3 -c "import json;print(len(json.load(open('$SITES'))))")"
+echo "▶ $COUNT mutation site(s) in $APP"
+
+# 3) per-mutant loop
+python3 - "$SITES" "$OPS_FILTER" <<'PY' | while IFS=$'\t' read -r FILE OP SCREEN; do
+import sys, json
+sites = json.load(open(sys.argv[1]))
+flt = set(filter(None, sys.argv[2].split(",")))
+for s in sites:
+    if flt and s["op"] not in flt: continue
+    print(f"{s['file']}\t{s['op']}\t{s['screen']}")
+PY
+  echo "  ↳ mutate $OP @ $(basename "$FILE") [$SCREEN]"
+  cp "$FILE" "$FILE.bak"                                   # back up
+  if python3 "$ROOT/scripts/mutation_operators.py" apply "$FILE" "$OP"; then
+    if build_install; then
+      MUT_LOGS=($(explore_k "mut_${OP}"))
+      # append a record: baseline + this mutant's K logs -> mutation_lib.score_mutant
+      python3 - "$ROOT/scripts" "$RECORDS" "$SITES" "$OP" "$FILE" "$BASE_JSON" "${MUT_LOGS[@]}" <<'PY'
+import sys, json
+sys.path.insert(0, sys.argv[1])          # scripts/ dir, so `import mutation_lib` resolves
+from mutation_lib import parse_log, score_mutant
+records_path, sites_path, op, file, base_path = sys.argv[2:7]
+mut_logs = sys.argv[7:]
+sites = json.load(open(sites_path))
+inj = next(s for s in sites if s["op"] == op and s["file"] == file)
+base = json.load(open(base_path))
+base_findings = set(tuple(x) for x in base["findings"])
+runs = [parse_log(p) for p in mut_logs]
+rec = score_mutant(inj, base_findings, runs)
+recs = json.load(open(records_path)); recs.append(rec); json.dump(recs, open(records_path, "w"))
+PY
+    else
+      echo "    (build failed — mutant likely uncompilable; recorded as skipped)"
+    fi
+  else
+    echo "    (operator no-op here — skipped)"
+  fi
+  mv "$FILE.bak" "$FILE"                                   # ALWAYS restore
+done
+
+# 4) restore pristine build + report
+build_install >/dev/null 2>&1
+echo; python3 "$ROOT/scripts/mutation_lib.py" report "$RECORDS"

package/scripts/mutation_lib.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+"""Parse harness logs and score mutation-recall (companion to mutation_operators.py).
+
+SKETCH / WIP. Reuses the exact marker-parse shape as validation-matrix.sh / coverage_eval_parse.py
+so "caught" means the same thing the product's regression gate means: a finding matched by
+`type|screen` (see mcp-server/src/report.js computeRegression, matched by type|screen).
+
+Two subcommands:
+  parse  <log>                       -> {"screens":[...], "findings":[["type","screen"],...]}
+  report <records.json>              -> human table + machine JSON of the recall decomposition
+"""
+import sys, re, json
+
+
+def parse_log(path):
+    """-> (set of reached screen names, set of (type, screen) finding keys)."""
+    screens, findings = set(), set()
+    for line in open(path, encoding="utf-8", errors="replace").read().splitlines():
+        m = re.search(r"OCQA_STATE:(\{.*\})", line)
+        if m:
+            try:
+                s = (json.loads(m.group(1)).get("screen") or "").strip()
+                if s and s not in ("Unknown", "pending"):
+                    screens.add(s)
+            except Exception:
+                pass
+            continue
+        m = re.search(r"OCQA_ISSUE:(\{.*\})", line)
+        if m:
+            try:
+                d = json.loads(m.group(1))
+                findings.add((d.get("type", "unknown"), (d.get("screen") or "").strip()))
+            except Exception:
+                pass
+    return screens, findings
+
+
+def score_mutant(injection, baseline_findings, runs):
+    """injection: {expected_type, screen, in_taxonomy, ...}
+       baseline_findings: set of (type,screen) seen on the CLEAN build (union across baseline runs)
+       runs: list of (screens_set, findings_set) — one per repeated run of THIS mutant
+
+    Returns a record with the coverage-vs-detection decomposition, per run and rolled up."""
+    scr, typ = injection["screen"], injection["expected_type"]
+    key = (typ, scr)
+
+    reached_any = any(scr in s for s, _ in runs)
+    # absolute: expected finding present on the mutant (in-taxonomy only — out-of-taxonomy has no
+    # expected type, so its only hope is a *differential* finding of any type at the screen).
+    def caught_abs(findings):
+        if typ is not None:
+            return key in findings
+        return any(s == scr for _, s in findings)   # any finding at the injected screen
+    # differential: caught AND not already in the clean baseline (the real gate signal)
+    def caught_diff(findings):
+        if typ is not None:
+            return key in findings and key not in baseline_findings
+        return any(s == scr and (t, s) not in baseline_findings for t, s in findings)
+
+    abs_hits = [caught_abs(f) for _, f in runs]
+    diff_hits = [caught_diff(f) for _, f in runs]
+    return {
+        **{k: injection[k] for k in ("file", "op", "bug_class", "expected_type", "in_taxonomy", "screen")},
+        "reached": reached_any,
+        "caught_abs_any": any(abs_hits), "caught_abs_all": all(abs_hits),
+        "caught_diff_any": any(diff_hits), "caught_diff_all": all(diff_hits),
+        "runs": len(runs),
+    }
+
+
+def _rate(num, den):
+    return f"{(100*num/den):5.1f}%  ({num}/{den})" if den else "   n/a"
+
+
+def report(records):
+    def bucket(recs, label):
+        n = len(recs)
+        reached = sum(r["reached"] for r in recs)
+        caught = sum(r["caught_diff_any"] for r in recs)          # headline = differential (gate signal)
+        detect = sum(r["caught_diff_any"] for r in recs if r["reached"])
+        print(f"\n{label}  (n={n})")
+        print(f"  recall (caught / all)          {_rate(caught, n)}")
+        print(f"  reach_rate (coverage)          {_rate(reached, n)}")
+        print(f"  detect|reached (detection)     {_rate(detect, reached)}")
+
+    in_tax = [r for r in records if r["in_taxonomy"]]
+    out_tax = [r for r in records if not r["in_taxonomy"]]
+
+    print("=" * 64)
+    print("MUTATION-RECALL BENCHMARK  —  differential (gate-equivalent) catch")
+    print("=" * 64)
+    bucket(records, "OVERALL")
+    bucket(in_tax, "IN-TAXONOMY   (detector exists — should be HIGH)")
+    bucket(out_tax, "OUT-OF-TAXONOMY (no detector — the honest CEILING)")
+
+    print("\nPer class (in-taxonomy):")
+    classes = {}
+    for r in in_tax:
+        classes.setdefault(r["op"], []).append(r)
+    for op, recs in sorted(classes.items()):
+        c = sum(x["caught_diff_any"] for x in recs)
+        print(f"  {op:18s} {_rate(c, len(recs))}   expects OCQA_ISSUE.type={recs[0]['expected_type']}")
+
+    print("\nMisses that REACHED the screen but weren't detected (need a detector/oracle):")
+    for r in records:
+        if r["reached"] and not r["caught_diff_any"]:
+            tag = "in-tax" if r["in_taxonomy"] else "OUT"
+            print(f"  [{tag}] {r['op']:16s} {r['screen']:16s} {r['bug_class']}")
+
+    summary = {
+        "overall_recall": sum(r["caught_diff_any"] for r in records) / max(len(records), 1),
+        "in_taxonomy_recall": sum(r["caught_diff_any"] for r in in_tax) / max(len(in_tax), 1),
+        "out_taxonomy_recall": sum(r["caught_diff_any"] for r in out_tax) / max(len(out_tax), 1),
+        "n": len(records),
+    }
+    print("\nJSON:", json.dumps(summary))
+    return summary
+
+
+if __name__ == "__main__":
+    cmd = sys.argv[1] if len(sys.argv) > 1 else ""
+    if cmd == "parse":
+        scr, fnd = parse_log(sys.argv[2])
+        print(json.dumps({"screens": sorted(scr), "findings": sorted(map(list, fnd))}))
+    elif cmd == "report":
+        report(json.load(open(sys.argv[2])))
+    else:
+        print(__doc__); sys.exit(2)