@mmerterden/multi-agent-pipeline 16.29.0 → 16.30.0

package/pipeline/scripts/run-ui-tests.sh

@@ -0,0 +1,380 @@
+#!/usr/bin/env bash
+#
+# run-ui-tests.sh  -  find the repo's own UI test target, pick the tests that
+# cover the files this task changed, and run them.
+#
+# Contract: multi-agent-refs/features/visual-evidence.md section 4 (video tier 1).
+#
+# Tier 1 of the flow video is "the repo's own UI test drove the screen". That is
+# worth more than a generated flow for one reason: the test already runs in CI,
+# so the recording shows a path somebody committed to keeping green. This script
+# is the half that finds and runs it; capture-evidence.sh records around it.
+#
+# Two sub-commands, deliberately in one file so detection has exactly one
+# implementation. The capability probe needs the same answer BEFORE the user is
+# asked which test depth to run, and a second copy of "does this repo have a UI
+# test target" is a second place for the answer to drift.
+#
+#   run-ui-tests.sh detect --platform <ios|android> [--repo <path>] [--changed <f>[,<f>...]]
+#       Print KEY=VALUE lines and exit. Never builds, never runs a test.
+#         UI_TEST_TARGET=<name>|            (empty until one candidate is chosen)
+#         UI_TEST_TARGETS=<name[,name...]> (every candidate found)
+#         UI_TEST_TARGET_REASON=<why it is empty>
+#         UI_TEST_MATCHES=<Target/Class[,Target/Class...]>|
+#         UI_TEST_MATCH_REASON=<why it is empty>
+#         UI_TEST_CONTAINER=<-project X|-workspace X>|
+#         UI_TEST_SCHEME=<name>|
+#
+#   run-ui-tests.sh run --platform <ios|android> [--repo <path>] [--changed <f>...]
+#                       [--device <udid|serial>] [--log <path>] [--all]
+#       Run the matching tests (or the whole UI suite with --all).
+#
+# Env:
+#   UI_TEST_LOG   default "$PWD/.pipeline/ui-test.log"
+#
+# Exit:
+#   0  ran, and the run passed
+#   1  ran, and the run failed
+#   2  usage / environment
+#   3  a UI test target exists but nothing matches the changed files
+#   4  no UI test target in this project
+#
+# 3 and 4 are reasons to fall to video tier 2, not failures. Only 1 is a red test.
+set -uo pipefail
+
+MODE="${1:-}"
+shift 2>/dev/null || true
+
+PLATFORM=""; REPO="$PWD"; CHANGED=""; DEVICE=""; LOG=""; RUN_ALL=0
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    --platform) PLATFORM="${2:-}"; shift 2 ;;
+    --repo) REPO="${2:-}"; shift 2 ;;
+    --changed) CHANGED="${CHANGED:+$CHANGED,}${2:-}"; shift 2 ;;
+    --device) DEVICE="${2:-}"; shift 2 ;;
+    --log) LOG="${2:-}"; shift 2 ;;
+    --all) RUN_ALL=1; shift ;;
+    *) echo "run-ui-tests: unknown option $1" >&2; exit 2 ;;
+  esac
+done
+
+case "$MODE" in detect | run) ;; *)
+  echo "usage: run-ui-tests.sh detect|run --platform <ios|android> [--repo <path>] [--changed <f>]" >&2
+  exit 2 ;;
+esac
+case "$PLATFORM" in ios | android) ;; *)
+  echo "run-ui-tests: unsupported platform '$PLATFORM'" >&2; exit 2 ;;
+esac
+[ -d "$REPO" ] || { echo "run-ui-tests: no such repo directory: $REPO" >&2; exit 2; }
+
+LOG="${LOG:-${UI_TEST_LOG:-$PWD/.pipeline/ui-test.log}}"
+
+TARGET=""; TARGET_REASON=""; TARGETS=""; SOURCES=""
+CONTAINER_ARGV=()
+MATCHES=""; MATCH_REASON=""
+CONTAINER=""; SCHEME=""
+
+# The names a changed UI file can be known by inside a UI test: the type it
+# declares, and the file's own basename. A UI test refers to a screen by its
+# accessibility identifier far more often than by its type name, and identifiers
+# are generated from the same names, so both land on the same string often enough
+# to be worth trying. This is a heuristic and the caller is told so: an empty
+# match set falls to tier 2 rather than claiming the screen is untested.
+changed_names() {
+  # The trailing newline matters: `while read` returns non-zero on an
+  # unterminated final line and drops it, so a one-element list read this way
+  # yields nothing at all and every change looks like it has no testable name.
+  printf '%s\n' "$CHANGED" | tr ',' '\n' | while IFS= read -r f; do
+    [ -n "$f" ] || continue
+    b="${f##*/}"
+    printf '%s\n' "${b%.*}"
+  done | sort -u
+}
+
+# Detection reads the filesystem, never `xcodebuild -list`. On a real app
+# workspace that call resolves the SPM graph first and took 72 seconds here,
+# and detection runs at intake while the user is waiting for a question. The
+# scheme it would have told us is only needed to RUN, so it is resolved there,
+# behind a timeout.
+resolve_ios_container() {
+  local ws proj
+  # CONTAINER is reported as text for the detect contract, but the RUN path uses
+  # CONTAINER_ARGV: a checkout under "~/My Projects/" splits an unquoted
+  # -project /path/with space into two arguments and xcodebuild is handed a
+  # project that does not exist.
+  # A standalone .xcworkspace wins, but *.xcodeproj/project.xcworkspace is the
+  # implicit one Xcode keeps inside every project. Passing that to -workspace
+  # builds a different, schemeless container, so it must never be treated as a
+  # workspace the repo chose.
+  ws=$(find "$REPO" -maxdepth 2 -name "*.xcworkspace" -not -path "*/.*" \
+       -not -path "*.xcodeproj/*" 2>/dev/null | head -1)
+  proj=$(find "$REPO" -maxdepth 2 -name "*.xcodeproj" -not -path "*/.*" 2>/dev/null | head -1)
+  if [ -n "$ws" ]; then
+    CONTAINER="-workspace $ws"
+    CONTAINER_ARGV=(-workspace "$ws")
+  elif [ -n "$proj" ]; then
+    CONTAINER="-project $proj"
+    CONTAINER_ARGV=(-project "$proj")
+  fi
+}
+
+# Prune every dotted directory, not a hand-listed few. `.worktrees` is where the
+# pipeline puts other tasks' checkouts: scanning it makes this run match a UI test
+# belonging to somebody else's branch, and the name `worktrees` in a prune list
+# does not match `.worktrees`.
+swift_sources() {
+  find "$REPO" \
+    -name ".*" -type d -prune -o \
+    -type d \( -name .build -o -name Pods -o -name build -o -name DerivedData \
+               -o -name node_modules \) -prune -o \
+    -type f -name "*.swift" -print0 2>/dev/null
+}
+
+# The signal is XCUIApplication, not a directory called *UITests. In the
+# reference app 477 files sit under a *UITests path and exactly 2 drive the UI;
+# the other 475 are snapshot tests, which render a view and compare pixels
+# without ever launching the app. Recording video around one of those produces a
+# still frame and calls it a flow. XCUIApplication is the only API that drives
+# another process's UI, which is precisely the precondition a flow recording has.
+# Computed once into SOURCES by detect_ios, never memoised inside a function: the
+# consumers read it through $(...) and a variable a subshell assigns is gone by
+# the time the parent looks. It is a repo-wide grep, and running it twice put the
+# intake probe at 13 seconds with the user waiting on a question.
+ui_test_sources() {
+  printf '%s\n' "$SOURCES" | grep -v '^$'
+}
+
+# The target is the nearest ancestor directory whose name ends in Tests - the
+# test bundle, by Apple's own template convention. Derived from the path because
+# `xcodebuild -list` costs over a minute on a real workspace and detection runs
+# while the user waits for a question.
+target_of() {
+  printf '%s\n' "$1" | awk -F/ '{
+    for (i = NF - 1; i > 0; i--)
+      if (tolower($i) ~ /tests$/) { print $i; exit }
+  }'
+}
+
+ios_targets() {
+  ui_test_sources | while IFS= read -r f; do target_of "$f"; done | sort -u
+}
+
+detect_ios() {
+  resolve_ios_container
+  [ -n "$CONTAINER" ] || { TARGET_REASON="no xcodeproj or xcworkspace under $REPO"; return; }
+
+  SOURCES=$(swift_sources | xargs -0 grep -l "XCUIApplication" 2>/dev/null)
+  TARGETS=$(ios_targets | paste -sd, -)
+  [ -n "$TARGETS" ] || { TARGET_REASON="no swift test source drives XCUIApplication under $REPO"; return; }
+
+  # One candidate is an answer on its own; several are not, until a match names
+  # one. Reporting a single target when there are several would be the same guess
+  # with a more confident face on it.
+  case "$TARGETS" in
+    *,*) TARGET="" ;;
+    *) TARGET="$TARGETS" ;;
+  esac
+
+  [ -n "$CHANGED" ] || { MATCH_REASON="no changed-file list supplied"; return; }
+
+  local names hits=""
+  names=$(changed_names)
+  [ -n "$names" ] || { MATCH_REASON="changed-file list held no usable names"; return; }
+
+  while IFS= read -r src; do
+    [ -n "$src" ] || continue
+    local tgt cls
+    tgt=$(target_of "$src")
+    [ -n "$tgt" ] || continue
+    cls=$(grep -oE 'class[[:space:]]+[A-Za-z0-9_]+' "$src" 2>/dev/null | head -1 | awk '{print $2}')
+    [ -n "$cls" ] || continue
+    while IFS= read -r n; do
+      [ -n "$n" ] || continue
+      if grep -qF "$n" "$src" 2>/dev/null; then
+        case ",$hits," in *",$tgt/$cls,"*) ;; *) hits="${hits:+$hits,}$tgt/$cls" ;; esac
+        break
+      fi
+    done <<EOF
+$names
+EOF
+  done <<EOF
+$(ui_test_sources)
+EOF
+
+  MATCHES="$hits"
+  if [ -n "$MATCHES" ]; then
+    [ -n "$TARGET" ] || TARGET="${MATCHES%%/*}"
+  else
+    MATCH_REASON="no UI test class mentions any changed file name"
+    [ -n "$TARGET" ] || TARGET_REASON="$(printf '%s\n' "$TARGETS" | tr ',' '\n' | wc -l | tr -d ' ') candidates and no match to choose between them"
+  fi
+}
+
+android_test_dirs() {
+  find "$REPO" \
+    -name ".*" -type d -prune -o \
+    -type d -name build -prune -o \
+    -type d -name "androidTest" -print 2>/dev/null
+}
+
+# module path -> Gradle task path, e.g. feature/auth/impl -> :feature:auth:impl
+android_module_of() {
+  printf '%s\n' "${1#"$REPO"/}" | sed 's#/src/androidTest.*##; s#^#:#; s#/#:#g'
+}
+
+detect_android() {
+  local dirs
+  dirs=$(android_test_dirs)
+  [ -n "$dirs" ] || { TARGET_REASON="no src/androidTest source set under $REPO"; return; }
+
+  CONTAINER="$REPO"
+  TARGETS=$(printf '%s\n' "$dirs" | while IFS= read -r d; do
+    [ -n "$d" ] || continue
+    printf '%s:connectedAndroidTest\n' "$(android_module_of "$d")"
+  done | sort -u | paste -sd, -)
+
+  # Same rule as iOS: several candidates is not an answer until a match picks
+  # one. The reference app has eight instrumentation source sets.
+  case "$TARGETS" in
+    *,*) TARGET="" ;;
+    *) TARGET="$TARGETS"; SCHEME="${TARGET%:connectedAndroidTest}" ;;
+  esac
+
+  [ -n "$CHANGED" ] || { MATCH_REASON="no changed-file list supplied"; return; }
+
+  local names hits="" hit_target=""
+  names=$(changed_names)
+  [ -n "$names" ] || { MATCH_REASON="changed-file list held no usable names"; return; }
+
+  while IFS= read -r src; do
+    [ -n "$src" ] || continue
+    local cls pkg
+    cls=$(grep -oE 'class[[:space:]]+[A-Za-z0-9_]+' "$src" 2>/dev/null | head -1 | awk '{print $2}')
+    pkg=$(grep -oE '^package[[:space:]]+[A-Za-z0-9_.]+' "$src" 2>/dev/null | head -1 | awk '{print $2}')
+    [ -n "$cls" ] || continue
+    while IFS= read -r n; do
+      [ -n "$n" ] || continue
+      if grep -qF "$n" "$src" 2>/dev/null; then
+        local fq="${pkg:+$pkg.}$cls"
+        case ",$hits," in *",$fq,"*) ;; *) hits="${hits:+$hits,}$fq" ;; esac
+        # The module that owns the matched test is the one to run.
+        [ -n "$hit_target" ] || hit_target="$(android_module_of "$src"):connectedAndroidTest"
+        break
+      fi
+    done <<EOF
+$names
+EOF
+  done <<EOF
+$(android_test_dirs | while IFS= read -r d; do find "$d" -type f \( -name "*.kt" -o -name "*.java" \) 2>/dev/null; done)
+EOF
+
+  MATCHES="$hits"
+  if [ -n "$MATCHES" ]; then
+    [ -n "$TARGET" ] || { TARGET="$hit_target"; SCHEME="${TARGET%:connectedAndroidTest}"; }
+  else
+    MATCH_REASON="no instrumentation test class mentions any changed file name"
+    [ -n "$TARGET" ] || TARGET_REASON="$(printf '%s\n' "$TARGETS" | tr ',' '\n' | wc -l | tr -d ' ') candidates and no match to choose between them"
+  fi
+}
+
+case "$PLATFORM" in
+  ios) detect_ios ;;
+  android) detect_android ;;
+esac
+
+if [ "$MODE" = "detect" ]; then
+  printf 'UI_TEST_TARGET=%s\n' "$TARGET"
+  printf 'UI_TEST_TARGETS=%s\n' "$TARGETS"
+  printf 'UI_TEST_TARGET_REASON=%s\n' "$TARGET_REASON"
+  printf 'UI_TEST_MATCHES=%s\n' "$MATCHES"
+  printf 'UI_TEST_MATCH_REASON=%s\n' "$MATCH_REASON"
+  printf 'UI_TEST_CONTAINER=%s\n' "$CONTAINER"
+  printf 'UI_TEST_SCHEME=%s\n' "$SCHEME"
+  exit 0
+fi
+
+# run
+#
+# 4 and 3 are different facts and the caller acts on them differently: 4 means
+# this repo has nothing to record a flow from, 3 means it does but nothing covers
+# what changed. Keying 4 off TARGET rather than TARGETS conflated them, because
+# TARGET is deliberately empty while several candidates exist and no match has
+# chosen between them.
+[ -n "$TARGETS$TARGET" ] || { echo "run-ui-tests: ${TARGET_REASON:-no ui test target}" >&2; exit 4; }
+if [ -z "$MATCHES" ] && [ "$RUN_ALL" -eq 0 ]; then
+  echo "run-ui-tests: ${MATCH_REASON:-no matching test}" >&2
+  exit 3
+fi
+[ -n "$TARGET" ] || {
+  echo "run-ui-tests: ${TARGET_REASON:-several candidates and no match to choose between them}" >&2
+  exit 3
+}
+
+mkdir -p "$(dirname "$LOG")"
+
+case "$PLATFORM" in
+  ios)
+    ONLY_ARGV=()
+    if [ "$RUN_ALL" -eq 0 ]; then
+      OLDIFS="$IFS"; IFS=','
+      for m in $MATCHES; do ONLY_ARGV+=("-only-testing:$m"); done
+      IFS="$OLDIFS"
+    else
+      ONLY_ARGV=("-only-testing:$TARGET")
+    fi
+    # The scheme is resolved here and not in detect: `xcodebuild -list` walks the
+    # SPM graph and can take over a minute on a real app, which detect cannot
+    # afford. Behind a timeout, because a resolver that hangs would otherwise hang
+    # the phase; on timeout fall back to the target name, which is the scheme name
+    # under Apple's own template.
+    if [ -z "$SCHEME" ]; then
+      TO=""
+      command -v timeout >/dev/null 2>&1 && TO="timeout 180"
+      # shellcheck disable=SC2086
+      LIST=$(cd "$REPO" && $TO xcodebuild -list -json "${CONTAINER_ARGV[@]}" 2>/dev/null)
+      SCHEME=$(printf '%s' "$LIST" | node -e '
+        let s=""; process.stdin.on("data",d=>s+=d).on("end",()=>{
+          let j={}; try{j=JSON.parse(s)}catch{ process.exit(0) }
+          const c=j.project||j.workspace||{};
+          const sch=c.schemes||[];
+          process.stdout.write(sch.find(n=>/uitests?$/i.test(n))||sch[0]||"");
+        });' 2>/dev/null)
+      [ -n "$SCHEME" ] || SCHEME="$TARGET"
+    fi
+
+    # `id=booted` is not a destination xcodebuild accepts, so resolve the udid
+    # rather than assigning a placeholder and hoping it is overwritten.
+    if [ -z "$DEVICE" ]; then
+      DEVICE=$(xcrun simctl list devices booted 2>/dev/null | grep -oE '[0-9A-F-]{36}' | head -1)
+      [ -n "$DEVICE" ] || { echo "run-ui-tests: no booted simulator and no --device" >&2; exit 2; }
+    fi
+    DEST="platform=iOS Simulator,id=$DEVICE"
+    (cd "$REPO" && xcodebuild test "${CONTAINER_ARGV[@]}" -scheme "$SCHEME" \
+      -destination "$DEST" "${ONLY_ARGV[@]}") >"$LOG" 2>&1
+    RC=$?
+    ;;
+  android)
+    command -v adb >/dev/null 2>&1 || { echo "run-ui-tests: adb unavailable" >&2; exit 2; }
+    adb shell true >/dev/null 2>&1 || { echo "run-ui-tests: no attached device" >&2; exit 2; }
+    GRADLE="$REPO/gradlew"
+    [ -x "$GRADLE" ] || { echo "run-ui-tests: no executable gradlew at $GRADLE" >&2; exit 2; }
+    ARGS=""
+    if [ "$RUN_ALL" -eq 0 ]; then
+      ARGS="-Pandroid.testInstrumentationRunnerArguments.class=$MATCHES"
+    fi
+    # shellcheck disable=SC2086
+    (cd "$REPO" && "$GRADLE" "$TARGET" $ARGS) >"$LOG" 2>&1
+    RC=$?
+    ;;
+esac
+
+printf 'UI_TEST_LOG=%s\n' "$LOG"
+printf 'UI_TEST_SELECTED=%s\n' "${MATCHES:-$TARGET}"
+
+# The exit code alone is not the verdict here for the same reason it is not one
+# for the build: a runner that died before it reached the tests also exits
+# non-zero, and a caller that reads only the code cannot tell "the UI test found
+# a bug" from "the simulator never booted". The log is the evidence; the caller
+# runs evidence-gate.mjs over it.
+[ "$RC" -eq 0 ] && exit 0
+exit 1

package/pipeline/skills/shared/core/multi-agent-manual-test/SKILL.md

@@ -43,5 +43,14 @@ Lets you switch to the task branch for manual testing in Xcode before the PR is
 5. **Wait for the user's response**
 
 6. **Based on the result**:
-   - **OK** → recreate the worktree, proceed to Phase 6
+   - **OK** → an "ok" is a claim, and the pass has to be substantiated before it is recorded. First write `$WORKTREE/.pipeline/manual-test.json`, one entry per acceptance criterion taken from the analysis doc test plan, the plan tasks, or the user's own words:
+     ```json
+     {"criteria":[{"spec":"<quote>","source":"analysis 15.2 | plan task 3 | user","observed":"<what was seen>","verdict":"pass|fail|not-tested","reason":"<required when not-tested>","screenshot":"<path or null>"}],"verdict":"passed|failed"}
+     ```
+     then run the gate, adding `--require-screenshot` when `state.visualEvidence.required` is true:
+     ```bash
+     node $HOME/.claude/scripts/evidence-gate.mjs --claim manual --status passed \
+       --evidence "$WORKTREE/.pipeline/manual-test.json"
+     ```
+     Exit 1 means the "ok" is not accepted: name the criterion that lacks evidence and wait for the next reply. Exit 0 → recreate the worktree, proceed to Phase 6. Full contract: `$HOME/.claude/multi-agent-refs/phases/phase-5-test.md` step 5.
    - **Fix needed** → recreate the worktree, apply the fix