@mayurdluffy/ccwf 0.0.0-stage → 0.1.0

package/templates/skills/verify-ui/run.sh

@@ -0,0 +1,113 @@
+#!/usr/bin/env zsh
+# Run browser scenarios with agent-browser and print one PASS/FAIL line each, then a SUITE line.
+# Usage: run.sh <scenario.txt|dir> [...]     BASE_URL comes from verify-ui.conf
+# Exit 0 = all pass, 1 = a check failed, 2 = setup problem.
+#
+# Scenario lines (one per line; # comments and blank lines ignored; every expect waits up to 5s):
+#   open <path-or-url>                    navigate (a path is joined to BASE_URL)
+#   expect-text <text>                    the accessibility tree contains the text
+#   expect-no-text <text>                 the text stays absent for 1.5s
+#   expect-url <substring>                the current URL contains it
+#   expect-visible <selector>             the element is visible
+#   expect-count <selector> <n>           exactly n elements match
+#   expect-value <selector> <text>        an input holds exactly this value
+#   expect-storage <key> <substring>      localStorage[key] contains it (what was really saved)
+#   expect-request <METHOD> <url-part> [status]   the page made this request (the API was really called)
+#   expect-status <path> <code>           HTTP status of a direct request, without the browser
+#   expect-eval <js expression>           evaluates to true in the page
+#   expect-a11y [impacts]                 no axe violations of these impacts (default critical,serious)
+#   allow-console <regex>                 console errors matching this are expected in this scenario
+#   allow-request <regex>                 4xx/5xx requests matching this are expected (e.g. a 401 check)
+#   shot <name>                           save a screenshot as evidence (a drawing, layout); path is printed
+#   <anything else>                       an agent-browser command, must succeed:
+#                                         find role button click --name "Save", fill, press, wait --text,
+#                                         set viewport 390 844, set offline on, network route <url> --abort
+# Arguments with spaces: quote them where a command takes more than one (expect-value "#name" "Sam").
+#
+# After every scenario, automatically: no uncaught page errors, no console errors, no 4xx/5xx
+# request to the app (unhandled promise rejections count as page errors). On FAIL the full tree, console,
+# errors, network list and a screenshot are saved; the FAIL block prints the folder.
+. ${0:A:h}/lib.sh
+need_server
+
+files=()
+for a in "$@"; do
+  if [[ -d $a ]]; then files+=("$a"/*.txt(N)); else files+=("$a"); fi
+done
+(( ${#files} )) || { echo "SETUP: no scenario files"; exit 2; }
+
+has_text()   { tree | grep -qF -- "$1"; }
+url_has()    { [[ $(ab get url) == *$1* ]]; }
+is_visible() { [[ $(ab is visible "$1") == true ]]; }
+count_is()   { [[ $(ab get count "$1") == $2 ]]; }
+value_is()   { [[ $(ab get value "$1") == $2 ]]; }
+stored()     { ab storage local "$1" | grep -qF -- "$2"; }
+requested()  { requests | grep -F -- "] $1 " | grep -F -- "$2" | grep -qE "\) ${3:-[0-9]*}\$"; }
+js_true()    { [[ $(ab eval -b "$(print -rn -- "$1" | base64)") == true ]]; }
+
+passed=0; failed=0; S=""
+trap '[[ -n $S ]] && ab close >/dev/null 2>&1' EXIT; trap 'exit 130' INT TERM   # an interrupted run must not leave a browser behind
+for f in $files; do
+  name=${f:t:r}; S=verify-$name-$$; out=$outroot/$name
+  step=0; fail=""; okcon=""; okreq=""; shots=()
+  rm -rf "$out"
+  while IFS= read -r line || [[ -n $line ]]; do
+    [[ -z ${line// } || $line == \#* ]] && continue
+    (( step++ ))
+    cmd=${line%% *}; arg=${line#* }; w=("${(@Q)${(z)arg}}")
+    case $cmd in
+      open)           o=$(ab_open "$arg") || fail="open $arg: $(print -r -- "$o" | tail -1)" ;;
+      expect-text)    poll has_text "$arg" || fail="text not found: $arg" ;;
+      expect-no-text) for i in 1 2 3; do has_text "$arg" && { fail="unexpected text: $arg"; break; }; sleep 0.5; done ;;
+      expect-url)     poll url_has "$arg" || fail="url is $(ab get url), wanted *$arg*" ;;
+      expect-visible) poll is_visible "$arg" || fail="not visible: $arg" ;;
+      expect-count)   poll count_is "${(j: :)w[1,-2]}" "$w[-1]" || fail="count of ${(j: :)w[1,-2]} is $(ab get count "${(j: :)w[1,-2]}"), wanted $w[-1]" ;;
+      expect-value)   poll value_is "$w[1]" "${(j: :)w[2,-1]}" || fail="value of $w[1] is '$(ab get value "$w[1]")', wanted '${(j: :)w[2,-1]}'" ;;
+      expect-storage) poll stored "$w[1]" "${(j: :)w[2,-1]}" || fail="localStorage $w[1] lacks '${(j: :)w[2,-1]}': $(ab storage local "$w[1]" | short)" ;;
+      expect-request) poll requested "$w[1]" "$w[2]" "$w[3]" || fail="no $w[1] request matching '$w[2]'${w[3]:+ with status $w[3]}; api calls seen: $(api_requests | tail -3 | short | tr '\n' ' ')" ;;
+      expect-status)  got=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 "$base$w[1]")
+                      [[ $got == $w[2] ]] || fail="$w[1] returned $got, wanted $w[2]" ;;
+      expect-eval)    poll js_true "$arg" || fail="not true: $arg" ;;
+      expect-a11y)    [[ $arg == $cmd ]] && arg=critical,serious
+                      v=$(ab a11y | grep -E "^\[(${arg//,/|})\]" | short)
+                      [[ -n $v ]] && fail="accessibility: $(print -r -- "$v" | head -4 | tr '\n' ' ')" ;;
+      allow-console)  okcon=${okcon:+$okcon|}$arg ;;
+      allow-request)  okreq=${okreq:+$okreq|}$arg ;;
+      shot)           mkdir -p "$out"; o=$(ab screenshot "$out/$arg.png") && shots+=("$out/$arg.png") || fail="screenshot: $o" ;;
+      *)              o=$(eval "ab $line") || fail="'$line' failed: $(print -r -- "$o" | tail -2 | tr '\n' ' ')" ;;
+    esac
+    [[ -n $fail ]] && { fail="step $step: $fail"; break; }
+  done < "$f"
+
+  if [[ -z $fail ]]; then
+    e=$(page_errors | head -3 | short)
+    [[ -n $e ]] && fail="page errors: $e"
+  fi
+  if [[ -z $fail ]]; then
+    e=$(console_of error | grep -vE -- "${okcon:-^$}" | sort -u | head -3 | short)
+    [[ -n $e ]] && fail="console errors: $e"
+  fi
+  if [[ -z $fail ]]; then
+    e=$(bad_requests | grep -vE -- "${okreq:-^$}" | head -3 | short)
+    [[ -n $e ]] && fail="failed requests: $e"
+  fi
+
+  # Build the result first and print it after the browser is closed, so a reader that stops
+  # early (| head) cannot strand the browser.
+  if [[ -n $fail ]]; then
+    rep="FAIL $name: $fail"$'\n'"  url: $(ab get url)"$'\n'"$(itree | head -15 | short | sed 's/^/  | /')"
+    evidence "$out"; ab screenshot "$out/fail.png" >/dev/null
+    rep+=$'\n'"  evidence: $out (tree.txt console.txt errors.txt network.txt fail.png)"
+    (( failed++ ))
+  else
+    warn=$(console_of warning | wc -l | tr -d ' ')
+    (( warn )) && warn=", $warn console warnings" || warn=""
+    rep="PASS $name ($step steps$warn)"
+    (( ${#shots} )) && rep+=$'\n'"  shots: $shots"
+    (( passed++ ))
+  fi
+  ab close >/dev/null 2>&1; S=""
+  print -r -- "$rep"
+done
+echo "SUITE: $passed passed, $failed failed"
+(( failed == 0 ))