@techierathore/techieflow 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/commands/TechieFlow/agents/flow-master.md +2 -0
- package/.claude/commands/TechieFlow/agents/verifier.md +2 -2
- package/.claude/commands/TechieFlow/tasks/_develop-mode.md +34 -0
- package/.claude/commands/TechieFlow/tasks/_smoke-test-policy.md +7 -5
- package/.claude/commands/TechieFlow/tasks/_yolo-mode.md +2 -2
- package/.claude/commands/TechieFlow/tasks/amend-docs.md +1 -1
- package/.claude/commands/TechieFlow/tasks/day1-brownfield.md +2 -2
- package/.claude/commands/TechieFlow/tasks/day1-greenfield.md +4 -4
- package/.claude/commands/TechieFlow/tasks/develop-end-to-end.md +30 -0
- package/.claude/commands/TechieFlow/tasks/fix-issues.md +1 -0
- package/.claude/commands/TechieFlow/tasks/mockups.md +1 -1
- package/.claude/commands/TechieFlow/tasks/triage-and-fix.md +2 -2
- package/.claude/commands/TechieFlow/tasks/verify-phase.md +12 -11
- package/.opencode/plugin/techieflow.js +15 -7
- package/.tfcore/agents/flow-master.md +2 -0
- package/.tfcore/agents/verifier.md +2 -2
- package/.tfcore/core-config.yaml +2 -1
- package/.tfcore/hooks/block-git.sh +25 -2
- package/.tfcore/hooks/guard-db.sh +106 -9
- package/.tfcore/hooks/guard-status-html.sh +5 -1
- package/.tfcore/hooks/guard-verify-deps.sh +16 -1
- package/.tfcore/hooks/yolo-allow.sh +27 -0
- package/.tfcore/standards/blazor-zoom.cs.txt +191 -0
- package/.tfcore/standards/coding-standards-dotnet.md +31 -3
- package/.tfcore/tasks/_develop-mode.md +34 -0
- package/.tfcore/tasks/_smoke-test-policy.md +7 -5
- package/.tfcore/tasks/_yolo-mode.md +2 -2
- package/.tfcore/tasks/amend-docs.md +1 -1
- package/.tfcore/tasks/day1-brownfield.md +2 -2
- package/.tfcore/tasks/day1-greenfield.md +4 -4
- package/.tfcore/tasks/develop-end-to-end.md +30 -0
- package/.tfcore/tasks/fix-issues.md +1 -0
- package/.tfcore/tasks/mockups.md +1 -1
- package/.tfcore/tasks/triage-and-fix.md +2 -2
- package/.tfcore/tasks/verify-phase.md +12 -11
- package/.tfcore/telemetry/SCHEMA.md +6 -2
- package/.tfcore/telemetry/tf-metrics.sh +19 -0
- package/.tfcore/templates/stack-defaults/dotnet.md +5 -1
- package/.tfcore/templates/v4custom/app-architecture-tmpl.md +5 -0
- package/.tfcore/templates/v4custom/app-brd-tmpl.md +2 -1
- package/.tfcore/templates/v4custom/app-checklist-tmpl.md +11 -0
- package/.tfcore/templates/v4custom/app-phases-tmpl.md +2 -1
- package/.tfcore/templates/v4custom/app-usageguide-tmpl.md +4 -1
- package/.tfcore/templates/v4custom/html-render-shell.md +2 -0
- package/.tfcore/utils/__pycache__/tf-checklist-edit.cpython-312.pyc +0 -0
- package/.tfcore/utils/__pycache__/tf-devguide-list.cpython-312.pyc +0 -0
- package/.tfcore/utils/__pycache__/tf-doc-check.cpython-312.pyc +0 -0
- package/.tfcore/utils/__pycache__/tf-split-brd.cpython-312.pyc +0 -0
- package/.tfcore/utils/__pycache__/tf-status-facts.cpython-312.pyc +0 -0
- package/.tfcore/utils/__pycache__/tf-verify-list.cpython-312.pyc +0 -0
- package/.tfcore/utils/tf-appium.mjs +179 -0
- package/.tfcore/utils/tf-brd-status.py +10 -1
- package/.tfcore/utils/tf-build-list.py +16 -1
- package/.tfcore/utils/tf-build.sh +75 -0
- package/.tfcore/utils/tf-checklist-edit.py +28 -3
- package/.tfcore/utils/tf-day1-files.py +55 -28
- package/.tfcore/utils/tf-develop.sh +264 -0
- package/.tfcore/utils/tf-devguide-list.py +6 -2
- package/.tfcore/utils/tf-doc-check.py +124 -21
- package/.tfcore/utils/tf-doc-tests.sh +40 -3
- package/.tfcore/utils/tf-emit.sh +40 -10
- package/.tfcore/utils/tf-fix-close.sh +13 -7
- package/.tfcore/utils/tf-maccatalyst-check.sh +92 -0
- package/.tfcore/utils/tf-menus.mjs +401 -0
- package/.tfcore/utils/tf-mockup-parity.mjs +361 -28
- package/.tfcore/utils/tf-mockup-parity.sh +14 -3
- package/.tfcore/utils/tf-owner-text.py +27 -5
- package/.tfcore/utils/tf-phase.sh +24 -4
- package/.tfcore/utils/tf-render-html.py +19 -3
- package/.tfcore/utils/tf-req-ids.py +36 -0
- package/.tfcore/utils/tf-stack-check.sh +3 -0
- package/.tfcore/utils/tf-status-facts.py +90 -15
- package/.tfcore/utils/tf-theme.mjs +189 -0
- package/.tfcore/utils/tf-triage.py +30 -4
- package/.tfcore/utils/tf-verify-boot.sh +296 -21
- package/.tfcore/utils/tf-verify-emit.sh +37 -17
- package/.tfcore/utils/tf-verify-list.py +219 -20
- package/.tfcore/utils/tf-verify-list.sh +3 -1
- package/.tfcore/utils/tf-verify-native.mjs +307 -0
- package/.tfcore/utils/tf-verify-screens.mjs +184 -20
- package/.tfcore/utils/tf-verify-screens.sh +17 -4
- package/.tfcore/utils/tf-verify-tests.sh +107 -17
- package/.tfcore/utils/tf-verify-verdict.py +118 -15
- package/.tfcore/utils/tf-verify-zoom.mjs +193 -0
- package/.tfcore/utils/tf-verify-zoom.sh +29 -0
- package/.tfcore/utils/tf-yolo-alias.sh +84 -0
- package/.tfcore/utils/tf-yolo-perms.py +135 -0
- package/.tfcore/utils/tf-yolo-start.sh +44 -0
- package/.tfcore/utils/tf-yolo.sh +31 -3
- package/.tfcore/utils/tf_feedback.py +86 -18
- package/.tfcore/utils/tf_stack_check.py +93 -0
- package/README.md +2 -2
- package/opencode.jsonc +6 -2
- package/package.json +1 -1
- package/scripts/install.mjs +14 -3
- package/.tfcore/utils/__pycache__/tf-brd-status.cpython-312.pyc +0 -0
- package/.tfcore/utils/__pycache__/tf-codex-bind.cpython-312.pyc +0 -0
- package/.tfcore/utils/__pycache__/tf-codex-telemetry.cpython-312.pyc +0 -0
- package/.tfcore/utils/__pycache__/tf-day1-files.cpython-312.pyc +0 -0
- package/.tfcore/utils/__pycache__/tf-log-miss.cpython-312.pyc +0 -0
- package/.tfcore/utils/__pycache__/tf-misses-md.cpython-312.pyc +0 -0
- package/.tfcore/utils/__pycache__/tf-mockups-locate.cpython-312.pyc +0 -0
- package/.tfcore/utils/__pycache__/tf-opencode-audit.cpython-312.pyc +0 -0
- package/.tfcore/utils/__pycache__/tf-render-html.cpython-312.pyc +0 -0
- package/.tfcore/utils/__pycache__/tf-triage.cpython-312.pyc +0 -0
- package/.tfcore/utils/__pycache__/tf-verify-verdict.cpython-312.pyc +0 -0
|
@@ -9,14 +9,23 @@
|
|
|
9
9
|
# # (repeatable; defaults cover one framework's id and [data-error-ui])
|
|
10
10
|
# … --route-value ProviderId=21 # a sample for /x/{ProviderId:long} (repeatable; or "route_values"
|
|
11
11
|
# # in the list); a screen whose route still needs one is SKIPped
|
|
12
|
+
# bash .tfcore/utils/tf-verify-screens.sh --list … --appium http://localhost:4723 # a MAUI Mac Catalyst head
|
|
13
|
+
# (tf-verify-native.mjs: AutomationId for data-testid, the app's window for the screenshot; the
|
|
14
|
+
# bundle id comes from boot.json or --bundle; no Playwright needed)
|
|
12
15
|
# A mockup control marked data-tf-state="<state>" (on it or a box around it) belongs to another state
|
|
13
16
|
# of the screen: it is not owed on the first view.
|
|
17
|
+
# … --themes light,dark | light | dark # default auto: light, then dark once the app shows it has a
|
|
18
|
+
# # dark theme (what it paints changes when dark is put on it)
|
|
19
|
+
# … --min-contrast 3 # the text contrast floor, ratio : 1 (Lekhak TF-024)
|
|
14
20
|
#
|
|
15
21
|
# For every screen, at 1280 and 390 px: every control the mockup anchors is present and shows
|
|
16
22
|
# something, no header-only table, no blank page, no Blazor or console error (render); nothing
|
|
17
23
|
# overlaps, nothing has zero size or sits off-screen, no sideways scroll, a stylesheet is loaded
|
|
18
|
-
# (visual)
|
|
19
|
-
#
|
|
24
|
+
# (visual); every text run reaches 3.0 : 1 against the colour painted behind it (disabled controls
|
|
25
|
+
# exempt) and, in dark mode, no box over 2% of the window paints a light panel (visual). Each theme is
|
|
26
|
+
# put on the page through prefers-color-scheme and the light/dark value of its data-*theme attributes
|
|
27
|
+
# and mode classes, and taken off again after. Overlap and off-screen are measured on the rectangle an
|
|
28
|
+
# element paints in, clipped by every scrolling ancestor, and the page is read only after its first render (--render-wait,
|
|
20
29
|
# default 5000 ms). Saves a screenshot per screen and width under tests/.artifacts/verify/screens/ and
|
|
21
30
|
# writes tests/.artifacts/verify/screens.json; that JSON is the evidence a smoke names before a
|
|
22
31
|
# row is written Implemented, and what the verdict script reads. It decides no row's status.
|
|
@@ -26,10 +35,14 @@ HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
26
35
|
# `--help` is a request and exits 0; no argument at all is a mistake and exits 3. (The one-line
|
|
27
36
|
# form this replaces computed "0--help" as its exit code and printed a bash error under the help.)
|
|
28
37
|
case "${1:-}" in
|
|
29
|
-
-h|--help) sed -n '2,
|
|
30
|
-
"") sed -n '2,
|
|
38
|
+
-h|--help) sed -n '2,32p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
|
39
|
+
"") sed -n '2,32p' "$0" | sed 's/^# \{0,1\}//'; exit 3 ;;
|
|
31
40
|
esac
|
|
32
41
|
command -v node >/dev/null 2>&1 || { echo "tf-verify-screens: node is required (bash .tfcore/utils/tf-verify-env.sh)" >&2; exit 4; }
|
|
42
|
+
case " $* " in *" --appium "*)
|
|
43
|
+
source "$HERE/tf-lock.sh"; tf_take_lock verify-screens
|
|
44
|
+
node "$HERE/tf-verify-native.mjs" "$@"; exit $? ;;
|
|
45
|
+
esac
|
|
33
46
|
node -e "import('playwright').then(()=>process.exit(0)).catch(()=>process.exit(1))" 2>/dev/null || {
|
|
34
47
|
echo "tf-verify-screens: the 'playwright' package is not resolvable from this folder; run bash .tfcore/utils/tf-verify-env.sh" >&2; exit 4; }
|
|
35
48
|
source "$HERE/tf-lock.sh"; tf_take_lock verify-screens # one browser check at a time (TF-048)
|
|
@@ -7,8 +7,10 @@
|
|
|
7
7
|
# bash .tfcore/utils/tf-verify-tests.sh --merge <tests-a.json> <tests-b.json>… [--json-out tests/.artifacts/verify/tests.json]
|
|
8
8
|
#
|
|
9
9
|
# Browser tests: `npx playwright test` over tests/verify/ (the JSON reporter). Unit tests:
|
|
10
|
-
# `tf-build.sh test` with normal console verbosity. A test belongs to a row when its name
|
|
11
|
-
# the row's id (REQ-UI-004
|
|
10
|
+
# `tf-build.sh test` with normal console verbosity. A test belongs to a row when its name STARTS with
|
|
11
|
+
# the row's id (REQ-UI-004 …; tf-req-ids.py): an id later in the name, or in a theory case's arguments,
|
|
12
|
+
# is what the test is about, not what it proves, and a row only named that way is listed under
|
|
13
|
+
# named_only_by_other_rows_tests (Chatur TF-012). A row is PASS when a test carrying its id passed and none failed,
|
|
12
14
|
# FAIL when one failed (the reason and the screenshot, when the reporter kept one, are recorded),
|
|
13
15
|
# NOT-TESTED when every test carrying its id was SKIPPED, and absent when no test carries its id.
|
|
14
16
|
# A skipped clause is a third outcome, never a failure: `test.skip(!SEEDED, …)` says the state does
|
|
@@ -20,6 +22,9 @@
|
|
|
20
22
|
# When tf-verify-list.sh was given a list of ids, the browser run keeps only the tests carrying them
|
|
21
23
|
# (`--grep`), and is skipped when no row has a screen and no browser test names one (Lekhak TF-017).
|
|
22
24
|
# `--list` names another list.json; `--all-specs` runs every browser test whatever the scope.
|
|
25
|
+
# A desktop head's debugging address (--base answering /json/version, or boot.json mode=cdp) also
|
|
26
|
+
# reaches the tests as CDP_URL; a suite that reads it nowhere is refused, not run (Lekhak TF-021).
|
|
27
|
+
# A Mac Catalyst head (boot.json mode=appium) reaches them as APPIUM_URL, TF_BUNDLE_ID and TF_APP_PATH.
|
|
23
28
|
# Exit 0 ran (whatever the results) · 2 nothing could run.
|
|
24
29
|
set -uo pipefail
|
|
25
30
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
@@ -37,7 +42,7 @@ while [[ $# -gt 0 ]]; do
|
|
|
37
42
|
--all-specs) ALLSPECS=1; shift ;;
|
|
38
43
|
--merge) shift; while [[ $# -gt 0 && "$1" != --* ]]; do MERGE+=("$1"); shift; done ;;
|
|
39
44
|
--json-out) OUT="${2:-}"; shift 2 ;;
|
|
40
|
-
-h|--help) sed -n '2,
|
|
45
|
+
-h|--help) sed -n '2,26p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
|
41
46
|
*) echo "tf-verify-tests: unknown argument $1" >&2; exit 2 ;;
|
|
42
47
|
esac
|
|
43
48
|
done
|
|
@@ -51,8 +56,11 @@ mkdir -p "$(dirname "$OUT")" tests/.artifacts/verify
|
|
|
51
56
|
|
|
52
57
|
# ---- --merge: the parts into one tests.json, no test run ------------------------------------------
|
|
53
58
|
if [[ ${#MERGE[@]} -gt 0 ]]; then
|
|
54
|
-
TF_OUT="$OUT" python3 - "${MERGE[@]}" <<'PY'
|
|
55
|
-
import json, os, sys
|
|
59
|
+
TF_HERE="$HERE" TF_OUT="$OUT" python3 - "${MERGE[@]}" <<'PY'
|
|
60
|
+
import json, os, re, sys
|
|
61
|
+
import importlib.util
|
|
62
|
+
_spec = importlib.util.spec_from_file_location("tf_req_ids", os.path.join(os.environ.get("TF_HERE", ".tfcore/utils"), "tf-req-ids.py"))
|
|
63
|
+
rq = importlib.util.module_from_spec(_spec); _spec.loader.exec_module(rq)
|
|
56
64
|
# Lekhak TF-007. The parts are read oldest first (by the time each ran, else in the order given), and
|
|
57
65
|
# a later run of the SAME test stands for it: a test that failed because its service was not started,
|
|
58
66
|
# run again once it was, clears the row. A later skip never replaces a run that happened. A part
|
|
@@ -65,7 +73,7 @@ for i, p in enumerate(sys.argv[1:]):
|
|
|
65
73
|
print(f"merge: {p} unreadable ({str(e)[:80]}); skipped")
|
|
66
74
|
loaded.sort(key=lambda x: (x[0].get("ran_at") or "", x[1]) if all(y[0].get("ran_at") for y in loaded) else (x[1],))
|
|
67
75
|
reqs, browser, unit, parts = {}, {"ran": False, "passed": 0, "failed": 0, "skipped": 0, "tests": 0}, {"ran": False, "line": ""}, []
|
|
68
|
-
outs, sources, superseded = {}, {}, 0
|
|
76
|
+
outs, sources, superseded, elsewhere = {}, {}, 0, {}
|
|
69
77
|
for d, _i, p in loaded:
|
|
70
78
|
parts.append(p)
|
|
71
79
|
for rid, r in d.get("reqs", {}).items():
|
|
@@ -76,6 +84,17 @@ for d, _i, p in loaded:
|
|
|
76
84
|
oc.update({t: {"outcome": "skip", "reason": r.get("reason", ""), "screenshot": ""} for t in r.get("skipped", [])})
|
|
77
85
|
o = outs.setdefault(rid, {})
|
|
78
86
|
for name, rec in oc.items():
|
|
87
|
+
# a part read before Chatur TF-012 counted a test for every id its name carried: a test that
|
|
88
|
+
# does not claim this row is dropped here, and the row noted if that test passed
|
|
89
|
+
# (a browser test's name was stored with its describe titles joined on, so there each title is read)
|
|
90
|
+
segs = re.split(r"\s+(?=REQ-)", name) if rec.get("source", r.get("source")) == "browser" else [name]
|
|
91
|
+
if not any(rid in rq.ids_of(seg) for seg in segs):
|
|
92
|
+
if rec.get("outcome") == "pass":
|
|
93
|
+
elsewhere.setdefault(rid, []).append(name[:160])
|
|
94
|
+
continue
|
|
95
|
+
# a part written before outcomes carried their source: the row's, when it had only one
|
|
96
|
+
if "source" not in rec:
|
|
97
|
+
rec = dict(rec, source=r.get("source") if r.get("source") in ("browser", "unit") else "")
|
|
79
98
|
old = o.get(name)
|
|
80
99
|
if old and rec["outcome"] == "skip" and old["outcome"] != "skip":
|
|
81
100
|
continue
|
|
@@ -92,16 +111,29 @@ for d, _i, p in loaded:
|
|
|
92
111
|
u = d.get("unit", {})
|
|
93
112
|
if u.get("ran") and not unit["ran"]:
|
|
94
113
|
unit = u
|
|
114
|
+
for d, _i, p in loaded:
|
|
115
|
+
for rid, names in (d.get("named_only_by_other_rows_tests") or {}).items():
|
|
116
|
+
elsewhere.setdefault(rid, []).extend(names)
|
|
95
117
|
for rid, o in outs.items():
|
|
118
|
+
if not o:
|
|
119
|
+
continue
|
|
96
120
|
fails = [(n, x) for n, x in o.items() if x["outcome"] == "fail"]
|
|
97
121
|
npass = sum(1 for x in o.values() if x["outcome"] == "pass")
|
|
98
122
|
skips = [(n, x) for n, x in o.items() if x["outcome"] == "skip"]
|
|
99
123
|
result = "FAIL" if fails else "PASS" if npass else "NOT-TESTED"
|
|
100
124
|
reason = fails[0][1]["reason"] if fails else (skips[0][1]["reason"] if skips and not npass else "")
|
|
125
|
+
# Sevak TF-004: a unit pass does not stand in for a row's on-app tests when they were all skipped
|
|
126
|
+
app = [x for x in o.values() if x.get("source") == "browser"]
|
|
127
|
+
if result == "PASS" and app and all(x["outcome"] == "skip" for x in app):
|
|
128
|
+
why = next((x["reason"] for x in app if x["reason"]), "the test was skipped")
|
|
129
|
+
result = "NOT-TESTED"
|
|
130
|
+
reason = ("on-app test skipped (" + why + "); a unit test does not measure on-app acceptance")[:200]
|
|
101
131
|
reqs[rid] = {"result": result, "source": sources.get(rid, ""), "tests": [n for n, x in o.items() if x["outcome"] != "skip"],
|
|
102
132
|
"skipped": [n for n, _ in skips], "passed": npass, "failed": len(fails), "reason": reason,
|
|
103
133
|
"screenshot": fails[0][1].get("screenshot", "") if fails else "", "outcomes": o}
|
|
104
|
-
|
|
134
|
+
mentioned = {rid: sorted(set(v))[:3] for rid, v in elsewhere.items() if rid not in reqs}
|
|
135
|
+
json.dump({"reqs": reqs, "browser": browser, "unit": unit, "merged_from": parts, "superseded": superseded,
|
|
136
|
+
"named_only_by_other_rows_tests": mentioned},
|
|
105
137
|
open(os.environ["TF_OUT"], "w"), indent=1)
|
|
106
138
|
p = sum(1 for r in reqs.values() if r["result"] == "PASS"); f = sum(1 for r in reqs.values() if r["result"] == "FAIL")
|
|
107
139
|
ns = sum(1 for r in reqs.values() if r["result"] == "NOT-TESTED")
|
|
@@ -111,6 +143,8 @@ print(f"merged {len(parts)} part(s): rows with a test: {len(reqs)} — {p} PASS,
|
|
|
111
143
|
for rid, r in sorted(reqs.items()):
|
|
112
144
|
if r["result"] == "FAIL":
|
|
113
145
|
print(f" FAIL {rid} — {r['reason']}" + (f" — {r['screenshot']}" if r["screenshot"] else ""))
|
|
146
|
+
for rid, names in sorted(mentioned.items()):
|
|
147
|
+
print(f" NO TEST {rid} — only named inside another row's passing test, which does not test it: {names[0]}")
|
|
114
148
|
print(f"JSON: {os.environ['TF_OUT']}")
|
|
115
149
|
PY
|
|
116
150
|
exit 0
|
|
@@ -121,6 +155,25 @@ PWJSON="tests/.artifacts/verify/playwright${PART:+-$PART}.json"; PWLOG="tests/.a
|
|
|
121
155
|
ran_any=0
|
|
122
156
|
|
|
123
157
|
if [[ $BROWSER -eq 1 ]]; then
|
|
158
|
+
# A desktop head is reached over its debugging address (tf-verify-boot.sh prints mode=cdp). The tests
|
|
159
|
+
# get it as CDP_URL as well as BASE_URL: Lekhak's helper fell back to its own default port and every
|
|
160
|
+
# desktop test failed after ten minutes (Lekhak TF-021). --base is that address when it answers
|
|
161
|
+
# /json/version; without --base, boot.json's url when its mode is cdp.
|
|
162
|
+
CDPURL=""
|
|
163
|
+
if [[ -n "$BASE" ]]; then
|
|
164
|
+
curl -s -m 3 "${BASE%/}/json/version" 2>/dev/null | grep -q webSocketDebuggerUrl && CDPURL="${BASE%/}"
|
|
165
|
+
elif [[ -f tests/.artifacts/verify/boot.json ]]; then
|
|
166
|
+
CDPURL="$(python3 -c 'import json,sys
|
|
167
|
+
b = json.load(open(sys.argv[1]))
|
|
168
|
+
print(b.get("url", "") if b.get("mode") == "cdp" and not b.get("stopped") else "")' tests/.artifacts/verify/boot.json 2>/dev/null)"
|
|
169
|
+
[[ -n "$CDPURL" ]] && BASE="$CDPURL"
|
|
170
|
+
fi
|
|
171
|
+
# A Mac Catalyst head (boot.json mode=appium) is reached through Appium: its tests get APPIUM_URL
|
|
172
|
+
# TF_BUNDLE_ID and TF_APP_PATH, and open their session with .tfcore/utils/tf-appium.mjs.
|
|
173
|
+
APPIUMURL=""; BUNDLEID=""; APPPATH=""
|
|
174
|
+
[[ -f tests/.artifacts/verify/boot.json ]] && read -r APPIUMURL BUNDLEID APPPATH < <(python3 -c 'import json,sys
|
|
175
|
+
b = json.load(open(sys.argv[1]))
|
|
176
|
+
print(b.get("url", ""), b.get("bundle_id", ""), b.get("app_path", "")) if b.get("mode") == "appium" and not b.get("stopped") else print()' tests/.artifacts/verify/boot.json 2>/dev/null)
|
|
124
177
|
if ls tests/verify/*.spec.* >/dev/null 2>&1 || ls tests/verify/**/*.spec.* >/dev/null 2>&1; then
|
|
125
178
|
# --base reaches the tests only as BASE_URL, which Playwright never reads by itself. When neither
|
|
126
179
|
# the config nor a spec reads it, every test opens the config's own address, where an older build
|
|
@@ -128,6 +181,9 @@ if [[ $BROWSER -eq 1 ]]; then
|
|
|
128
181
|
if [[ -n "$BASE" ]] && ! grep -qs "BASE_URL" playwright.config.* tests/verify/*.spec.* tests/verify/**/*.spec.*; then
|
|
129
182
|
echo "browser tests: NOT RUN — nothing reads BASE_URL, so the tests would open another address than --base $BASE; run bash .tfcore/utils/tf-verify-env.sh, which makes playwright.config.ts read it"
|
|
130
183
|
rm -f "$PWJSON"
|
|
184
|
+
elif [[ -n "$CDPURL" ]] && ! grep -qsE "env(\.|\[['\"])CDP_URL" playwright.config.* tests/verify/*.ts tests/verify/**/*.ts; then
|
|
185
|
+
echo "browser tests: NOT RUN — $CDPURL is a desktop app's debugging address and no test reads CDP_URL, so they would attach to another address; make the connectOverCDP helper read process.env.CDP_URL first"
|
|
186
|
+
rm -f "$PWJSON"
|
|
131
187
|
elif node -e "require.resolve('@playwright/test')" >/dev/null 2>&1; then
|
|
132
188
|
PWARGS=(--reporter=json); [[ -n "$SHARD" ]] && PWARGS+=("--shard=$SHARD")
|
|
133
189
|
# (the block below ends in SCOPEPY, not PY: tests/regression lifts the PY blocks out verbatim)
|
|
@@ -155,7 +211,7 @@ SCOPEPY
|
|
|
155
211
|
rm -f "$PWJSON"
|
|
156
212
|
else
|
|
157
213
|
[[ -n "$SCOPE" ]] && PWARGS+=(--grep "${SCOPE#* }")
|
|
158
|
-
BASE_URL="$BASE" PLAYWRIGHT_JSON_OUTPUT_NAME="$PWJSON" npx playwright test "${PWARGS[@]}" ${SPECS[@]+"${SPECS[@]}"} > "$PWLOG" 2>&1
|
|
214
|
+
APPIUM_URL="$APPIUMURL" TF_BUNDLE_ID="$BUNDLEID" TF_APP_PATH="$APPPATH" CDP_URL="$CDPURL" BASE_URL="$BASE" PLAYWRIGHT_JSON_OUTPUT_NAME="$PWJSON" npx playwright test "${PWARGS[@]}" ${SPECS[@]+"${SPECS[@]}"} > "$PWLOG" 2>&1
|
|
159
215
|
echo "browser tests: ran${SHARD:+ shard $SHARD}${SPECS:+ (${#SPECS[@]} spec argument(s))}${SCOPE:+ only the tests carrying ${SCOPE#* } (the scope in $LIST)} (log $PWLOG)"; ran_any=1
|
|
160
216
|
fi
|
|
161
217
|
else
|
|
@@ -209,7 +265,9 @@ PY
|
|
|
209
265
|
--report-*) UARGS+=("-p:TestingPlatformCommandLineArguments=$MTP") ;;
|
|
210
266
|
mixed) echo "unit tests: note the Testing Platform projects here take different report switches, so none was passed; their tests are not mapped to rows" ;;
|
|
211
267
|
esac
|
|
212
|
-
|
|
268
|
+
# the stamp sits on the project's own drive: one in /tmp takes its time from WSL's clock and the
|
|
269
|
+
# report on /mnt/c from Windows', which drift apart, so a fresh report could read as older (2026-10-02)
|
|
270
|
+
mkdir -p tests/.artifacts; STAMP="$(mktemp tests/.artifacts/.unit-stamp.XXXXXX)"
|
|
213
271
|
UNITLINE="$(bash "$HERE/tf-build.sh" test ${TARGET:+"$TARGET"} -- "${UARGS[@]}" 2>&1 | grep -m1 -E '^(PASS|FAIL|NOT-RUN)' || true)"
|
|
214
272
|
UNITLOG="$(sed -n 's/.*log \(tests\/\.artifacts\/build\/[^ ;]*\).*/\1/p' <<<"$UNITLINE" | head -1)"
|
|
215
273
|
TRX="$(find . -name '*.trx' -newer "$STAMP" -not -path '*/node_modules/*' 2>/dev/null)"; rm -f "$STAMP"
|
|
@@ -219,10 +277,23 @@ PY
|
|
|
219
277
|
fi
|
|
220
278
|
fi
|
|
221
279
|
|
|
222
|
-
TF_PWJSON="$PWJSON" TF_UNITLOG="$UNITLOG" TF_UNITLINE="$UNITLINE" TF_TRX="${TRX:-}" TF_OUT="$OUT" python3 - <<'PY'
|
|
280
|
+
TF_HERE="$HERE" TF_PWJSON="$PWJSON" TF_UNITLOG="$UNITLOG" TF_UNITLINE="$UNITLINE" TF_TRX="${TRX:-}" TF_OUT="$OUT" python3 - <<'PY'
|
|
223
281
|
import json, os, re
|
|
224
282
|
import xml.etree.ElementTree as ET
|
|
225
|
-
|
|
283
|
+
import importlib.util
|
|
284
|
+
_spec = importlib.util.spec_from_file_location("tf_req_ids", os.path.join(os.environ.get("TF_HERE", ".tfcore/utils"), "tf-req-ids.py"))
|
|
285
|
+
rq = importlib.util.module_from_spec(_spec); _spec.loader.exec_module(rq)
|
|
286
|
+
ID, ids_of = rq.ID, rq.ids_of # Chatur TF-012: a test claims the ids its name starts with (tf-req-ids.py)
|
|
287
|
+
elsewhere = {} # row -> names of passing tests that mention it without claiming it
|
|
288
|
+
def claims(names, outcome):
|
|
289
|
+
"""the ids a test claims: the leading run of each name given (a browser test's describe titles and
|
|
290
|
+
its own title are separate names); the other ids it mentions are noted against those rows"""
|
|
291
|
+
own = {i for n in names for i in ids_of(n)}
|
|
292
|
+
for n in names:
|
|
293
|
+
for rid in {x.upper() for x in ID.findall(n)} - own:
|
|
294
|
+
if outcome == "pass":
|
|
295
|
+
elsewhere.setdefault(rid, []).append(" ".join(names).strip()[:160])
|
|
296
|
+
return own
|
|
226
297
|
reqs = {}
|
|
227
298
|
browser = {"ran": False, "passed": 0, "failed": 0, "skipped": 0, "tests": 0}
|
|
228
299
|
unit = {"ran": False, "line": os.environ.get("TF_UNITLINE", "")}
|
|
@@ -240,8 +311,11 @@ def add(rid, outcome, name, source, reason="", shot=""):
|
|
|
240
311
|
r = reqs.setdefault(rid.upper(), {"result": "NOT-TESTED", "source": source, "tests": [],
|
|
241
312
|
"skipped": [], "passed": 0, "failed": 0,
|
|
242
313
|
"reason": "", "screenshot": "", "outcomes": {}})
|
|
243
|
-
# each test's own outcome, so a merge can let a later run of the same test stand (Lekhak TF-007)
|
|
244
|
-
|
|
314
|
+
# each test's own outcome, so a merge can let a later run of the same test stand (Lekhak TF-007),
|
|
315
|
+
# and where it ran, so a unit pass never stands in for a skipped on-app test (Sevak TF-004)
|
|
316
|
+
r["outcomes"][name] = {"outcome": outcome, "reason": (reason or "")[:200], "screenshot": shot, "source": source}
|
|
317
|
+
if r["source"] != source:
|
|
318
|
+
r["source"] = "browser+unit"
|
|
245
319
|
if outcome == "skip":
|
|
246
320
|
r["skipped"].append(name)
|
|
247
321
|
if not r["reason"]:
|
|
@@ -292,7 +366,7 @@ if pw and os.path.isfile(pw):
|
|
|
292
366
|
shot = a["path"].replace("\\", "/")
|
|
293
367
|
shot = shot[shot.find("tests/"):] if "tests/" in shot else shot
|
|
294
368
|
break
|
|
295
|
-
for rid in
|
|
369
|
+
for rid in claims(path + [sp.get("title", "")], outcome):
|
|
296
370
|
add(rid, outcome, title.strip(), "browser", err, shot)
|
|
297
371
|
for s in suite.get("suites", []):
|
|
298
372
|
walk(s, path + [s.get("title", "")] if s.get("title") and not s["title"].endswith(".ts") and not s["title"].endswith(".js") and not s["title"].endswith(".mjs") else path)
|
|
@@ -320,7 +394,7 @@ for trx in [p for p in os.environ.get("TF_TRX", "").splitlines() if p.strip()]:
|
|
|
320
394
|
continue
|
|
321
395
|
seen.add((name, outcome))
|
|
322
396
|
msg = next(((m.text or "").strip() for m in el.iter() if m.tag.endswith("Message") and (m.text or "").strip()), "")
|
|
323
|
-
for rid in
|
|
397
|
+
for rid in claims([name], outcome):
|
|
324
398
|
add(rid, outcome, name, "unit",
|
|
325
399
|
("unit test failed: " + (msg.splitlines()[0] if msg else name))[:200] if outcome == "fail"
|
|
326
400
|
else ("unit test skipped: " + name[:120]) if outcome == "skip" else "")
|
|
@@ -341,17 +415,31 @@ if ul and os.path.isfile(ul):
|
|
|
341
415
|
continue
|
|
342
416
|
seen.add((name, outcome))
|
|
343
417
|
# a pass has no reason: every non-failure was once written "unit test skipped" (Lekhak TF-018)
|
|
344
|
-
for rid in
|
|
418
|
+
for rid in claims([name], outcome):
|
|
345
419
|
add(rid, outcome, name, "unit",
|
|
346
420
|
"unit test failed: " + name[:120] if outcome == "fail"
|
|
347
421
|
else "unit test skipped: " + name[:120] if outcome == "skip" else "")
|
|
422
|
+
# Sevak TF-004: a row that has an on-app test is accepted on the app. When every on-app test of the
|
|
423
|
+
# row was skipped (`test.skip()`: not observable on this host) and none failed, a passing unit test
|
|
424
|
+
# with the same id does not measure it: the row is NOT-TESTED, with the skip's reason. A failure, from
|
|
425
|
+
# either side, still makes it FAIL; a row with no on-app test is graded on its unit tests as before.
|
|
426
|
+
for rid, r in reqs.items():
|
|
427
|
+
app = [x for x in r["outcomes"].values() if x.get("source") == "browser"]
|
|
428
|
+
if app and all(x["outcome"] == "skip" for x in app) and r["result"] == "PASS":
|
|
429
|
+
why = next((x["reason"] for x in app if x["reason"]), "the test was skipped")
|
|
430
|
+
r["result"] = "NOT-TESTED"
|
|
431
|
+
r["reason"] = ("on-app test skipped (" + why + "); a unit test does not measure on-app acceptance")[:200]
|
|
432
|
+
# a row no test claims, whose id a passing test about another row carries, is flagged: it was once
|
|
433
|
+
# counted as tested on that name, and it is not
|
|
434
|
+
mentioned = {rid: sorted(set(v))[:3] for rid, v in elsewhere.items() if rid not in reqs}
|
|
348
435
|
unit["passed"] = sum(1 for r in reqs.values() if r["source"] == "unit" and r["result"] == "PASS")
|
|
349
436
|
unit["failed"] = sum(1 for r in reqs.values() if r["source"] == "unit" and r["result"] == "FAIL")
|
|
350
437
|
unit["not_tested"] = sum(1 for r in reqs.values() if r["source"] == "unit" and r["result"] == "NOT-TESTED")
|
|
351
438
|
|
|
352
439
|
import datetime
|
|
353
440
|
ran_at = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
354
|
-
json.dump({"reqs": reqs, "browser": browser, "unit": unit, "ran_at": ran_at
|
|
441
|
+
json.dump({"reqs": reqs, "browser": browser, "unit": unit, "ran_at": ran_at, "named_only_by_other_rows_tests": mentioned},
|
|
442
|
+
open(os.environ["TF_OUT"], "w"), indent=1)
|
|
355
443
|
p = sum(1 for r in reqs.values() if r["result"] == "PASS")
|
|
356
444
|
f = sum(1 for r in reqs.values() if r["result"] == "FAIL")
|
|
357
445
|
ns = sum(1 for r in reqs.values() if r["result"] == "NOT-TESTED")
|
|
@@ -362,6 +450,8 @@ for rid, r in sorted(reqs.items()):
|
|
|
362
450
|
print(f" FAIL {rid} — {r['reason']}" + (f" — {r['screenshot']}" if r["screenshot"] else ""))
|
|
363
451
|
elif r["result"] == "NOT-TESTED":
|
|
364
452
|
print(f" NOT-TESTED {rid} — {len(r['skipped'])} test(s) skipped, none ran — {r['reason']}")
|
|
453
|
+
for rid, names in sorted(mentioned.items()):
|
|
454
|
+
print(f" NO TEST {rid} — only named inside another row's passing test, which does not test it: {names[0]}")
|
|
365
455
|
print(f"JSON: {os.environ['TF_OUT']}")
|
|
366
456
|
PY
|
|
367
457
|
[[ $ran_any -eq 1 ]] && exit 0 || exit 2
|
|
@@ -14,7 +14,10 @@ For each row, the seven checks in order, and the first that fails is the verdict
|
|
|
14
14
|
render every anchored control on its screen shows something — screens.json
|
|
15
15
|
assets every stylesheet and script its screen declares arrived — assets.json
|
|
16
16
|
visual nothing overlaps, nothing clipped or off-screen — screens.json
|
|
17
|
-
mockup the screen carries the structure its mockup draws — parity.json
|
|
17
|
+
mockup the screen carries the structure its mockup draws — parity.json, found by the screen's
|
|
18
|
+
name, its mockup's file name or its route ({id} in the row's route = one path segment, so
|
|
19
|
+
/processes/6/run counts for /processes/{id}/run); a driven screen with a mockup and no
|
|
20
|
+
comparison found FAILs this check, it is never skipped (Chatur TF-008)
|
|
18
21
|
speed within the row's perf-budget, only when it declares one — perf/<REQ>.json
|
|
19
22
|
(standards has no script yet and is never listed as run)
|
|
20
23
|
|
|
@@ -42,6 +45,7 @@ import sys
|
|
|
42
45
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
43
46
|
sys.path.insert(0, HERE)
|
|
44
47
|
import tf_defect # noqa: E402
|
|
48
|
+
import tf_stack_check # noqa: E402
|
|
45
49
|
FAILING = {"BUILD-FAIL": "build", "FAIL": "acceptance", "RENDER-FAIL": "render", "ASSET-FAIL": "assets",
|
|
46
50
|
"VISUAL-FAIL": "visual", "MOCKUP-FAIL": "mockup-parity", "PERF-FAIL": "perf"}
|
|
47
51
|
STATUS = {"PASS": "Verified", "FAIL": "FAIL", "BUILD-FAIL": "FAIL", "RENDER-FAIL": "Needs re-verify", "ASSET-FAIL": "Needs re-verify",
|
|
@@ -75,6 +79,34 @@ def norm_route(r):
|
|
|
75
79
|
return r or "/"
|
|
76
80
|
|
|
77
81
|
|
|
82
|
+
def route_matches(pattern, route):
|
|
83
|
+
"""A filled-in route against its pattern: /processes/6/run matches /processes/{id}/run, one path
|
|
84
|
+
segment per {…} (Chatur TF-008). A pattern with no {…} matches only itself. A query string on the
|
|
85
|
+
opened route (/start?all=1) is not part of the screen."""
|
|
86
|
+
p, r = norm_route(pattern), norm_route((route or "").split("?")[0].split("#")[0])
|
|
87
|
+
if "{" not in p:
|
|
88
|
+
return p == r
|
|
89
|
+
rx = "".join("[^/]+" if part.startswith("{") else re.escape(part) for part in re.split(r"(\{[^}]*\})", p))
|
|
90
|
+
return re.fullmatch(rx, r) is not None
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def stem(path):
|
|
94
|
+
return os.path.basename(path or "")[:-5].lower() if (path or "").lower().endswith(".html") else ""
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def parity_for(r, mockup, comparisons):
|
|
98
|
+
"""The mockup comparison that graded a row's screen: named for the screen, for its mockup file
|
|
99
|
+
(the Mockup parity: line names each screen by its mockup's stem), or opened at the screen's route,
|
|
100
|
+
a {value} in it standing for any one segment. The worst one when several match: one per value
|
|
101
|
+
(Chatur TF-008)."""
|
|
102
|
+
screen, route = (r.get("screen") or "").lower(), r.get("route") or ""
|
|
103
|
+
hits = [s for s in comparisons
|
|
104
|
+
if (s.get("screen") or "").lower() in (screen, stem(mockup))
|
|
105
|
+
or (route and s.get("route") and route_matches(route, s.get("route")))]
|
|
106
|
+
order = {"FAIL": 0, "ERROR": 1, "UNGRADEABLE": 2, "NO-MOCKUP": 3, "PASS": 4}
|
|
107
|
+
return min(hits, key=lambda s: order.get(s.get("verdict"), 5)) if hits else None
|
|
108
|
+
|
|
109
|
+
|
|
78
110
|
def quoted(s, n=None):
|
|
79
111
|
"""A test's own words — its title or its failure message — in backticks, so the Remark shows them as
|
|
80
112
|
quoted and tf-doc-check reads them as quoted: a title saying "Found/Not found" failed the checker's
|
|
@@ -109,12 +141,20 @@ def main(argv):
|
|
|
109
141
|
assets = load(os.path.join(d, "assets.json")) or {}
|
|
110
142
|
pages = {norm_route(p.get("path")): p for p in assets.get("pages", [])}
|
|
111
143
|
parity = load(os.path.join(d, "parity.json")) or {}
|
|
112
|
-
|
|
113
|
-
|
|
144
|
+
comparisons = parity.get("screens", []) or []
|
|
145
|
+
# the mockup each listed screen is graded against; a row's own *Mockup:* line when its screen has none
|
|
146
|
+
mock_of = {s.get("name"): s.get("mockup") or "" for s in lst.get("screens", []) or []}
|
|
114
147
|
grade = perf_grader()
|
|
115
148
|
|
|
116
149
|
booted = boot.get("mode") not in (None, "none")
|
|
117
150
|
build_failed = boot.get("reason_kind") == "build-error"
|
|
151
|
+
# the packages the Architecture requires are in the code (tf_stack_check.py): a model once replaced the
|
|
152
|
+
# mandated UI library with plain components and had every row Verified
|
|
153
|
+
stack = tf_stack_check.check(os.getcwd(), app)
|
|
154
|
+
if stack["verdict"] == "FAIL":
|
|
155
|
+
build_failed = True
|
|
156
|
+
boot = dict(boot, reason=f"the Architecture requires {', '.join(stack['missing'])} and no project file references "
|
|
157
|
+
+ ("it" if len(stack["missing"]) == 1 else "them") + " (tf-stack-check.sh): add the package, never a workaround")
|
|
118
158
|
boot_text = (f"{boot.get('head')} via {boot.get('rung')} at {boot.get('url')}" if booted
|
|
119
159
|
else f"not booted: {boot.get('reason')}")
|
|
120
160
|
checks_ran = set()
|
|
@@ -128,6 +168,7 @@ def main(argv):
|
|
|
128
168
|
notes = []
|
|
129
169
|
scr = scr_by_name.get(r.get("screen") or "")
|
|
130
170
|
driven = bool(scr) and scr.get("render") != "UNREACHABLE"
|
|
171
|
+
nonames = " (names not measured on a Mac)" if scr and scr.get("anchors_not_measured") else ""
|
|
131
172
|
t = tests.get(rid)
|
|
132
173
|
# A row whose every clause was SKIPPED is NOT-TESTED: the acceptance gate did not run on
|
|
133
174
|
# it. It is neither a pass nor a defect, so it grades exactly like a row with no test at
|
|
@@ -136,6 +177,21 @@ def main(argv):
|
|
|
136
177
|
if skipped_only:
|
|
137
178
|
notes.append(f"acceptance not measured ({len(t.get('skipped') or [])} test(s) skipped, none ran)")
|
|
138
179
|
t = None
|
|
180
|
+
# the OS zoom keys (coding-standards-dotnet.md §9b): a row whose acceptance carries "zoom-check: keys"
|
|
181
|
+
# or "zoom-check: kept" is graded by tf-verify-zoom.sh --json-out zoom/<REQ>.json, which stands as its
|
|
182
|
+
# test; a failing zoom outranks a passing test, and "kept" counts only a run made after a restart
|
|
183
|
+
if r.get("zoom_check"):
|
|
184
|
+
zf = load(os.path.join(d, "zoom", f"{rid}.json"))
|
|
185
|
+
if zf is None:
|
|
186
|
+
notes.append("zoom not measured (no tf-verify-zoom run for this row)")
|
|
187
|
+
elif r["zoom_check"] == "kept" and not zf.get("kept_checked"):
|
|
188
|
+
notes.append("zoom kept across a restart not measured (run with --expect-restored after a restart)")
|
|
189
|
+
else:
|
|
190
|
+
zfail = [c for c in zf.get("checks", []) if not c.get("ok")]
|
|
191
|
+
zt = {"result": "FAIL" if zfail else "PASS", "tests": [f"tf-verify-zoom {zf.get('head', '')}".strip()],
|
|
192
|
+
"reason": f"zoom: {zfail[0]['text']}" if zfail else "", "screenshot": zf.get("evidence", "")}
|
|
193
|
+
if not t or (zfail and t.get("result") == "PASS"):
|
|
194
|
+
t = zt
|
|
139
195
|
# every check on its own: (name, ran, failed, verdict, class, detail, evidence); the first failure wins
|
|
140
196
|
checks = []
|
|
141
197
|
|
|
@@ -154,6 +210,10 @@ def main(argv):
|
|
|
154
210
|
w, f = bad[0] if bad else (None, None)
|
|
155
211
|
checks.append(("render", True, bool(bad), "RENDER-FAIL", f["class"] if f else None,
|
|
156
212
|
f"{f['detail']} on {r['screen']} @{w['width']}" if f else "", w.get("screenshot", "") if w else ""))
|
|
213
|
+
# a Mac web view head: blank, error bar, overlap and the picture were checked, the
|
|
214
|
+
# controls by name were not (owner decision A, 2026-10-07): said, never counted as a pass
|
|
215
|
+
if scr.get("anchors_not_measured") and bad:
|
|
216
|
+
notes.append(f"control names not measured on {r['screen']} (a Mac web view does not show them)")
|
|
157
217
|
# 4 assets
|
|
158
218
|
pg = pages.get(norm_route(r.get("route"))) if r.get("route") else None
|
|
159
219
|
if pg and pg.get("graded", 0) > 0:
|
|
@@ -169,16 +229,40 @@ def main(argv):
|
|
|
169
229
|
checks.append(("visual", True, bool(bad), "VISUAL-FAIL", f["class"] if f else None,
|
|
170
230
|
f"{f['detail']} on {r['screen']}" if f else "", w.get("screenshot", "") if w else ""))
|
|
171
231
|
# 6 mockup parity
|
|
172
|
-
|
|
232
|
+
mockup = mock_of.get(r.get("screen")) or r.get("mockup") or ""
|
|
233
|
+
ps = parity_for(r, mockup, comparisons) if r.get("screen") else None
|
|
234
|
+
if not ps and driven and mockup and boot.get("mode") == "appium":
|
|
235
|
+
# a Mac head: tf-mockup-parity does not run there (verify-phase step 5), so no comparison is
|
|
236
|
+
# expected, and the row is never failed for its absence; said, not counted as a pass
|
|
237
|
+
notes.append("mockup not compared on a Mac head (compare it on the web or Windows head)")
|
|
238
|
+
elif not ps and driven and mockup:
|
|
239
|
+
# Chatur TF-008: /processes/6/run was compared and FAILed, but the row's route reads
|
|
240
|
+
# /processes/{id}/run and the comparison was named process-run, so it was never found and
|
|
241
|
+
# 11 rows were Verified without it. A driven screen that has a mockup and no comparison fails.
|
|
242
|
+
checks.append(("mockup-parity", True, True, "MOCKUP-FAIL", "other",
|
|
243
|
+
f"mockup-parity not found: {r['screen']} ({r.get('route')}) has mockup {mockup} and "
|
|
244
|
+
+ ("parity.json has no comparison for it" if comparisons else "no mockup comparison ran")
|
|
245
|
+
+ f"; run tf-mockup-parity.sh --screen {stem(mockup) or 'name'}={r.get('route')} with its value filled in",
|
|
246
|
+
os.path.join(d, "parity.json") if comparisons else ""))
|
|
173
247
|
if ps:
|
|
174
248
|
v = ps.get("verdict")
|
|
175
249
|
if v in ("PASS", "FAIL"):
|
|
176
250
|
f = (ps.get("findings") or [{}])[0]
|
|
177
251
|
checks.append(("mockup-parity", True, v == "FAIL", "MOCKUP-FAIL", "mockup-drift",
|
|
178
|
-
f"{f.get('class', 'drift')} on {f.get('key', ps.get('screen'))} @{f.get('width', '')}
|
|
252
|
+
f"{f.get('class', 'drift')} on {f.get('key', ps.get('screen'))} @{f.get('width', '')}"
|
|
253
|
+
+ (f" in {f['theme']} mode" if f.get("theme") else "") + f": {str(f.get('detail', ''))[:80]}",
|
|
254
|
+
os.path.join(d, "parity.json")))
|
|
255
|
+
elif v == "ERROR":
|
|
256
|
+
checks.append(("mockup-parity", True, True, "MOCKUP-FAIL", "other",
|
|
257
|
+
f"mockup-parity could not open {ps.get('route')}: "
|
|
258
|
+
+ str(next((w.get("error") for w in ps.get("widths", []) if w.get("error")), "every width failed"))[:80],
|
|
179
259
|
os.path.join(d, "parity.json")))
|
|
180
260
|
elif v == "UNGRADEABLE":
|
|
181
261
|
notes.append("mockup-parity UNGRADEABLE (add data-testid anchors to the mockup)")
|
|
262
|
+
nm = (ps.get("coverage") or {}).get("not_measured") or []
|
|
263
|
+
if nm: # sample rows the app had no data for (Chatur TF-002)
|
|
264
|
+
notes.append(f"mockup-parity: {len(nm)} sample box(es) not measured, the app drew no data there "
|
|
265
|
+
"(a tests/verify/seed/ script puts it in the drawn state)")
|
|
182
266
|
elif v == "NO-MOCKUP":
|
|
183
267
|
notes.append("no mockup to compare")
|
|
184
268
|
# 7 speed
|
|
@@ -208,21 +292,31 @@ def main(argv):
|
|
|
208
292
|
verdict = "NOT-DRIVEN"
|
|
209
293
|
sk = scr_skipped.get(r["screen"])
|
|
210
294
|
detail = (f"{boot.get('head', 'the app')} not driven: {boot.get('reason')}" if not booted
|
|
295
|
+
else f"screen {r['screen']} was not driven: {sk['reason']}" if sk and sk.get("reason")
|
|
211
296
|
else f"screen {r['screen']} was not driven: its route {sk['route']} needs --route-value "
|
|
212
297
|
+ ", ".join(f"{n}=<id>" for n in sk.get("needs", [])) if sk
|
|
213
298
|
else f"screen {r['screen']} was not driven in this run")
|
|
214
299
|
elif not t:
|
|
215
300
|
verdict = "NOT-TESTED"
|
|
216
301
|
detail = (f"every test named {rid} was skipped" if skipped_only else f"no test named {rid} ran") \
|
|
217
|
-
+ (f"; screen {r['screen']} renders and looks right" if driven else "")
|
|
218
|
-
elif r.get("screen") and not driven
|
|
302
|
+
+ (f"; screen {r['screen']} renders and looks right{nonames}" if driven else "")
|
|
303
|
+
elif r.get("screen") and not driven:
|
|
304
|
+
# a passing test is not a screen check: a row on a screen that was not driven is never
|
|
305
|
+
# Verified, booted or not. The Settings rows of /settings/{tab} were (Chatur TF-003).
|
|
219
306
|
verdict = "NOT-DRIVEN"
|
|
220
|
-
|
|
307
|
+
sk = scr_skipped.get(r["screen"])
|
|
308
|
+
detail = ("test passed but " + (f"{boot.get('head', 'the app')} was not driven: {boot.get('reason')}" if not booted
|
|
309
|
+
else f"screen {r['screen']} was not driven: {sk['reason']}" if sk and sk.get("reason")
|
|
310
|
+
else f"screen {r['screen']} was not driven: its route {sk['route']} needs --route-value "
|
|
311
|
+
+ ", ".join(f"{n}=<id>" for n in sk.get("needs", [])) if sk
|
|
312
|
+
else f"screen {r['screen']} was not driven in this run"))
|
|
221
313
|
else:
|
|
222
314
|
verdict = "PASS"
|
|
223
315
|
parts = [f"test {quoted(t['tests'][0], 60)}" if t and t.get("tests") else "test passed"]
|
|
224
316
|
if driven:
|
|
225
|
-
|
|
317
|
+
ws = list(dict.fromkeys(str(w['width']) for w in scr['widths']))
|
|
318
|
+
th = list(dict.fromkeys(w['theme'] for w in scr['widths'] if w.get('theme')))
|
|
319
|
+
parts.append(f"{r['screen']} renders and looks right @{'/'.join(ws)}" + (f" in {' and '.join(th)} mode" if len(th) > 1 else "") + nonames)
|
|
226
320
|
if "mockup-parity" in ran:
|
|
227
321
|
parts.append("matches its mockup")
|
|
228
322
|
if "perf" in ran:
|
|
@@ -269,23 +363,32 @@ def main(argv):
|
|
|
269
363
|
# with the date and run it was graded in; the rest keep theirs. A scope-all run starts it afresh.
|
|
270
364
|
# The verify guard unlocks a Verified only for a row graded today, and fix-close reads only the rows
|
|
271
365
|
# graded since its fix began, by these dates.
|
|
366
|
+
# Sevak TF-005: every row carries the phase whose checklist it was graded in, and a scope-all run
|
|
367
|
+
# starts afresh only for its own phase. Phase 2's run used to wipe phase 1's verdicts. A row from an
|
|
368
|
+
# older ledger that names no phase is kept.
|
|
369
|
+
phase = lst.get("phase") or 1
|
|
272
370
|
all_rows, row_dates, row_runs = dict(ledger_rows), {k: today for k in ledger_rows}, {k: started for k in ledger_rows}
|
|
371
|
+
row_phases = {k: phase for k in ledger_rows}
|
|
273
372
|
prev = load(os.path.join("docs", ".last-verify.json")) if os.path.isfile(os.path.join("docs", ".last-verify.json")) else None
|
|
274
|
-
if
|
|
373
|
+
if isinstance(prev, dict) and prev.get("app") in (None, app) \
|
|
275
374
|
and isinstance(prev.get("rows"), dict) and prev.get("mode") != "reconcile":
|
|
276
375
|
for k, v in prev["rows"].items():
|
|
277
|
-
|
|
376
|
+
was = (prev.get("row_phases") or {}).get(k, prev.get("phase"))
|
|
377
|
+
if k not in all_rows and (lst.get("scope", "all") != "all" or was != phase):
|
|
278
378
|
all_rows[k] = v
|
|
279
379
|
row_dates[k] = (prev.get("row_dates") or {}).get(k, prev.get("date", ""))
|
|
280
380
|
row_runs[k] = (prev.get("row_runs") or {}).get(k, prev.get("run_id", ""))
|
|
281
|
-
|
|
381
|
+
if was is not None:
|
|
382
|
+
row_phases[k] = was
|
|
383
|
+
ledger = {"date": today, "app": app, "scope": lst.get("scope", "all"), "phase": phase, "checklist": lst.get("checklist"),
|
|
384
|
+
"booted": boot_text, "gates": checks,
|
|
282
385
|
"evidence": d, "run_id": started, "rows": all_rows, "row_dates": row_dates, "row_runs": row_runs,
|
|
283
|
-
"this_run": sorted(ledger_rows)}
|
|
386
|
+
"row_phases": row_phases, "this_run": sorted(ledger_rows)}
|
|
284
387
|
os.makedirs("docs", exist_ok=True)
|
|
285
388
|
with open(os.path.join("docs", ".last-verify.json"), "w", encoding="utf-8") as f:
|
|
286
389
|
json.dump(ledger, f, indent=1)
|
|
287
390
|
with open(os.path.join(d, "verdicts.json"), "w", encoding="utf-8") as f:
|
|
288
|
-
json.dump({"app": app, "scope": lst.get("scope", "all"), "date": today, "started": started, "booted": boot_text,
|
|
391
|
+
json.dump({"app": app, "scope": lst.get("scope", "all"), "phase": phase, "date": today, "started": started, "booted": boot_text,
|
|
289
392
|
"build_result": "fail" if build_failed else ("pass" if booted else "not-run"), "checks": checks,
|
|
290
393
|
"checklist": lst.get("checklist"), "rows": rows_out}, f, indent=1)
|
|
291
394
|
|
|
@@ -314,7 +417,7 @@ def main(argv):
|
|
|
314
417
|
counts = {}
|
|
315
418
|
for x in rows_out:
|
|
316
419
|
counts[x["verdict"]] = counts.get(x["verdict"], 0) + 1
|
|
317
|
-
print(f"# tf-verify-verdict — {app} — scope {lst.get('scope')} — booted: {boot_text}")
|
|
420
|
+
print(f"# tf-verify-verdict — {app} — phase {phase} ({lst.get('checklist')}) — scope {lst.get('scope')} — booted: {boot_text}")
|
|
318
421
|
print(f"Checks that ran: {', '.join(checks) or 'none'}. Rows: " + ", ".join(f"{v} {k}" for k, v in sorted(counts.items())))
|
|
319
422
|
print()
|
|
320
423
|
print("| ID | Verdict | First failing check | Status | Detail |")
|