bmad-method-test-architecture-enterprise 1.22.6 → 1.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.github/ISSUE_TEMPLATE/rule_quality_report.md +46 -0
- package/.github/workflows/quality.yaml +51 -0
- package/CHANGELOG.md +18 -0
- package/eslint.config.mjs +5 -2
- package/package.json +6 -2
- package/src/agents/bmad-tea/resources/knowledge/component-tdd.md +36 -0
- package/src/agents/bmad-tea/resources/knowledge/data-factories.md +39 -0
- package/src/agents/bmad-tea/resources/knowledge/test-quality.md +183 -0
- package/src/agents/bmad-tea/resources/knowledge/timing-debugging.md +92 -0
- package/src/agents/bmad-tea/resources/tea-index.csv +4 -4
- package/src/workflows/testarch/bmad-teach-me-testing/data/curriculum.yaml +1 -1
- package/src/workflows/testarch/bmad-teach-me-testing/data/session-content-map.yaml +1 -1
- package/src/workflows/testarch/bmad-teach-me-testing/data/tea-resources-index.yaml +1 -1
- package/src/workflows/testarch/bmad-teach-me-testing/instructions.md +1 -1
- package/src/workflows/testarch/bmad-teach-me-testing/steps-c/step-03-session-menu.md +1 -1
- package/src/workflows/testarch/bmad-teach-me-testing/steps-c/step-04-session-01.md +3 -3
- package/src/workflows/testarch/bmad-teach-me-testing/steps-c/step-04-session-07.md +29 -6
- package/src/workflows/testarch/bmad-teach-me-testing/steps-c/step-05-completion.md +2 -2
- package/src/workflows/testarch/bmad-teach-me-testing/templates/completion-summary-template.md +1 -1
- package/src/workflows/testarch/bmad-teach-me-testing/workflow-plan-teach-me-testing.md +6 -6
- package/src/workflows/testarch/bmad-testarch-atdd/resources/knowledge/component-tdd.md +36 -0
- package/src/workflows/testarch/bmad-testarch-atdd/resources/knowledge/data-factories.md +39 -0
- package/src/workflows/testarch/bmad-testarch-atdd/resources/knowledge/test-quality.md +183 -0
- package/src/workflows/testarch/bmad-testarch-atdd/resources/knowledge/timing-debugging.md +92 -0
- package/src/workflows/testarch/bmad-testarch-atdd/resources/tea-index.csv +6 -6
- package/src/workflows/testarch/bmad-testarch-automate/resources/knowledge/component-tdd.md +36 -0
- package/src/workflows/testarch/bmad-testarch-automate/resources/knowledge/data-factories.md +39 -0
- package/src/workflows/testarch/bmad-testarch-automate/resources/knowledge/test-quality.md +183 -0
- package/src/workflows/testarch/bmad-testarch-automate/resources/knowledge/timing-debugging.md +92 -0
- package/src/workflows/testarch/bmad-testarch-automate/resources/tea-index.csv +7 -7
- package/src/workflows/testarch/bmad-testarch-ci/resources/knowledge/component-tdd.md +36 -0
- package/src/workflows/testarch/bmad-testarch-ci/resources/knowledge/data-factories.md +39 -0
- package/src/workflows/testarch/bmad-testarch-ci/resources/knowledge/test-quality.md +183 -0
- package/src/workflows/testarch/bmad-testarch-ci/resources/knowledge/timing-debugging.md +92 -0
- package/src/workflows/testarch/bmad-testarch-ci/resources/tea-index.csv +7 -7
- package/src/workflows/testarch/bmad-testarch-framework/checklist.md +10 -0
- package/src/workflows/testarch/bmad-testarch-framework/resources/hooks/tea-enforce.cjs +1098 -0
- package/src/workflows/testarch/bmad-testarch-framework/resources/knowledge/component-tdd.md +36 -0
- package/src/workflows/testarch/bmad-testarch-framework/resources/knowledge/data-factories.md +39 -0
- package/src/workflows/testarch/bmad-testarch-framework/resources/knowledge/test-quality.md +183 -0
- package/src/workflows/testarch/bmad-testarch-framework/resources/knowledge/timing-debugging.md +92 -0
- package/src/workflows/testarch/bmad-testarch-framework/resources/tea-index.csv +7 -7
- package/src/workflows/testarch/bmad-testarch-framework/steps-c/step-04-docs-and-scripts.md +95 -1
- package/src/workflows/testarch/bmad-testarch-nfr/resources/knowledge/component-tdd.md +36 -0
- package/src/workflows/testarch/bmad-testarch-nfr/resources/knowledge/data-factories.md +39 -0
- package/src/workflows/testarch/bmad-testarch-nfr/resources/knowledge/test-quality.md +183 -0
- package/src/workflows/testarch/bmad-testarch-nfr/resources/knowledge/timing-debugging.md +92 -0
- package/src/workflows/testarch/bmad-testarch-nfr/resources/tea-index.csv +7 -7
- package/src/workflows/testarch/bmad-testarch-test-design/resources/knowledge/component-tdd.md +36 -0
- package/src/workflows/testarch/bmad-testarch-test-design/resources/knowledge/data-factories.md +39 -0
- package/src/workflows/testarch/bmad-testarch-test-design/resources/knowledge/test-quality.md +183 -0
- package/src/workflows/testarch/bmad-testarch-test-design/resources/knowledge/timing-debugging.md +92 -0
- package/src/workflows/testarch/bmad-testarch-test-design/resources/tea-index.csv +7 -7
- package/src/workflows/testarch/bmad-testarch-test-review/resources/knowledge/component-tdd.md +36 -0
- package/src/workflows/testarch/bmad-testarch-test-review/resources/knowledge/data-factories.md +39 -0
- package/src/workflows/testarch/bmad-testarch-test-review/resources/knowledge/test-quality.md +183 -0
- package/src/workflows/testarch/bmad-testarch-test-review/resources/knowledge/timing-debugging.md +92 -0
- package/src/workflows/testarch/bmad-testarch-test-review/resources/tea-index.csv +7 -7
- package/src/workflows/testarch/bmad-testarch-trace/resources/knowledge/component-tdd.md +36 -0
- package/src/workflows/testarch/bmad-testarch-trace/resources/knowledge/data-factories.md +39 -0
- package/src/workflows/testarch/bmad-testarch-trace/resources/knowledge/test-quality.md +183 -0
- package/src/workflows/testarch/bmad-testarch-trace/resources/knowledge/timing-debugging.md +92 -0
- package/src/workflows/testarch/bmad-testarch-trace/resources/tea-index.csv +7 -7
- package/test/eval-fragment-selection.js +472 -0
- package/test/evals/bmad-testarch-atdd/evals.json +98 -0
- package/test/evals/bmad-testarch-automate/evals.json +198 -0
- package/test/evals/bmad-testarch-ci/evals.json +78 -0
- package/test/evals/bmad-testarch-framework/evals.json +111 -0
- package/test/evals/bmad-testarch-nfr/evals.json +78 -0
- package/test/evals/bmad-testarch-test-design/evals.json +136 -0
- package/test/evals/bmad-testarch-test-review/evals.json +76 -0
- package/test/evals/bmad-testarch-trace/evals.json +62 -0
- package/test/test-enforce-hook.js +413 -0
- package/test/test-knowledge-base.js +82 -0
- package/tools/validate-criteria-fragments.js +446 -0
|
@@ -31,7 +31,7 @@
|
|
|
31
31
|
"name": "bmad-method-test-architecture-enterprise",
|
|
32
32
|
"source": "./",
|
|
33
33
|
"description": "Master Test Architect module for quality strategy, test automation, CI/CD quality gates, and structured testing education. Part of the BMad Method ecosystem.",
|
|
34
|
-
"version": "1.
|
|
34
|
+
"version": "1.23.0",
|
|
35
35
|
"author": {
|
|
36
36
|
"name": "Murat K Ozcan (TEA Creator) & Brian (BMad) Madison"
|
|
37
37
|
},
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: Rule Quality Report
|
|
3
|
+
about: An agent misread, ignored, or was misled by a TEA rule or knowledge fragment
|
|
4
|
+
title: ''
|
|
5
|
+
labels: 'rule-quality'
|
|
6
|
+
assignees: ''
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
TEA ships rules. This report is about a rule that failed to steer an agent, not about TEA crashing. One report per rule, please.
|
|
10
|
+
|
|
11
|
+
**Agent and model**
|
|
12
|
+
Which agent ran, and which model: e.g. Claude Code / Claude Opus 4.6, Cursor / GPT-5.1, Windsurf, Codex.
|
|
13
|
+
|
|
14
|
+
**Which rule**
|
|
15
|
+
The file and the section inside it. Examples:
|
|
16
|
+
|
|
17
|
+
- `criteria-registry.md`, row H3
|
|
18
|
+
- `resources/knowledge/network-first.md`, the "declare the intercept before navigating" section
|
|
19
|
+
- a step file such as `steps-c/step-03-generate-tests.md`
|
|
20
|
+
|
|
21
|
+
**Which workflow was running**
|
|
22
|
+
e.g. `test-design`, `automate`, `atdd`, `test-review`, `trace`, `nfr-assess`, `ci`, `framework`, `teach-me-testing`.
|
|
23
|
+
|
|
24
|
+
**The prompt you gave**
|
|
25
|
+
|
|
26
|
+
```text
|
|
27
|
+
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
**What it produced**
|
|
31
|
+
The non-compliant part only. Trim to the smallest excerpt that still shows the violation.
|
|
32
|
+
|
|
33
|
+
```text
|
|
34
|
+
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
**What it should have done**
|
|
38
|
+
State the behavior the rule was supposed to produce, and why you read the rule as requiring it.
|
|
39
|
+
|
|
40
|
+
**Your read of the cause** (optional)
|
|
41
|
+
|
|
42
|
+
- [ ] The rule says the right thing and the agent ignored it
|
|
43
|
+
- [ ] The rule is ambiguous and the agent picked a defensible wrong reading
|
|
44
|
+
- [ ] The rule is wrong or out of date
|
|
45
|
+
- [ ] The rule was never loaded (wrong fragment selected, or none)
|
|
46
|
+
- [ ] Not sure
|
|
@@ -7,6 +7,18 @@ name: Quality & Validation
|
|
|
7
7
|
# - Schema validation (YAML structure)
|
|
8
8
|
# - Agent schema tests (fixture-based validation)
|
|
9
9
|
# - Installation component tests (compilation)
|
|
10
|
+
#
|
|
11
|
+
# The `validate` job must stay a superset of `npm test`'s non-lint half. Three
|
|
12
|
+
# suites used to run ONLY from .husky/pre-commit — test:knowledge, test:changelog,
|
|
13
|
+
# and test:tea-workflow-descriptions — and `git commit --no-verify` skips that
|
|
14
|
+
# hook, as does every edit made through the GitHub web UI. test:knowledge is the
|
|
15
|
+
# one carrying the knowledge-copy parity check across the eight bmad-testarch workflows, so for as long as it was
|
|
16
|
+
# absent here nothing in CI stopped the workflow copies of the knowledge base from
|
|
17
|
+
# silently diverging from the agent's.
|
|
18
|
+
#
|
|
19
|
+
# test:cli is the deliberate exception in the other direction: it runs here and
|
|
20
|
+
# NOT in `npm test`, because it takes ~12 minutes (measured: 12m21s) and `npm test`
|
|
21
|
+
# runs on every commit through the pre-commit hook.
|
|
10
22
|
|
|
11
23
|
"on":
|
|
12
24
|
pull_request:
|
|
@@ -116,5 +128,44 @@ jobs:
|
|
|
116
128
|
- name: Test agent compilation components
|
|
117
129
|
run: npm run test:install
|
|
118
130
|
|
|
131
|
+
- name: Test knowledge base (index sync, link resolution, workflow-copy parity)
|
|
132
|
+
run: npm run test:knowledge
|
|
133
|
+
|
|
134
|
+
- name: Validate criteria-registry to fragment traceability
|
|
135
|
+
run: npm run test:criteria-fragments
|
|
136
|
+
|
|
137
|
+
- name: Test write-time enforcement hook
|
|
138
|
+
run: npm run test:enforce-hook
|
|
139
|
+
|
|
140
|
+
- name: Validate fragment-selection eval data
|
|
141
|
+
run: npm run test:eval-data
|
|
142
|
+
|
|
143
|
+
- name: Test changelog stamping
|
|
144
|
+
run: npm run test:changelog
|
|
145
|
+
|
|
146
|
+
- name: Validate TEA workflow descriptions
|
|
147
|
+
run: npm run test:tea-workflow-descriptions
|
|
148
|
+
|
|
149
|
+
# Split out of `validate` on its own: it takes ~12 minutes, and leaving it in
|
|
150
|
+
# line meant every fast check behind it reported 12 minutes late.
|
|
151
|
+
cli:
|
|
152
|
+
runs-on: ubuntu-latest
|
|
153
|
+
steps:
|
|
154
|
+
- name: Checkout
|
|
155
|
+
uses: actions/checkout@v5
|
|
156
|
+
with:
|
|
157
|
+
# This job only reads. actions/checkout persists the job token in
|
|
158
|
+
# .git/config by default, where any step or test process could read it.
|
|
159
|
+
persist-credentials: false
|
|
160
|
+
|
|
161
|
+
- name: Setup Node
|
|
162
|
+
uses: actions/setup-node@v6
|
|
163
|
+
with:
|
|
164
|
+
node-version-file: ".nvmrc"
|
|
165
|
+
cache: "npm"
|
|
166
|
+
|
|
167
|
+
- name: Install dependencies
|
|
168
|
+
run: npm ci
|
|
169
|
+
|
|
119
170
|
- name: Test TEA test-review CLI
|
|
120
171
|
run: npm run test:cli
|
package/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [1.23.0] - 2026-08-15
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
|
|
14
|
+
- Write-time enforcement hook, scaffolded by the `framework` workflow. TEA was advisory (knowledge fragments) plus post-hoc (`test-review` scoring) with nothing at the write itself, so a `.only`, a `waitForTimeout`, or a `Thread.sleep` could land, be committed, and only surface at review. `src/workflows/testarch/bmad-testarch-framework/resources/hooks/tea-enforce.cjs` blocks the write instead, and `steps-c/step-04-docs-and-scripts.md` installs it plus its `.claude/settings.json` registration into the target project. Rules are the mechanically decidable `Absolute` rows of `criteria-registry.md` (C2, C3, C4 for Maestro flows, H1, H5, H6, H8 block; C1 warns, because its row is conditioned on a documented, still-true reason that no pattern can check). Multi-language by construction: Playwright, Cypress, Vitest/Jest, Pact, pytest, JUnit, Go test, and Maestro flows each get their own predicates and their own comment and string stripping, so a `waitForTimeout` inside a comment or a doc example is not a violation. Three passes rather than one: `--pre` blocks the fragment about to be written, `--post` re-reads the whole file from disk (which is what catches writes made through Bash, violations split across two edits, and whole-file rules), and `--stop` sweeps test files modified during the turn (which is what catches a codegen script that wrote files it never named). The hook fails open on any error of its own.
|
|
15
|
+
- The hook honours the registry's `Gate` column structurally. `.tea/enforce-config.json` carries only the globs for the stack the `framework` workflow actually detected, so a repo with no Maestro flows cannot fire the Maestro rows and a repo with no pact config cannot fire H6 or H8. `excludeGlobs` keeps k6 scripts out, where `sleep(1)` is the documented way to model think-time and H1 would be confidently wrong. A closed gate is not a violation, which is defect #1 from the couture-cast PR #103 postmortem in `DESIGN-CRITERIA-REGISTRY.md`, applied to a second enforcement surface.
|
|
16
|
+
- The scaffold records the hook's sha256 in `.tea/enforce-config.json`, and the hook compares its own file against it on `--stop` only, warning once and never blocking. The instruction to copy the script byte for byte pointed at a test that lives in the TEA repository, so nothing inside the target project would have noticed a locally edited copy — and a locally edited copy is exactly the one that is no longer covered by the test keeping its rules in agreement with the registry.
|
|
17
|
+
- `tools/validate-criteria-fragments.js`: traceability between `criteria-registry.md` and the knowledge fragments. Nothing asserted that a registry row still had a fragment teaching it, nor that a mapped fragment still carried its claim at the registry's pinned severity. It fails on a mapped fragment that lost its anchor, a manifest row pointing at a fragment that does not exist or is not indexed in `tea-index.csv`, a severity that no longer matches the registry, and a registry row that is neither mapped nor declared a gap. It found 14 rows with no fragment teaching them at all, which are closed in this same release (below), taking coverage from 21/35 to 35/35 over 46 anchors. The gap-declaration mechanism stays: a registry row that is neither mapped to a fragment nor declared a gap fails the build, and with no real gap left to exercise that path the tool now self-checks it against a synthetic row so a refactor cannot quietly kill the guard.
|
|
18
|
+
- Knowledge for the 14 registry rows nothing taught, added to the four fragments that already own the surrounding material rather than as 14 new files, since fragmenting the base makes selection worse. `test-quality.md` gains committed skips and committed focus (C1, C2), assertions that cannot fail (C3 tautological, C5 asserted against the test's own mock, C6 unreachable), and suite structure and naming (M3 counted by subject rather than by `expect` call, M4 grouping, M7 nesting, L5 behavioral names, L7 one assertion dialect). `timing-debugging.md` gains wall-clock fixtures with fake timers as the fix (H2) and unawaited promises in test bodies (M6). `component-tdd.md` gains user-level interaction over raw event dispatch (M5), gated on the project already depending on such an API. `data-factories.md` gains naming the domain literals a test hardcodes on purpose (L6), which is the half the factory patterns never covered. Examples appear in the languages each row's own predicate names — Python and JUnit skip forms for C1, `assert x == x` for C3, `freeze_time` for H2 — rather than in TypeScript alone. The `tags` and `description` columns for all four fragments gained the terms that make the new material selectable, in the agent index and all eight workflow copies.
|
|
19
|
+
- `test/eval-fragment-selection.js` plus per-workflow eval data under `test/evals/`: measures whether the right fragment comes out of `tea-index.csv` for a given task, which is the failure mode where an agent loads the wrong fragment, or none, and answers from prior. Twenty-four cases across all eight workflows that ship a knowledge base (`automate`, `test-design`, `atdd`, `ci`, `framework`, `nfr`, `test-review`, `trace`), seeded from Playwright, Cypress, pytest, JUnit, Go, Pact, GitLab CI, and Maestro scenarios rather than from Playwright alone, since fragment selection for a non-JavaScript stack is where TEA is most likely to route wrong. `teach-me-testing` is excluded because it ships no `resources/knowledge`: its fragment browsing is a menu the learner drives rather than a routing decision the agent makes. Every non-JavaScript case asserts the run-level precondition that nothing else was measuring: both library flags default true, and in a repo with no JavaScript manifest neither package can be installed, so both mandates must stay closed. Ground truth is quoted from each workflow's own step files, never from `tea-index.csv` tags, because tags describe a fragment while step files decide what a run loads. `--validate-only` runs in CI with no vendor cost and rejects a name in either direction that does not exist or is not indexed for that workflow, so a typo in a forbidden list cannot pass vacuously; the scored run needs a logged-in `claude` or `codex`.
|
|
20
|
+
- `.github/ISSUE_TEMPLATE/rule_quality_report.md`: a low-friction report for the failure that matters most in a rules product, an agent misreading, ignoring, or being misled by a TEA rule or knowledge fragment. Captures the agent and model, the rule and section, the workflow, the prompt, the non-compliant output, and what should have happened.
|
|
21
|
+
|
|
22
|
+
### Fixed
|
|
23
|
+
|
|
24
|
+
- `teach-me-testing` session 7 could reach only 42 of the 59 knowledge fragments. Every mobile fragment (`maestro-flows`, `mobile-test-strategy`, `mobile-ci-device-lab`), the entire seven-fragment webhook family, both integration mandates, `library-integration-mandate`, `confidence-gate`, and `evidence-integrity` had no category to appear under, so the only browsable view of the knowledge base silently hid a third of it. The menu gains a Mobile category and a Webhooks category, the existing categories absorb the rest, and the sixteen places that advertised "42 fragments" now state the real number. `test-knowledge-base.js` Test Suite 6 asserts all of it: every fragment appears exactly once, no phantom entries, each category subtotal matches its own list, and every stated total matches the base. The count matching the short menu is what kept the shortfall invisible.
|
|
25
|
+
|
|
26
|
+
- `quality.yaml` did not run the full test suite. `test:knowledge`, `test:changelog`, and `test:tea-workflow-descriptions` ran only from `.husky/pre-commit`, which `git commit --no-verify` skips and which no GitHub web-UI edit ever reaches. `test:knowledge` is the suite carrying the knowledge-base parity check across the eight workflow copies, so until now nothing in CI stopped the workflow copies from silently diverging from the agent's. All three now run in the `validate` job, alongside the three new checks above. `test:cli` moved to its own job: it takes 12m21s measured, and leaving it in line made every fast check behind it report twelve minutes late. It stays out of `npm test` for the same reason, since `npm test` runs on every commit.
|
|
27
|
+
|
|
10
28
|
## [1.22.6] - 2026-08-14
|
|
11
29
|
|
|
12
30
|
### Fixed
|
package/eslint.config.mjs
CHANGED
|
@@ -81,9 +81,12 @@ export default [
|
|
|
81
81
|
},
|
|
82
82
|
},
|
|
83
83
|
|
|
84
|
-
// CLI scripts under cli/**, tools/** and test
|
|
84
|
+
// CLI scripts under cli/**, tools/** and test/**, plus the hook scripts the
|
|
85
|
+
// framework workflow copies verbatim into a user's project. Those must stay
|
|
86
|
+
// plain, dependency-free CommonJS that runs on whatever Node the project has,
|
|
87
|
+
// so the same relaxations apply.
|
|
85
88
|
{
|
|
86
|
-
files: ['cli/**/*.js', 'tools/**/*.js', 'tools/**/*.mjs', 'test/**/*.js'],
|
|
89
|
+
files: ['cli/**/*.js', 'tools/**/*.js', 'tools/**/*.mjs', 'test/**/*.js', 'src/workflows/**/resources/hooks/*.cjs'],
|
|
87
90
|
rules: {
|
|
88
91
|
// Allow CommonJS patterns for Node CLI scripts
|
|
89
92
|
'unicorn/prefer-module': 'off',
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://json.schemastore.org/package.json",
|
|
3
3
|
"name": "bmad-method-test-architecture-enterprise",
|
|
4
|
-
"version": "1.
|
|
4
|
+
"version": "1.23.0",
|
|
5
5
|
"description": "Master Test Architect for quality strategy, test automation, and release gates",
|
|
6
6
|
"keywords": [
|
|
7
7
|
"bmad",
|
|
@@ -39,6 +39,7 @@
|
|
|
39
39
|
"docs:fix-links": "node tools/fix-doc-links.js --write",
|
|
40
40
|
"docs:preview": "npm --prefix website run preview",
|
|
41
41
|
"docs:validate-links": "node tools/validate-doc-links.js",
|
|
42
|
+
"eval:fragment-selection": "node test/eval-fragment-selection.js",
|
|
42
43
|
"eval:test-review": "node test/eval-test-review.js",
|
|
43
44
|
"format:check": "prettier --check .",
|
|
44
45
|
"format:fix": "prettier --write .",
|
|
@@ -51,10 +52,13 @@
|
|
|
51
52
|
"release:minor": "gh workflow run publish.yaml -f channel=latest -f bump=minor",
|
|
52
53
|
"release:next": "gh workflow run publish.yaml -f channel=next",
|
|
53
54
|
"release:patch": "gh workflow run publish.yaml -f channel=latest -f bump=patch",
|
|
54
|
-
"test": "npm run test:schemas && npm run test:install && npm run test:knowledge && npm run test:release-metadata && npm run test:changelog && npm run test:tea-workflow-descriptions && npm run validate:schemas && npm run lint && npm run lint:md && npm run format:check",
|
|
55
|
+
"test": "npm run test:schemas && npm run test:install && npm run test:knowledge && npm run test:criteria-fragments && npm run test:enforce-hook && npm run test:eval-data && npm run test:release-metadata && npm run test:changelog && npm run test:tea-workflow-descriptions && npm run validate:schemas && npm run lint && npm run lint:md && npm run format:check",
|
|
55
56
|
"test:changelog": "node test/test-stamp-changelog.js",
|
|
56
57
|
"test:cli": "node test/test-test-review-cli.js",
|
|
57
58
|
"test:coverage": "c8 npm test",
|
|
59
|
+
"test:criteria-fragments": "node tools/validate-criteria-fragments.js",
|
|
60
|
+
"test:enforce-hook": "node test/test-enforce-hook.js",
|
|
61
|
+
"test:eval-data": "node test/eval-fragment-selection.js --validate-only",
|
|
58
62
|
"test:install": "node test/test-installation-components.js",
|
|
59
63
|
"test:knowledge": "node test/test-knowledge-base.js",
|
|
60
64
|
"test:release-metadata": "node test/test-release-metadata.js",
|
|
@@ -455,6 +455,42 @@ export default defineConfig({
|
|
|
455
455
|
- Update snapshots when visual changes are intentional
|
|
456
456
|
- Visual tests catch unintended CSS/layout regressions
|
|
457
457
|
|
|
458
|
+
### Example 5: User-Level Interaction, Not Raw Event Dispatch
|
|
459
|
+
|
|
460
|
+
**Context**: A component test exists to prove the component behaves the way a person driving it would experience. `fireEvent` dispatches one synthetic event straight at the node. A real interaction is a sequence: pointer down, focus, key events, input events, pointer up, blur. Dispatching only the middle one skips everything the component might legitimately depend on, so the test can pass against a component no user can operate: a button that never receives focus, a field whose `onKeyDown` handler is never exercised, an input that ignores paste.
|
|
461
|
+
|
|
462
|
+
This is gated on the project already depending on a user-level API. Where `userEvent` (or its equivalent) is a project dependency, it is the interaction API and `fireEvent` is the deviation. Where it is not installed, `fireEvent` is what the project has and this row does not fire. Adding a dependency is a project decision, not a review finding.
|
|
463
|
+
|
|
464
|
+
`fireEvent` remains the right tool for the events a user cannot produce directly: a synthetic `error` on an image, a `transitionend`, a scroll event from an observer.
|
|
465
|
+
|
|
466
|
+
**Implementation**:
|
|
467
|
+
|
|
468
|
+
```typescript
|
|
469
|
+
// ❌ BAD: dispatches change directly. Focus, key events, and input events never
|
|
470
|
+
// happen, so a component that validates on keystroke is never exercised.
|
|
471
|
+
fireEvent.change(screen.getByLabelText('Email'), { target: { value: 'ada@example.com' } });
|
|
472
|
+
fireEvent.click(screen.getByRole('button', { name: 'Submit' }));
|
|
473
|
+
expect(onSubmit).toHaveBeenCalled();
|
|
474
|
+
|
|
475
|
+
// ✅ GOOD: the full interaction sequence a person produces
|
|
476
|
+
const user = userEvent.setup();
|
|
477
|
+
await user.type(screen.getByLabelText('Email'), 'ada@example.com');
|
|
478
|
+
await user.click(screen.getByRole('button', { name: 'Submit' }));
|
|
479
|
+
expect(onSubmit).toHaveBeenCalledWith({ email: 'ada@example.com' });
|
|
480
|
+
|
|
481
|
+
// ✅ ACCEPTABLE: an event no user can dispatch by hand
|
|
482
|
+
fireEvent.error(screen.getByRole('img', { name: 'Avatar' }));
|
|
483
|
+
expect(screen.getByTestId('avatar-fallback')).toBeVisible();
|
|
484
|
+
```
|
|
485
|
+
|
|
486
|
+
**Key Points**:
|
|
487
|
+
|
|
488
|
+
- Where a user-level API is already a project dependency, it is the interaction API
|
|
489
|
+
- The failure mode is a passing test for a component a real user cannot operate
|
|
490
|
+
- `userEvent` is asynchronous: its calls are awaited, which also removes a class of unawaited-promise flake
|
|
491
|
+
- `fireEvent` stays correct for events users cannot produce (`error`, `transitionend`, observer-driven scroll)
|
|
492
|
+
- If the project has no user-level API installed, this is not a violation
|
|
493
|
+
|
|
458
494
|
## Integration Points
|
|
459
495
|
|
|
460
496
|
- **Used in workflows**: `*atdd` (component test generation), `*automate` (component test expansion), `*framework` (component testing setup)
|
|
@@ -437,6 +437,45 @@ test('free accounts cannot access analytics', async ({ page, apiRequest }) => {
|
|
|
437
437
|
- Still allow overrides: `createProAccount({ maxUsers: 50 })` works
|
|
438
438
|
- Test intent clear: `createProAccount()` vs `createAccount({ plan: 'pro', features: [...] })`
|
|
439
439
|
|
|
440
|
+
### Example 6: Naming the Literals You Do Hardcode
|
|
441
|
+
|
|
442
|
+
**Context**: Everything above is about generating data so tests stay parallel-safe and unique. This is the other half, and the factories do not cover it: the literals a test writes on purpose because the assertion is about that exact value. A boundary, a rate, a limit, a status code, a currency scale.
|
|
443
|
+
|
|
444
|
+
Those are correct to hardcode. Leaving them anonymous is what costs. `expect(fee).toBe(2.9)` tells the next reader that the fee is 2.9 and nothing about why, so when the number changes nobody can tell whether the test encodes a requirement or someone's old guess. The reader has to go find the pricing document, and usually does not.
|
|
445
|
+
|
|
446
|
+
The fix is a name, not a comment, and not a constant file. A named constant beside the test, or a factory override that reads as the domain fact, puts the meaning at the point of use.
|
|
447
|
+
|
|
448
|
+
**Implementation**:
|
|
449
|
+
|
|
450
|
+
```typescript
|
|
451
|
+
// ❌ BAD: two unexplained literals. Which is a requirement, which is arbitrary?
|
|
452
|
+
test('applies the processing fee', async () => {
|
|
453
|
+
const order = createOrder({ subtotal: 100 });
|
|
454
|
+
expect(feeFor(order)).toBe(2.9);
|
|
455
|
+
});
|
|
456
|
+
|
|
457
|
+
// ✅ GOOD: each number states what it is
|
|
458
|
+
const STRIPE_PERCENT_FEE = 0.029; // per the payments contract, section 4
|
|
459
|
+
|
|
460
|
+
test('applies the processing fee', async () => {
|
|
461
|
+
const subtotal = 100;
|
|
462
|
+
const order = createOrder({ subtotal });
|
|
463
|
+
expect(feeFor(order)).toBe(subtotal * STRIPE_PERCENT_FEE);
|
|
464
|
+
});
|
|
465
|
+
|
|
466
|
+
// ✅ ALSO GOOD: the factory carries the domain fact, so the test reads as
|
|
467
|
+
// behavior and the limit needs no name at the call site at all
|
|
468
|
+
const order = createOrderAtItemLimit();
|
|
469
|
+
await expect(addItem(order)).rejects.toThrow('order is full');
|
|
470
|
+
```
|
|
471
|
+
|
|
472
|
+
**Key Points**:
|
|
473
|
+
|
|
474
|
+
- Generate data that only needs to be unique; name data that carries meaning
|
|
475
|
+
- The test is where the requirement gets encoded, so the number needs to say which requirement
|
|
476
|
+
- Name at the point of use; a shared constants file moves the meaning away from the reader again
|
|
477
|
+
- A value used once, whose meaning the test name already states, does not need a second name; this is about unexplained literals, not about every number
|
|
478
|
+
|
|
440
479
|
## Integration Points
|
|
441
480
|
|
|
442
481
|
- **Used in workflows**: `*atdd` (test generation), `*automate` (test expansion), `*framework` (factory setup)
|
|
@@ -639,6 +639,183 @@ test('admin action', async ({ page }) => {
|
|
|
639
639
|
- Reuse auth sessions via `storageState` (Playwright) or `setCookie` (Cypress)
|
|
640
640
|
- Skip unnecessary flows (email verification, multi-step signups)
|
|
641
641
|
|
|
642
|
+
### Example 6: Committed Skips and Committed Focus
|
|
643
|
+
|
|
644
|
+
**Context**: A suite reports green. Two of the ways it does that have nothing to do with the code being correct: a test that was turned off, and a test that turned every one of its siblings off.
|
|
645
|
+
|
|
646
|
+
A skip is not automatically a defect. A skip whose reason nobody can read is. The reason belongs on the line or the line directly above it, it has to name the condition that will make the test runnable again, and it has to still be true. A `FIXME` pointing at a bug closed six months ago is a deleted test with extra steps. If you cannot write that reason, delete the test; a deleted test is honest about the coverage you no longer have, and a permanently skipped one is not.
|
|
647
|
+
|
|
648
|
+
Focus is different, and worse. `.only` is a debugging tool that changes what the whole file runs. Committed, the file still passes, still reports as a passing file, and covers one test. Nothing in the output says the other nineteen did not run.
|
|
649
|
+
|
|
650
|
+
**Implementation**:
|
|
651
|
+
|
|
652
|
+
```typescript
|
|
653
|
+
// ❌ BAD: the reason is gone, so nobody can tell whether this is still needed
|
|
654
|
+
test.skip('checkout applies the promo code', async ({ page }) => {
|
|
655
|
+
/* ... */
|
|
656
|
+
});
|
|
657
|
+
|
|
658
|
+
// ✅ GOOD: the condition to remove the skip is on the line
|
|
659
|
+
// FIXME(TEA-412): promo service returns 500 in staging; re-enable when TEA-412 lands
|
|
660
|
+
test.skip('checkout applies the promo code', async ({ page }) => {
|
|
661
|
+
/* ... */
|
|
662
|
+
});
|
|
663
|
+
|
|
664
|
+
// ❌ BAD: the other tests in this file no longer run, and nothing says so
|
|
665
|
+
test.only('checkout applies the promo code', async ({ page }) => {
|
|
666
|
+
/* ... */
|
|
667
|
+
});
|
|
668
|
+
```
|
|
669
|
+
|
|
670
|
+
```python
|
|
671
|
+
# ❌ BAD: skipped with no reason anyone can act on
|
|
672
|
+
@pytest.mark.skip
|
|
673
|
+
def test_invoice_voids_after_payment():
|
|
674
|
+
...
|
|
675
|
+
|
|
676
|
+
# ✅ GOOD: pytest carries the reason in the marker itself
|
|
677
|
+
@pytest.mark.skip(reason="void endpoint returns 500 upstream, see TEA-412")
|
|
678
|
+
def test_invoice_voids_after_payment():
|
|
679
|
+
...
|
|
680
|
+
```
|
|
681
|
+
|
|
682
|
+
```java
|
|
683
|
+
// ❌ BAD: JUnit's bare form records nothing
|
|
684
|
+
@Disabled
|
|
685
|
+
void settlementRetriesWithBackoff() { }
|
|
686
|
+
|
|
687
|
+
// ✅ GOOD: the annotation takes the reason
|
|
688
|
+
@Disabled("flaky against the shared broker; unblocked by TEA-412")
|
|
689
|
+
void settlementRetriesWithBackoff() { }
|
|
690
|
+
```
|
|
691
|
+
|
|
692
|
+
**Key Points**:
|
|
693
|
+
|
|
694
|
+
- A skip with a documented, still-true reason is acceptable; a bare one is not
|
|
695
|
+
- Prefer deleting over skipping indefinitely: coverage you admit losing beats coverage you pretend to have
|
|
696
|
+
- `.only`, `fdescribe`, and `fit` must never be committed: they disable siblings silently
|
|
697
|
+
- Guard both in CI, not only in review: a grep in the pipeline costs nothing and a committed `.only` costs a release
|
|
698
|
+
|
|
699
|
+
### Example 7: Assertions That Cannot Fail
|
|
700
|
+
|
|
701
|
+
**Context**: Example 3 keeps assertions visible. Visible is not the same as meaningful. Three shapes execute, look like assertions in the diff, and prove nothing: one compares a value to itself, one checks the mock instead of the system, and one never runs at all. All three are worse than having no test, because the suite reports green and the coverage number goes up.
|
|
702
|
+
|
|
703
|
+
**Implementation**:
|
|
704
|
+
|
|
705
|
+
```typescript
|
|
706
|
+
// ❌ BAD: compares a value to itself; passes no matter what the app does
|
|
707
|
+
expect(true).toBe(true);
|
|
708
|
+
expect(user.id).toBe(user.id);
|
|
709
|
+
|
|
710
|
+
// ✅ GOOD: the expectation could differ from the actual
|
|
711
|
+
expect(response.status()).toBe(201);
|
|
712
|
+
expect(user.email).toBe(userData.email);
|
|
713
|
+
|
|
714
|
+
// ❌ BAD: the only assertion is against the mock this test configured.
|
|
715
|
+
// Nothing called into the system between setting it up and checking it,
|
|
716
|
+
// so this proves the mocking library works.
|
|
717
|
+
const repo = { save: vi.fn().mockResolvedValue({ id: 1 }) };
|
|
718
|
+
repo.save({ name: 'Ada' });
|
|
719
|
+
expect(repo.save).toHaveBeenCalledWith({ name: 'Ada' });
|
|
720
|
+
|
|
721
|
+
// ✅ GOOD: the system under test is what calls the mock, and the assertion
|
|
722
|
+
// is about what the system returned
|
|
723
|
+
const repo = { save: vi.fn().mockResolvedValue({ id: 1 }) };
|
|
724
|
+
const created = await createAccount(repo, { name: 'Ada' });
|
|
725
|
+
expect(created.id).toBe(1);
|
|
726
|
+
expect(repo.save).toHaveBeenCalledWith({ name: 'Ada' });
|
|
727
|
+
|
|
728
|
+
// ❌ BAD: unreachable. The return happens first.
|
|
729
|
+
test('rejects an expired token', async () => {
|
|
730
|
+
const result = await authorize(expiredToken);
|
|
731
|
+
return;
|
|
732
|
+
expect(result.ok).toBe(false);
|
|
733
|
+
});
|
|
734
|
+
|
|
735
|
+
// ❌ BAD: unreachable. The happy path never enters the catch, so a passing
|
|
736
|
+
// run asserts nothing and a broken run is swallowed.
|
|
737
|
+
try {
|
|
738
|
+
await authorize(expiredToken);
|
|
739
|
+
} catch (error) {
|
|
740
|
+
expect(error.code).toBe('EXPIRED');
|
|
741
|
+
}
|
|
742
|
+
|
|
743
|
+
// ✅ GOOD: assert on the rejection itself, against the same property the
|
|
744
|
+
// catch block was checking. `rejects.toThrow('EXPIRED')` matches the error
|
|
745
|
+
// MESSAGE, so swapping it in here would quietly assert something else.
|
|
746
|
+
await expect(authorize(expiredToken)).rejects.toMatchObject({ code: 'EXPIRED' });
|
|
747
|
+
```
|
|
748
|
+
|
|
749
|
+
```python
|
|
750
|
+
# ❌ BAD: tautological; true for every possible value of total
|
|
751
|
+
assert total == total
|
|
752
|
+
|
|
753
|
+
# ✅ GOOD
|
|
754
|
+
assert total == Decimal("42.00")
|
|
755
|
+
```
|
|
756
|
+
|
|
757
|
+
**Key Points**:
|
|
758
|
+
|
|
759
|
+
- If the assertion would pass against a completely broken implementation, it is not an assertion
|
|
760
|
+
- Configuring a mock and then asserting on that same mock, with no call into the system between, tests the mocking library
|
|
761
|
+
- An assertion after an unconditional `return`, or inside a `catch` the happy path never enters, or inside a callback the test never awaits, does not run
|
|
762
|
+
- Prefer `rejects`/`raises` forms over `try`/`catch` around the thing you expect to throw: they fail when nothing throws
|
|
763
|
+
|
|
764
|
+
### Example 8: Suite Structure, Naming, and One Dialect
|
|
765
|
+
|
|
766
|
+
**Context**: These do not make a test wrong. They make a failure expensive to read, which is the same cost paid every time the suite goes red for the next several years.
|
|
767
|
+
|
|
768
|
+
A test that asserts against three unrelated subjects does not localize: the failure says the test broke, not which behavior broke. Count subjects, not `expect` calls: three assertions about one response is one concern, and one assertion each about a response, a database row, and an email is three. An ungrouped file prints failures with no subject line. Nesting past three levels means the reader reconstructs the setup from four `beforeEach` blocks before they can read the test. A name that states the implementation goes stale the moment the implementation changes and tells the reader nothing when it fails. And a file that mixes assertion dialects makes every reader translate between two styles for no benefit.
|
|
769
|
+
|
|
770
|
+
**Implementation**:
|
|
771
|
+
|
|
772
|
+
```typescript
|
|
773
|
+
// ❌ BAD: three unrelated subjects; a failure does not say which one broke
|
|
774
|
+
test('checkout works correctly', async ({ page, request }) => {
|
|
775
|
+
await checkout(page);
|
|
776
|
+
expect(await orderCount(request)).toBe(1); // subject: the order API
|
|
777
|
+
expect(await inventoryFor(request, 'sku-1')).toBe(9); // subject: inventory
|
|
778
|
+
expect(await lastEmail()).toContain('Order confirmed'); // subject: email
|
|
779
|
+
});
|
|
780
|
+
|
|
781
|
+
// ✅ GOOD: one subject per test, grouped, named for the behavior
|
|
782
|
+
describe('checkout', () => {
|
|
783
|
+
test('records the order', async ({ request }) => {
|
|
784
|
+
/* one subject */
|
|
785
|
+
});
|
|
786
|
+
test('decrements inventory for the purchased sku', async ({ request }) => {
|
|
787
|
+
/* one subject */
|
|
788
|
+
});
|
|
789
|
+
test('sends the confirmation email', async () => {
|
|
790
|
+
/* one subject */
|
|
791
|
+
});
|
|
792
|
+
});
|
|
793
|
+
|
|
794
|
+
// ❌ BAD: names the implementation, or nothing at all
|
|
795
|
+
test('calls handleSubmit()', ...);
|
|
796
|
+
test('getUserById works correctly', ...);
|
|
797
|
+
|
|
798
|
+
// ✅ GOOD: names the behavior, so the failure line is the bug report
|
|
799
|
+
test('rejects a submission with no email', ...);
|
|
800
|
+
test('returns 404 for an unknown user id', ...);
|
|
801
|
+
|
|
802
|
+
// ❌ BAD: two dialects in one file
|
|
803
|
+
expect(response.status()).toBe(200);
|
|
804
|
+
assert.equal(body.role, 'admin');
|
|
805
|
+
|
|
806
|
+
// ✅ GOOD: pick the house dialect and keep it
|
|
807
|
+
expect(response.status()).toBe(200);
|
|
808
|
+
expect(body.role).toBe('admin');
|
|
809
|
+
```
|
|
810
|
+
|
|
811
|
+
**Key Points**:
|
|
812
|
+
|
|
813
|
+
- One concern per test, counted by subject rather than by `expect` call
|
|
814
|
+
- Group with `describe`/`context` once a file has three or more tests, so failures print with a subject
|
|
815
|
+
- Keep `describe` nesting and block nesting at three levels or fewer
|
|
816
|
+
- Name the behavior, not the method, the selector, or "works correctly"
|
|
817
|
+
- One assertion dialect per file, matching whatever the repo already uses
|
|
818
|
+
|
|
642
819
|
## Integration Points
|
|
643
820
|
|
|
644
821
|
- **Used in workflows**: `*atdd` (test generation quality), `*automate` (test expansion quality), `*test-review` (quality validation)
|
|
@@ -661,5 +838,11 @@ Every test must pass these criteria:
|
|
|
661
838
|
- [ ] **Explicit Assertions** - Keep `expect()` calls in test bodies, not hidden in helpers
|
|
662
839
|
- [ ] **Unique Data** - Use `faker` for dynamic data; never hardcode IDs or emails
|
|
663
840
|
- [ ] **Parallel-Safe** - Tests don't share state; run successfully with `--workers=4`
|
|
841
|
+
- [ ] **No Committed Focus** - No `.only`, `fdescribe`, or `fit` reaches the branch
|
|
842
|
+
- [ ] **Skips Documented** - Every skip carries a still-true reason naming what would re-enable it
|
|
843
|
+
- [ ] **Assertions Can Fail** - No self-comparison, no assertion against only the mock the test configured, nothing after an unconditional `return`
|
|
844
|
+
- [ ] **One Concern** - Counted by subject, not by `expect` call
|
|
845
|
+
- [ ] **Grouped and Shallow** - `describe`/`context` once a file has three tests; nesting three levels or fewer
|
|
846
|
+
- [ ] **Behavioral Names, One Dialect** - Names state the behavior; the file uses a single assertion style
|
|
664
847
|
|
|
665
848
|
_Source: Murat quality checklist, Definition of Done requirements (lines 370-381, 406-422)._
|
|
@@ -290,6 +290,98 @@ test.describe('Timing Anti-Patterns to Avoid', () => {
|
|
|
290
290
|
|
|
291
291
|
---
|
|
292
292
|
|
|
293
|
+
### Example 4: Fixtures Derived From the Live Clock
|
|
294
|
+
|
|
295
|
+
**Context**: A hard wait makes a test slow and flaky. Reading the live clock makes it flaky on a schedule nobody can reproduce. The test that builds an expiry, a token lifetime, a TTL, or a scheduling boundary from `Date.now()` passes all day and fails at a month boundary, at midnight UTC, on the last day of February, or on the CI runner whose clock drifted four seconds.
|
|
296
|
+
|
|
297
|
+
This is a HIGH rather than a MEDIUM because of what these values usually govern. A token lifetime computed from the wall clock is a security boundary tested against a moving target: the test cannot distinguish "the expiry logic is correct" from "the expiry has not happened yet."
|
|
298
|
+
|
|
299
|
+
The fix is to control time rather than to read it. Freeze it, then move it deliberately to the boundary the behavior is about.
|
|
300
|
+
|
|
301
|
+
**Implementation**:
|
|
302
|
+
|
|
303
|
+
```typescript
|
|
304
|
+
// ❌ BAD: the boundary moves with the clock, and the failure lands on a Tuesday
|
|
305
|
+
const token = issueToken({ expiresAt: Date.now() + 3600_000 });
|
|
306
|
+
await sleep(1000);
|
|
307
|
+
expect(isExpired(token)).toBe(false); // proves nothing about expiry
|
|
308
|
+
|
|
309
|
+
// ✅ GOOD: freeze, then step across the boundary on purpose.
|
|
310
|
+
// Restore in afterEach, never as the last line of the test: an assertion that
|
|
311
|
+
// throws would skip that line and leave every later test on a fake clock, which
|
|
312
|
+
// is the unreset-shared-state defect wearing a different hat.
|
|
313
|
+
afterEach(() => {
|
|
314
|
+
vi.useRealTimers();
|
|
315
|
+
});
|
|
316
|
+
|
|
317
|
+
test('the token expires at its TTL', () => {
|
|
318
|
+
vi.useFakeTimers();
|
|
319
|
+
vi.setSystemTime(new Date('2026-01-01T00:00:00Z'));
|
|
320
|
+
const token = issueToken({ ttlSeconds: 3600 });
|
|
321
|
+
expect(isExpired(token)).toBe(false);
|
|
322
|
+
vi.advanceTimersByTime(3601_000);
|
|
323
|
+
expect(isExpired(token)).toBe(true); // the boundary is what is under test
|
|
324
|
+
});
|
|
325
|
+
```
|
|
326
|
+
|
|
327
|
+
```python
|
|
328
|
+
# ❌ BAD: the fixture is different on every run
|
|
329
|
+
expires_at = time.time() + 3600
|
|
330
|
+
|
|
331
|
+
# ✅ GOOD: pin the clock, then move it
|
|
332
|
+
with freeze_time("2026-01-01T00:00:00Z") as frozen:
|
|
333
|
+
token = issue_token(ttl_seconds=3600)
|
|
334
|
+
assert not is_expired(token)
|
|
335
|
+
frozen.tick(3601)
|
|
336
|
+
assert is_expired(token)
|
|
337
|
+
```
|
|
338
|
+
|
|
339
|
+
**Key Points**:
|
|
340
|
+
|
|
341
|
+
- A time-bounded value built from the live clock is not a fixture, it is a variable
|
|
342
|
+
- Freeze the clock and advance it deliberately; the boundary is the behavior under test
|
|
343
|
+
- Restore real timers in `afterEach`, so a failing assertion cannot leave a later test on a fake clock
|
|
344
|
+
- Where the production code already takes an injectable clock, pass one. A production seam the application itself uses beats a test-only seam that only the suite knows about
|
|
345
|
+
- A timestamp merely stamped into a record and never asserted against is not this defect; the row is about values that govern an expiry, a lifetime, a TTL, or a schedule
|
|
346
|
+
|
|
347
|
+
### Example 5: Promises Nobody Awaited
|
|
348
|
+
|
|
349
|
+
**Context**: The most common way for a race condition to be in the test rather than in the application. A promise-returning call that is neither awaited nor returned starts its work and hands control straight to the next line, so the assertion runs against the state from before the effect. Usually it still passes, because the effect is fast. It fails on the loaded CI runner, which is the one machine where the failure is least reproducible.
|
|
350
|
+
|
|
351
|
+
An unawaited rejection is the second half: it surfaces as an unhandled rejection attributed to whichever test happened to be running when it settled, so the reported test and the broken test are different tests.
|
|
352
|
+
|
|
353
|
+
**Implementation**:
|
|
354
|
+
|
|
355
|
+
```typescript
|
|
356
|
+
// ❌ BAD: the click may not have landed when the assertion runs
|
|
357
|
+
test('adds the item to the cart', async ({ page }) => {
|
|
358
|
+
page.getByRole('button', { name: 'Add to cart' }).click();
|
|
359
|
+
await expect(page.getByTestId('cart-count')).toHaveText('1');
|
|
360
|
+
});
|
|
361
|
+
|
|
362
|
+
// ❌ BAD: the setup promise is still in flight
|
|
363
|
+
test('shows the seeded order', async ({ page, request }) => {
|
|
364
|
+
seedOrder(request, { id: 'ord-1' });
|
|
365
|
+
await page.goto('/orders/ord-1');
|
|
366
|
+
await expect(page.getByText('ord-1')).toBeVisible();
|
|
367
|
+
});
|
|
368
|
+
|
|
369
|
+
// ✅ GOOD: await the effect before asserting on it
|
|
370
|
+
test('adds the item to the cart', async ({ page }) => {
|
|
371
|
+
await page.getByRole('button', { name: 'Add to cart' }).click();
|
|
372
|
+
await expect(page.getByTestId('cart-count')).toHaveText('1');
|
|
373
|
+
});
|
|
374
|
+
```
|
|
375
|
+
|
|
376
|
+
**Key Points**:
|
|
377
|
+
|
|
378
|
+
- Every promise-returning call in a test body is awaited or explicitly returned
|
|
379
|
+
- The symptom is a test that passes locally and fails on a loaded runner, which reads as flake rather than as a missing `await`
|
|
380
|
+
- An unhandled rejection is usually attributed to the wrong test, so treat one as a signal to audit `await` coverage across the file, not only in the named test
|
|
381
|
+
- `no-floating-promises` in the linter catches this class before the suite ever runs; prefer that to catching it in review
|
|
382
|
+
|
|
383
|
+
---
|
|
384
|
+
|
|
293
385
|
## Async Debugging Techniques
|
|
294
386
|
|
|
295
387
|
### Technique 1: Promise Chain Analysis
|
|
@@ -2,8 +2,8 @@ id,name,description,tags,tier,fragment_file
|
|
|
2
2
|
library-integration-mandate,Library Integration Mandate,"General contract for every TEA integration library flag: the two gates (flag true plus package installed), REQUIRED vs RECOMMENDED levels, deviation protocol, scope discipline, the flag-to-mandate registry, and the ten places a new library must be wired","standards,governance,generation,review,integration,mandate",core,knowledge/library-integration-mandate.md
|
|
3
3
|
fixture-architecture,Fixture Architecture,"Composable fixture patterns (pure function → fixture → merge) and reuse rules","fixtures,architecture,playwright,cypress",core,knowledge/fixture-architecture.md
|
|
4
4
|
network-first,Network-First Safeguards,"Intercept-before-navigate workflow, HAR capture, deterministic waits, edge mocking","network,stability,playwright,cypress,ui",core,knowledge/network-first.md
|
|
5
|
-
data-factories,Data Factories and API Setup,"Factories with overrides, API seeding, cleanup discipline","data,factories,setup,api,backend,seeding",core,knowledge/data-factories.md
|
|
6
|
-
component-tdd,Component TDD Loop,"Red→green→refactor workflow, provider isolation, accessibility assertions","component-testing,tdd,ui",extended,knowledge/component-tdd.md
|
|
5
|
+
data-factories,Data Factories and API Setup,"Factories with overrides, API seeding, cleanup discipline, and naming the domain literals a test hardcodes on purpose","data,factories,setup,api,backend,seeding,magic-value,literals,naming,constants",core,knowledge/data-factories.md
|
|
6
|
+
component-tdd,Component TDD Loop,"Red→green→refactor workflow, provider isolation, accessibility assertions, and user-level interaction over raw event dispatch","component-testing,tdd,ui,user-event,fire-event,interaction",extended,knowledge/component-tdd.md
|
|
7
7
|
playwright-config,Playwright Config Guardrails,"Environment switching, timeout standards, artifact outputs","playwright,config,env",extended,knowledge/playwright-config.md
|
|
8
8
|
ci-burn-in,CI and Burn-In Strategy,"Staged jobs, shard orchestration, burn-in loops, artifact policy","ci,automation,flakiness",extended,knowledge/ci-burn-in.md
|
|
9
9
|
selective-testing,Selective Test Execution,"Tag/grep usage, spec filters, diff-based runs, promotion rules","risk-based,selection,strategy",extended,knowledge/selective-testing.md
|
|
@@ -14,13 +14,13 @@ error-handling,Error Handling Checks,"Scoped exception handling, retry validatio
|
|
|
14
14
|
visual-debugging,Visual Debugging Toolkit,"Trace viewer usage, artifact expectations, accessibility integration","debugging,dx,tooling,ui",specialized,knowledge/visual-debugging.md
|
|
15
15
|
risk-governance,Risk Governance,"Scoring matrix, category ownership, gate decision rules","risk,governance,gates",core,knowledge/risk-governance.md
|
|
16
16
|
probability-impact,Probability and Impact Scale,"Shared definitions for scoring matrix and gate thresholds","risk,scoring,scale",core,knowledge/probability-impact.md
|
|
17
|
-
test-quality,Test Quality Definition of Done,"Execution limits, isolation rules, green criteria","quality,definition-of-done,tests",core,knowledge/test-quality.md
|
|
17
|
+
test-quality,Test Quality Definition of Done,"Execution limits, isolation rules, green criteria, committed skips and focus, assertions that cannot fail, and suite structure and naming","quality,definition-of-done,tests,skip,only,focused,disabled,tautological,assertions,mocks,unreachable,concerns,grouping,nesting,naming,assertion-style",core,knowledge/test-quality.md
|
|
18
18
|
nfr-criteria,NFR Review Criteria,"Security, performance, reliability, maintainability status definitions","nfr,assessment,quality",extended,knowledge/nfr-criteria.md
|
|
19
19
|
test-levels,Test Levels Framework,"Guidelines for choosing unit, integration, or end-to-end coverage","testing,levels,selection,api,backend,ui",core,knowledge/test-levels-framework.md
|
|
20
20
|
test-priorities,Test Priorities Matrix,"P0–P3 criteria, coverage targets, execution ordering","testing,prioritization,risk",core,knowledge/test-priorities-matrix.md
|
|
21
21
|
test-healing-patterns,Test Healing Patterns,"Common failure patterns and automated fixes","healing,debugging,patterns",core,knowledge/test-healing-patterns.md
|
|
22
22
|
selector-resilience,Selector Resilience,"Robust selector strategies and debugging techniques","selectors,locators,debugging,ui",core,knowledge/selector-resilience.md
|
|
23
|
-
timing-debugging,Timing Debugging,"Race condition identification
|
|
23
|
+
timing-debugging,Timing Debugging,"Race condition identification, deterministic wait fixes, wall-clock fixtures and fake timers, and unawaited promises in test bodies","timing,async,debugging,clock,fake-timers,expiry,ttl,unawaited,await,floating-promise",extended,knowledge/timing-debugging.md
|
|
24
24
|
overview,Playwright Utils Overview,"Installation, design principles, fixture patterns for API and UI testing","playwright-utils,fixtures,api,backend,ui",core,knowledge/overview.md
|
|
25
25
|
playwright-utils-mandate,Playwright Utils Mandate,"Binding rule when tea_use_playwright_utils is true: substitution table (page.route to interceptNetworkCall, raw request to apiRequest, waitForTimeout to recurse), REQUIRED vs RECOMMENDED levels, banned patterns, pre-emit self-check, deviation protocol","playwright-utils,standards,generation,review,governance,mandate",core,knowledge/playwright-utils-mandate.md
|
|
26
26
|
api-request,API Request,"Typed HTTP client, schema validation, retry logic, operation-based overload for API and service testing","api,backend,service-testing,api-testing,playwright-utils,openapi,codegen,operation",core,knowledge/api-request.md
|
|
@@ -78,7 +78,7 @@ sessions:
|
|
|
78
78
|
duration: "ongoing"
|
|
79
79
|
difficulty: advanced
|
|
80
80
|
objective: "Deep-dive into specific knowledge fragments"
|
|
81
|
-
description: "Menu-driven exploration of
|
|
81
|
+
description: "Menu-driven exploration of 59 knowledge fragments organized by category"
|
|
82
82
|
recommended_for:
|
|
83
83
|
- experienced
|
|
84
84
|
prerequisites: []
|