bmad-method-test-architecture-enterprise 3.1.0 → 3.3.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "$schema": "https://json.schemastore.org/package.json",
   "name": "bmad-method-test-architecture-enterprise",
-  "version": "3.1.0",
+  "version": "3.3.0",
   "description": "Master Test Architect for quality strategy, test automation, and release gates",
   "keywords": [
     "bmad",
@@ -49,7 +49,8 @@
     "tea-test-review": "cli/test-review.js",
     "tea-trace": "cli/trace.js",
     "tea-trace-runner": "cli/trace-runner.js",
-    "tea-transcript-runner": "cli/transcript-runner.js"
+    "tea-transcript-runner": "cli/transcript-runner.js",
+    "tea-unit-check": "cli/unit-check.js"
   },
   "files": [
     "skills",
@@ -94,7 +95,8 @@
     "release:major": "gh workflow run publish.yaml -f bump=major",
     "release:minor": "gh workflow run publish.yaml -f bump=minor",
     "release:patch": "gh workflow run publish.yaml -f bump=patch",
-    "test": "npm run test:conflict-markers && npm run test:schemas && npm run test:install && npm run test:evaluate-guidance && npm run test:evaluate-authoring && npm run test:evaluate-gap-loop && npm run test:evaluate-dogfood && npm run test:evaluate-check && npm run test:evaluate-discovery && npm run test:evaluate-generator && npm run test:evaluate-candidate-review && npm run test:evaluate-compare && npm run test:evaluate-before-state && npm run test:evaluate-ci-gates && npm run test:evaluate-ci-tiers && npm run test:evaluate-ci-repositories:tagged-release-pr && npm run test:evaluate-ci-repositories:tagged-release-merge && npm run test:evaluate-ci-repositories:tagged-release-release && npm run test:evaluate-ci-repositories:nightly-deploy-pr && npm run test:evaluate-ci-repositories:nightly-deploy-merge && npm run test:evaluate-ci-repositories:nightly-deploy-scheduled && npm run test:evaluate-ci-repositories:nightly-deploy-release && npm run test:evaluate-ci-render && npm run test:evaluate-pr-mcp && npm run test:evaluate-pr-api && npm run test:evaluate-pr-suite && npm run test:evaluate-pr-workflow && npm run test:evaluate-pr-tool-use && npm run test:evaluate-pr-promptfoo && npm run test:evaluate-pr-ai-feature && npm run test:evaluate-pr-test-review && npm run test:evaluate-pr-gap-loop && npm run test:evaluate-pr-learn && npm run test:evaluate-pr-tutorial && npm run test:evaluate-partitions && npm run test:evaluate-partition-plans-views-and-check && npm run test:evaluate-partition-plans-preflight-and-run && npm run test:evaluate-partition-plans-score && npm run test:evaluate-partition-plans-attempts-seal-and-mappings && npm run test:evaluate-partition-plans-attempts-records && npm run test:evaluate-partition-plans-attempts-gameability && npm run test:evaluate-interpret && npm run test:evaluate-calibration && npm run test:evaluate-boundaries && npm run test:evaluate-preflight && npm run test:evaluate-isolation-conformance && npm run test:evaluate-mutation:qualification-basics && npm run test:evaluate-mutation:qualification-guards && npm run test:evaluate-mutation:recovery && npm run test:evaluate-mutation:journals && npm run test:evaluate-run && npm run test:evaluate-progress && npm run test:evaluate-aggregate && npm run test:evaluate-confinement && npm run test:evaluate-confinement-audit && npm run test:evaluate-confinement-history && npm run test:evaluate-confinement-git-state && npm run test:evaluate-confinement-network && npm run test:evaluate-held-inputs && npm run test:evaluate-arms-historical && npm run test:evaluate-arms-deployments && npm run test:evaluate-evaluators && npm run test:evaluate-evaluators-agent-version && npm run test:evaluate-agents && npm run test:evaluate-agents-importers && npm run test:evaluate-agents-sealed-brief && npm run test:evaluate-held-attempts && npm run test:evaluate-private && npm run test:evaluate-records && npm run test:evaluate-mcp && npm run test:evaluate-api && npm run test:evaluate-workflow && npm run test:evaluate-tool-use && npm run test:evaluate-promptfoo && npm run test:evaluate-learned-framework && npm run test:docs-tutorial && npm run test:knowledge && npm run test:criteria-fragments && npm run test:doc-count-sources && npm run test:planning-doc-sources && npm run test:doc-counts && npm run test:doc-claim-sources && npm run test:doc-claims && npm run test:doc-invocation-entry && npm run test:doc-invocations && npm run test:doc-link-anchors && npm run test:docs-build-names && npm run test:docs-evaluate-sidebar && npm run test:docs-gap-reader && npm run test:enforce-hook && npm run test:framework-scaffold && npm run test:isolation-primitives && npm run test:atdd-isolation && npm run test:atdd-net-guard && npm run test:atdd-scoring && npm run test:atdd-workflow-guidance && npm run test:generation-healing && npm run test:framework-scaffold-install-isolation && npm run test:transcript-harness && npm run test:suite-manifest && npm run test:teach-me-testing-scoring && npm run test:teach-cli && npm run test:eval-data && npm run test:eval-atdd-data && npm run test:eval-ci-data && npm run test:eval-framework-scaffold-data && npm run test:eval-nfr-data && npm run test:nfr-validate-corpus && npm run test:nfr-evidence-guidance && npm run test:nfr-cli && npm run test:eval-test-design-data && npm run test:eval-trace-data && npm run test:eval-routing-data && npm run test:eval-routing-boundaries && npm run test:eval-routing-evidence && npm run test:eval-transcript-data && npm run test:eval-teach-me-testing-data && npm run test:eval-schemas && npm run test:eval-diagnostics && npm run test:eval-replay && npm run test:eval-quality-corpus && npm run test:schema-versions && npm run test:contract-sources && npm run test:contracts && npm run test:contract-oracles && npm run test:probe-sources && npm run test:test-design-qualification && npm run test:test-review-qualification && npm run test:trace-qualification && npm run test:nfr-qualification && npm run test:ci-qualification && npm run test:probe-corpus && npm run test:compare-dominance && npm run test:compare-eval-runs && npm run test:configuration-experiment && npm run test:trial-set-scoring && npm run test:automate-cli && npm run test:automate-generation-evidence && npm run test:automate-eval-fixture && npm run test:eval-automate-data && npm run test:clock-port && npm run test:probe-conformance && npm run test:corpus-conformance && npm run test:file-system-conformance && npm run test:file-system-port && npm run test:probe-targets && npm run test:port-totality && npm run test:ci-coverage && npm run test:ci-coverage-filters && npm run test:install-actionlint && npm run test:workflows-lint && npm run test:install-linux-sandbox && npm run test:shards && npm run test:groups && npm run test:cli && npm run test:workflow-cli && npm run test:migrate-customizations && npm run test:framework-cli && npm run test:test-review-pr-scope && npm run test:test-review-report && npm run test:test-review-rubric && npm run test:bmad-output-gated && npm run test:release-metadata && npm run test:release-notes && npm run test:lockfile-age && npm run test:lockfile-age-cache && npm run test:licences && npm run test:supply-chain && npm run test:boundary && npm run test:direction && npm run test:lineage && npm run test:guard-publish && npm run test:layering-boundary-lineage && npm run test:changelog && npm run test:tea-workflow-descriptions && npm run test:docs-mermaid && npm run test:llms-headroom && npm run validate:schemas && npm run lint && npm run lint:md && npm run format:check",
+    "test": "npm run test:conflict-markers && npm run test:schemas && npm run test:install && npm run test:evaluate-guidance && npm run test:evaluate-authoring && npm run test:evaluate-gap-loop && npm run test:evaluate-dogfood && npm run test:evaluate-check && npm run test:evaluate-discovery && npm run test:evaluate-generator && npm run test:evaluate-candidate-review && npm run test:evaluate-compare && npm run test:evaluate-before-state && npm run test:evaluate-ci-gates && npm run test:evaluate-ci-tiers && npm run test:evaluate-ci-repositories:tagged-release-pr && npm run test:evaluate-ci-repositories:tagged-release-merge && npm run test:evaluate-ci-repositories:tagged-release-release && npm run test:evaluate-ci-repositories:nightly-deploy-pr && npm run test:evaluate-ci-repositories:nightly-deploy-merge && npm run test:evaluate-ci-repositories:nightly-deploy-scheduled && npm run test:evaluate-ci-repositories:nightly-deploy-release && npm run test:evaluate-ci-render && npm run test:evaluate-pr-mcp && npm run test:evaluate-pr-api && npm run test:evaluate-pr-suite && npm run test:evaluate-pr-workflow && npm run test:evaluate-pr-tool-use && npm run test:evaluate-pr-promptfoo && npm run test:evaluate-pr-ai-feature && npm run test:evaluate-pr-test-review && npm run test:evaluate-pr-gap-loop && npm run test:evaluate-pr-learn && npm run test:evaluate-pr-tutorial && npm run test:evaluate-partitions && npm run test:evaluate-partition-plans-views-and-check && npm run test:evaluate-partition-plans-preflight-and-run && npm run test:evaluate-partition-plans-score && npm run test:evaluate-partition-plans-attempts-seal-and-mappings && npm run test:evaluate-partition-plans-attempts-records && npm run test:evaluate-partition-plans-attempts-gameability && npm run test:evaluate-interpret && npm run test:evaluate-calibration && npm run test:evaluate-boundaries && npm run test:evaluate-preflight && npm run test:evaluate-isolation-conformance && npm run test:evaluate-mutation:qualification-basics && npm run test:evaluate-mutation:qualification-guards && npm run test:evaluate-mutation:recovery && npm run test:evaluate-mutation:journals && npm run test:evaluate-run && npm run test:evaluate-progress && npm run test:evaluate-aggregate && npm run test:evaluate-confinement && npm run test:evaluate-confinement-audit && npm run test:evaluate-confinement-history && npm run test:evaluate-confinement-git-state && npm run test:evaluate-confinement-network && npm run test:evaluate-held-inputs && npm run test:evaluate-arms-historical && npm run test:evaluate-arms-deployments && npm run test:evaluate-evaluators && npm run test:evaluate-evaluators-agent-version && npm run test:evaluate-agents && npm run test:evaluate-agents-importers && npm run test:evaluate-agents-sealed-brief && npm run test:evaluate-held-attempts && npm run test:evaluate-private && npm run test:evaluate-records && npm run test:evaluate-mcp && npm run test:evaluate-api && npm run test:evaluate-workflow && npm run test:evaluate-tool-use && npm run test:evaluate-promptfoo && npm run test:evaluate-learned-framework && npm run test:docs-tutorial && npm run test:knowledge && npm run test:criteria-fragments && npm run test:doc-count-sources && npm run test:planning-doc-sources && npm run test:doc-counts && npm run test:doc-claim-sources && npm run test:doc-claims && npm run test:doc-invocation-entry && npm run test:doc-invocations && npm run test:doc-link-anchors && npm run test:docs-build-names && npm run test:doc-shell-blocks && npm run test:doc-ci-samples && npm run test:doc-playwright-examples && npm run test:docs-evaluate-sidebar && npm run test:docs-general-claims && npm run test:docs-gap-reader && npm run test:enforce-hook && npm run test:framework-scaffold && npm run test:isolation-primitives && npm run test:atdd-isolation && npm run test:atdd-net-guard && npm run test:atdd-scoring && npm run test:atdd-workflow-guidance && npm run test:generation-healing && npm run test:framework-scaffold-install-isolation && npm run test:transcript-harness && npm run test:suite-manifest && npm run test:teach-me-testing-scoring && npm run test:teach-cli && npm run test:eval-data && npm run test:eval-atdd-data && npm run test:eval-ci-data && npm run test:eval-framework-scaffold-data && npm run test:eval-nfr-data && npm run test:nfr-validate-corpus && npm run test:nfr-evidence-guidance && npm run test:nfr-cli && npm run test:eval-test-design-data && npm run test:eval-trace-data && npm run test:eval-routing-data && npm run test:eval-routing-boundaries && npm run test:eval-routing-evidence && npm run test:eval-transcript-data && npm run test:eval-teach-me-testing-data && npm run test:eval-schemas && npm run test:eval-diagnostics && npm run test:eval-replay && npm run test:eval-quality-corpus && npm run test:schema-versions && npm run test:contract-sources && npm run test:contracts && npm run test:contract-oracles && npm run test:probe-sources && npm run test:test-design-qualification && npm run test:test-review-qualification && npm run test:trace-qualification && npm run test:nfr-qualification && npm run test:ci-qualification && npm run test:probe-corpus && npm run test:compare-dominance && npm run test:compare-eval-runs && npm run test:configuration-experiment && npm run test:evaluate-experiments && npm run test:trial-set-scoring && npm run test:automate-cli && npm run test:automate-unit-runners && npm run test:automate-generation-evidence && npm run test:automate-eval-fixture && npm run test:eval-automate-data && npm run test:clock-port && npm run test:probe-conformance && npm run test:corpus-conformance && npm run test:file-system-conformance && npm run test:file-system-port && npm run test:probe-targets && npm run test:port-totality && npm run test:ci-coverage && npm run test:ci-coverage-filters && npm run test:install-actionlint && npm run test:workflows-lint && npm run test:artifact-retry && npm run test:install-linux-sandbox && npm run test:shards && npm run test:groups && npm run test:cli && npm run test:workflow-cli && npm run test:migrate-customizations && npm run test:framework-cli && npm run test:test-review-pr-scope && npm run test:test-review-report && npm run test:test-review-rubric && npm run test:bmad-output-gated && npm run test:release-metadata && npm run test:release-notes && npm run test:lockfile-age && npm run test:lockfile-age-cache && npm run test:licences && npm run test:supply-chain && npm run test:boundary && npm run test:direction && npm run test:lineage && npm run test:guard-publish && npm run test:layering-boundary-lineage && npm run test:changelog && npm run test:tea-workflow-descriptions && npm run test:docs-mermaid && npm run test:llms-headroom && npm run validate:schemas && npm run lint && npm run lint:md && npm run format:check",
+    "test:artifact-retry": "node test/test-artifact-retry.js",
     "test:atdd-isolation": "node test/test-atdd-isolation.js",
     "test:atdd-net-guard": "node test/test-atdd-net-guard.js",
     "test:atdd-scoring": "node test/test-atdd-scoring.js",
@@ -102,6 +104,7 @@
     "test:automate-cli": "node test/test-automate-cli.js",
     "test:automate-eval-fixture": "node test/test-automate-eval-fixture.js",
     "test:automate-generation-evidence": "node test/test-automate-generation-evidence.js",
+    "test:automate-unit-runners": "node test/test-automate-unit-runners.js",
     "test:bmad-output-gated": "node test/test-bmad-output-gated.js",
     "test:boundary": "eval-quality-gates package-boundary",
     "test:changelog": "node test/test-stamp-changelog.js",
@@ -121,6 +124,7 @@
     "test:coverage": "c8 npm test",
     "test:criteria-fragments": "node tools/validate-criteria-fragments.js",
     "test:direction": "eval-quality-gates dependency-direction",
+    "test:doc-ci-samples": "node test/test-doc-ci-samples.js",
     "test:doc-claim-sources": "node test/test-doc-claim-sources.js",
     "test:doc-claims": "eval-quality-gates doc-claims",
     "test:doc-count-sources": "node test/test-doc-count-sources.js",
@@ -128,9 +132,12 @@
     "test:doc-invocation-entry": "node test/test-doc-invocation-entry.js",
     "test:doc-invocations": "eval-quality-gates doc-invocations",
     "test:doc-link-anchors": "node test/test-doc-link-anchors.js",
+    "test:doc-playwright-examples": "node test/test-doc-playwright-examples.js",
+    "test:doc-shell-blocks": "node test/test-doc-shell-blocks.js",
     "test:docs-build-names": "node test/test-docs-build-names.js",
     "test:docs-evaluate-sidebar": "node test/test-docs-evaluate-sidebar.js",
     "test:docs-gap-reader": "node test/test-docs-gap-reader.js",
+    "test:docs-general-claims": "node test/test-docs-general-claims.js",
     "test:docs-layout": "node test/test-docs-layout.js",
     "test:docs-mermaid": "node test/test-docs-mermaid.js",
     "test:docs-tutorial": "node test/test-docs-tutorial.js",
@@ -186,6 +193,7 @@
     "test:evaluate-dogfood": "node test/test-evaluate-dogfood.js",
     "test:evaluate-evaluators": "node test/test-evaluate-evaluators.js --group=evaluators",
     "test:evaluate-evaluators-agent-version": "node test/test-evaluate-evaluators.js --group=evaluators-agent-version",
+    "test:evaluate-experiments": "node test/test-evaluate-experiments.js",
     "test:evaluate-gap-loop": "node test/test-evaluate-gap-loop.js",
     "test:evaluate-generator": "node test/test-evaluate-generator.js",
     "test:evaluate-guidance": "node test/test-evaluate-guidance.js",
@@ -341,6 +349,7 @@
   "devDependencies": {
     "@astrojs/sitemap": "^3.6.0",
     "@astrojs/starlight": "^0.37.0",
+    "@babel/plugin-transform-modules-commonjs": "^7.29.7",
     "@eslint/js": "^9.33.0",
     "@langchain/core": "latest",
     "@playwright/test": "^1.63.0",
@@ -369,6 +378,7 @@
     "promptfoo": "latest",
     "sharp": "^0.33.5",
     "typescript": "^7.0.2",
+    "vitest": "^5.0.3",
     "yaml-eslint-parser": "^1.2.3",
     "yaml-lint": "^1.7.0",
     "zod": "3.25.76",

package/skills/bmad-tea-automate/red/checklist.md

@@ -430,7 +430,7 @@ Only Create runs `{skill-root}/resources/run-and-heal.md`; its execution and sto
 - [ ] On Resume, retained settings and `healing_rounds_used` loaded before any write; zero initialized at first loop entry, including legacy Resume with no counter and no prior repair evidence; otherwise saved count retained
 - [ ] Original generated scope, criterion registry, assertions and source baseline retained
 - [ ] All generated scaffolds executed in a disposable copy; every permanent scaffold skip call and business assertion preserved byte for byte
-- [ ] Compatible available `tea-atdd-red-check` used for browserless loopback tests with project-derived process budget; otherwise project-native runner uses original config/environment
+- [ ] Compatible available `tea-atdd-red-check` used for browserless loopback Playwright tests, or available `tea-unit-check` for Vitest and Jest scaffolds, with project-derived process budget; otherwise project-native runner uses original config/environment
 - [ ] Fresh per-file results capture load errors, every selected project/attempt's actual statuses and full assertion messages
 - [ ] Every criterion's verified red result reaches its intended missing behavior with genuine assertion provenance and no setup/load error
 - [ ] Selector, timing, data, network, hard wait and syntax/import/setup failures classified from runtime/source evidence

package/skills/bmad-tea-automate/resources/run-and-heal.md

@@ -41,9 +41,11 @@ For **red**, permanent acceptance scaffolds retain the existing `test.skip('titl
 
 For a compatible browserless loopback project, execute the verifier against the disposable copy with a fresh report path. Derive its `--per-file-timeout-ms` from the project's existing execution budget: tests selected in that file, configured per-test and hook timeouts, retries and relevant service/global deadlines. Allow the runner to complete those existing budgets plus startup/report overhead. Record the chosen wall-clock bound and its source. The verifier's default 15 seconds must not truncate a project whose declared budget exceeds it. This sets a process execution bound; keep every test/assertion timeout and retry policy unchanged. Supply `--server-command` only when its minimal environment and supported health/base-URL wiring fit the project; otherwise use the native fallback. Read the resulting JSON: verifier exit code 0 means a report was written, regardless of whether the tests passed or failed.
 
+**For Vitest and Jest scaffolds, prefer `tea-unit-check --mode red` when it is available.** Resolve it the same way: the installed executable or the host package's `cli/unit-check.js`. It detects the runner from `package.json` and config files, copies the project, or the whole workspace for a workspace package, to a temporary directory with its `node_modules` linked, blanks only each generated leaf's `.skip` in the copy, runs each `--file` alone with the project's installed runner and its own config, and leaves the saved scaffolds unchanged. Every generated leaf must be a literal `test.skip('title', ...)` or `it.skip('title', ...)` call. Derive `--per-file-timeout-ms` from the project's existing budgets as above. Its report records each test's attempts and failure reason (`assertion`, `timeout` or `error`), load and run errors, and changed files; exit code 0 means the report was written. A headless CLI run accepts only this report as Vitest or Jest red evidence. For expand, `tea-unit-check --mode expand` runs the files in place; a headless run needs it for Vitest, whose JSON reporter omits unhandled errors.
+
 **Use the project-native runner when the verifier is unavailable or incompatible**, including Playwright browser E2E, required project environment or services, unsupported scaffold extensions, and other test frameworks. Record the selected runner and precise fallback reason before executing. Activate only this run's generated leaf scaffolds in the disposable copy using the runner's normal skip mechanism. For Playwright, replace only the generated leaf `test.skip('title', ...)` wrapper with `test('title', ...)` in the copy. Retain all other skip calls, test bodies, assertions, fixtures and production/config files. Do not globally remove skips or activate unrelated tests. Compare each activated file against its baseline to confirm the activation wrapper is the only change. If a scaffold uses an unsupported skip shape or cannot be activated without changing semantics, report the limitation and do not invent a different test.
 
-Run the project's installed runner from the copy with its **original config and environment**, selected project/browser matrix, service startup and existing timeout/retry settings. Preserve needed environment values without printing credentials or copying secrets into a report. Use a supported reporter option or reporter environment setting to write fresh machine-readable results, preferably JSON, and retain the exact command. For Playwright, use the actual project config with its JSON reporter for the selected files; do not inject the verifier's fixture isolation preload or minimal environment into native execution. The evaluation CLI's isolation remains unchanged. Native fallback stays within the caller's existing authorized execution boundary; an evaluation sandbox that blocks a required capability remains a reported blocker and cannot be escaped through this fallback. If the runner cannot safely emit per-test execution/load results with existing tooling, report **could not measure** and the missing capability.
+Run the project's installed runner from the copy with its **original config and environment**, selected project/browser matrix, service startup and existing timeout/retry settings. Preserve needed environment values without printing credentials or copying secrets into a report. Use a supported reporter option or reporter environment setting to write fresh machine-readable results, preferably JSON, and retain the exact command. For Vitest, add `--reporter=default --reporter=json --outputFile.json=<path> --includeTaskLocation` and keep the exit code and unhandled-error banner, which the JSON omits; for Jest, add `--json --testLocationInResults --outputFile=<path>`. Test locations bind each result to its generated leaf. For Playwright, use the actual project config with its JSON reporter for the selected files; do not inject the verifier's fixture isolation preload or minimal environment into native execution. The evaluation CLI's isolation remains unchanged. Native fallback stays within the caller's existing authorized execution boundary; an evaluation sandbox that blocks a required capability remains a reported blocker and cannot be escaped through this fallback. If the runner cannot safely emit per-test execution/load results with existing tooling, report **could not measure** and the missing capability.
 
 Isolate each generated file's execution when a syntax/import error would otherwise prevent sibling files from running. A Playwright load error aborts an invocation before per-test results exist, so run the selected files individually, retaining their original project config and fresh report paths, then aggregate all results. Honor the project's declared execution budgets and service lifecycle in these invocations. Use actual attempt outcomes from each runner, including each selected project/browser result; retries must not conceal a wrong-reason failure as verified red. Every generated leaf must be accounted for. Missing, interrupted, timed-out or skipped results are reported explicitly. Preserve original sources, and compare the disposable copy's production/config files against their pre-execution content and permission baseline to detect additions, removals or modifications. Keep run outputs/cache directories outside that source comparison. Never repair production or config mutations to manufacture a red verdict.
 

package/skills/bmad-tea-evaluate/SKILL.md

@@ -21,6 +21,17 @@ You will continue to operate with your given name, identity, and communication_s
 - `{skill-name}` resolves to the skill directory's basename.
 - Resolve sibling files such as `references/...` and `assets/...` from `{skill-root}`.
 
+## Execution Authority
+
+Resolve intent before activation, customization resolution or any hook. Ordinary evaluation keeps its current workflow unchanged. Native clean/mutated trials, run comparison and baseline acceptance retain their existing authority and meaning.
+
+An explicit native compare/accept request keeps that command's authority, including a request that mentions consistency while comparing saved runs. Saved-result consistency is an experiment route only when the user asks to analyze variability in existing results. Resolve contextual words such as “these results” from the conversation before choosing a route.
+
+- Saved-result consistency or explicitly requested offline experiment analysis: load `references/configuration-experiments.md` and take its offline route directly. Suppress prepend, append and completion hooks. Do not run configuration resolver scripts, model calls, judge calls or fresh trials. Return after reporting the saved evidence.
+- Fresh repeatability or a one-factor configuration experiment: load `references/configuration-experiments.md` directly. Suppress prepend, append and completion hooks throughout this route. No hook can grant execution or spending authority.
+- “Is it repeatable?” without a saved/fresh choice: clarify saved records versus fresh runs before executing anything.
+- Ordinary evaluation, ordinary offline record evaluation or native compare/accept (including comparing saved runs): proceed with On Activation below. Do not load the experiment guide, analyze experiments or add trials after the normal result.
+
 ## On Activation
 
 Before resolving customizations or running hooks, inspect `{project-root}/_bmad/custom/` for `bmad-testarch-evaluate.toml` and `bmad-testarch-evaluate.user.toml`. If either exists, stop and tell the user to run `npx --package=bmad-method-test-architecture-enterprise@3 tea-migrate-customizations --project-root {project-root}`. Resolve any reported manual merge, then start a fresh chat. Do not continue with default settings while legacy overrides remain.

package/skills/bmad-tea-evaluate/references/configuration-experiments.md

@@ -0,0 +1,37 @@
+# Optional saved-result and configuration experiments
+
+Enter only after explicit experiment intent. Return directly after this route; activation and completion hooks stay suppressed. Ordinary Evaluate, native clean/mutated trials, native comparison and baseline acceptance keep their existing flow.
+
+## Route by intent
+
+- “Check how consistent the saved results from my latest Test Design evaluation are. Use only existing records. Do not rerun anything or make model calls.” means offline analysis.
+- “Compare Test Design at medium versus high reasoning effort on the same model” means a live proposal with a spending boundary.
+- “Compare Test Design model A versus B” means a one-factor proposal. “Compare one prompt-region change for Test Design” means a proposal that reports the unavailable prompt-region adapter before any spending approval.
+- “Run a fresh repeatability experiment” means a live proposal.
+- “Is it repeatable?” needs one clarification: existing saved results or fresh runs. Dispatch nothing before the answer.
+
+## Offline analysis
+
+Own discovery, input preparation, reader invocation and the report. Ask for the saved-run path or search root only when none is known from the conversation or evaluation output. Use the latest compatible records under that root and state every selected path. Do not ask the user to edit JSON, copy provenance or execute helper commands.
+
+Run the installed `tea-evaluate experiment --search-root <root> --out <new-report-directory>`, or use `--saved <exact-directory>`. In this repository use `node cli/evaluate.js experiment`. These commands read saved bytes and replay the deterministic Test Design scorer; they launch no target, model, judge or subprocess. Never invoke preflight, run, custom hooks or capture to fill a gap in saved evidence.
+
+Supported inputs are sealed legacy Test Design reader directories, native retained per-attempt directories and new experiment directories. Preserve original records and integrity files. For missing diagnostics, input identities or resources, report the resulting limitation. Use shipped ground truth only for matching native cases and input identity. An incompatible corpus needs its original companion ground truth through the legacy reader; this route cannot invent an adapter.
+
+No retrospective success threshold is invented. Explain spread, counts, recorded native stability and deterministic scorer replay separately. Historical receipts stay historical. A descriptive finding, a native gate failure, an inconclusive result, selection clearance and a confirmed recommendation remain distinct.
+
+## Live proposal and authority
+
+Check capability and corpus feasibility before asking for spending approval. Current Claude/Codex adapters cannot prove applied effort or enforce an all-in money cap. Codex also lacks verified money telemetry. Explain these blockers and dispatch zero attempts. Subscription access supplies no verified zero-dollar budget. The native corpus has two independent cases; selection consumes both, leaving no unseen confirmation cases. Repetitions cannot repair this limit. No corpus expansion is authorized.
+
+For a supported future mechanism or a labeled deterministic test double, batch consequential choices into one concrete plan approval: intended outcome, allowed quality loss or required improvement, critical failures, useful saving or latency limits, cases, repeats, retry bound, elapsed estimate, confirmation scope and verified all-in USD cap including overhead. Reuse previously approved thresholds only when they apply to the same identity. Missing decisions remain pending. Preparing a proposal grants no dispatch authority.
+
+The controller freezes declaration, native input/scorer identities, gates, analysis, schedule, stopping rules and execution identity before approval. Approval binds the exact plan digest and cap. The local unsigned journal validates recorded lifecycle order and bindings. It cannot prove tail completeness, resist wholesale rechaining or establish independent wall-clock chronology. Exposure history covers this experiment; prior candidate exposure is unknown. Every dispatch reserves its verified bound first. Failed, partial, canceled, timed-out and retried attempts stay in the archive and spending ledger. Missing telemetry, rejected capture integrity, archival failure or an unresolved reservation stops further dispatch. Recovery reports preserve originals and partial archives, and never rerun an unsettled attempt.
+
+The public CLI supports offline analysis, blocked native proposals through `--propose`, and reservation inspection through `--recover`. It supplies no approval or execution command. Approval and execution APIs currently serve the labeled deterministic integration tests; the native capability blockers prevent using those APIs to run providers.
+
+The implementation's process doubles are labeled `deterministic-test-double`. They verify orchestration and capture and establish no provider quality or savings. Paid smoke runs require separate approval. Production changes, defaults, baseline acceptance and recommendation adoption each retain their own authority.
+
+## Report
+
+Give a short result, what the evidence establishes, what remains unknown, all-attempt cost and resource limits, and links to the selected records and report. Name blockers precisely and give the next supported action. Keep this route optional; never start it after a routine evaluation result without explicit intent.

package/skills/bmad-tea-evaluate/references/evaluator.md

@@ -20,7 +20,7 @@ Stage 6 may have prepared a deterministic rubric judge. When choosing `command`
 
 `command` and `sealed-brief-agent` produce one JSON answer per trial: `{ "rows": [...], "recommendation": "PASS" }`. Each row names a mapped `key`, an `outcome` of `pass`, `fail` or `score`, and `observationIds`. A `fail` also needs a verbatim `quote`, `quoteChannel`, `confidence` and `comment`; a `score` needs an anchored integer `score`. TeA converts these rows through `evaluator/mapping.json` into sealed run records. `deterministic` produces its records within TeA. `records` imports schema-valid `SealedRunRecord` files from the adopter harness. See `assets/evaluators/command-evaluator.mjs`, `assets/evaluators/mapping.json` and `assets/evaluators/frameworks.json` for starter shapes.
 
-For a sealed-brief agent, set `evaluation.json.evaluator` to `{ "kind": "sealed-brief-agent", "agent": "claude", "model": "<immutable-model-id>", "timeoutMs": 30000 }`, then supply a compatible installed agent and credentials. Keep `evaluator/mapping.json` with the judgment keys and add `policy/evaluator-conditions.json` with `evaluator.modelSnapshot` naming the same immutable model ID as `evaluator.model`; `check` exits 10 when the two differ. Check that the selected adapter supports bridged runs and record its installed version in `evaluator/LEARNED.md`. The [Evaluate CLI reference](https://github.com/bmad-code-org/bmad-method-test-architecture-enterprise/blob/main/docs/reference/tea-evaluate-cli.md#the-evaluation-layer) documents the bridge and model conditions.
+For a sealed-brief agent, set `evaluation.json.evaluator` to `{ "kind": "sealed-brief-agent", "agent": "claude", "model": "<immutable-model-id>", "timeoutMs": 30000 }`, then supply a compatible installed agent and credentials. Keep `evaluator/mapping.json` with the judgment keys and add `policy/evaluator-conditions.json` with `evaluator.modelSnapshot` naming the same immutable model ID as `evaluator.model`; `check` exits 10 when the two differ. Check that the selected adapter supports bridged runs and record its installed version in `evaluator/LEARNED.md`. The [Evaluate CLI reference](https://github.com/bmad-code-org/bmad-method-test-architecture-enterprise/blob/main/docs/reference/tea-evaluate-evaluators.md#the-evaluation-layer) documents the bridge and model conditions.
 
 When the target itself uses no model, a valid starting shape for those conditions is `{ "schemaVersion": 1, "modelSnapshot": "none", "systemPromptDigest": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "evaluator": { "modelSnapshot": "<exact-agent-model-snapshot>" } }`. Replace the placeholder with the immutable ID that `evaluator.model` holds. When the target uses a model, name that target snapshot and its prompt digest in the top-level fields. Keep agent credentials outside committed files.
 
@@ -34,7 +34,7 @@ Write `evaluator-configuration.json` first (the two `tea.judgeCalibration*` bind
 
 `check` and `run` verify the judgments and bindings with one function: absent or unverifiable ones exit 10 (`check` reports the `judge-calibration` rule). `run` then computes agreement and exits 11 below the minimum before any record is copied, with `judge-calibration.json` in the run directory. A `records` evaluation with no rubric needs no judgments file. The runtime verifies the binding and the label-free input and cannot prove the harness ran its scorer; a harness that needs that proof runs its scorer through a `command` evaluator.
 
-The [Evaluate CLI reference](https://github.com/bmad-code-org/bmad-method-test-architecture-enterprise/blob/main/docs/reference/tea-evaluate-cli.md#the-evaluation-layer) gives the required record layout and the calibration judgments layout.
+The [Evaluate CLI reference](https://github.com/bmad-code-org/bmad-method-test-architecture-enterprise/blob/main/docs/reference/tea-evaluate-evaluators.md#the-evaluation-layer) gives the required record layout and the calibration judgments layout.
 
 ### Installed agent adapter version
 

package/skills/bmad-tea-evaluate/references/gaps.md

@@ -96,7 +96,7 @@ An empty `observedMounts` is evidence only where the audit ran and kept every re
 A Linux target whose call to a model provider or an outside HTTPS service fails to connect runs in a network namespace with a loopback and nothing else. List the host in `egress` on its entry with the addresses it resolves to, confirm `run.json` lists the entry under `egress`, read `egressRefusals` for the host and port the proxy refused, and rerun.
 The shim announces the proxy in `HTTPS_PROXY` alone and the proxy reads `CONNECT` alone, so a client that opens no `CONNECT` tunnel (a plain `http://` request, a database driver) has no route, and such a target opts out with `"confinement": false` and the adopter's recorded reason.
 A tunnel to a listed host and port carries whatever bytes the client sends, TLS or not, so a client that tunnels reaches a plain-HTTP gateway on the host's loopback that its entry lists.
-A confined target cannot reach a host service through a socket file that exists when its call starts: a connection to the Docker socket (testcontainers) or to a database's Unix socket (a `.s.PGSQL.5432` file) answers `ECONNREFUSED` from a Bubblewrap target, since the runtime mounts an empty device file over each socket file it lists outside the call's own grants, and `EPERM` from a macOS Seatbelt target, since its profile denies every connection to a socket path outside the call's own grants. A socket file a host process binds after the call started is reachable wherever nothing masks or denies its path: under Bubblewrap anywhere outside the masks, on macOS only inside the granted paths and the bridge's shape beneath the private root, since seccomp cannot read a socket's path, a network namespace does not scope path sockets and AppArmor needs a profile that root loads. A target that needs one opts out with `"confinement": false` and the adopter's recorded reason. An exit 12 that names file-system confinement or its audit is a host or project condition: repair it as the run guide's `## Run confined` describes, and find each further cause (a failed step in building the target's private git repository, an observer that fails mid-run) in the [Evaluate CLI reference](https://github.com/bmad-code-org/bmad-method-test-architecture-enterprise/blob/main/docs/reference/tea-evaluate-cli.md#file-system-confinement).
+A confined target cannot reach a host service through a socket file that exists when its call starts: a connection to the Docker socket (testcontainers) or to a database's Unix socket (a `.s.PGSQL.5432` file) answers `ECONNREFUSED` from a Bubblewrap target, since the runtime mounts an empty device file over each socket file it lists outside the call's own grants, and `EPERM` from a macOS Seatbelt target, since its profile denies every connection to a socket path outside the call's own grants. A socket file a host process binds after the call started is reachable wherever nothing masks or denies its path: under Bubblewrap anywhere outside the masks, on macOS only inside the granted paths and the bridge's shape beneath the private root, since seccomp cannot read a socket's path, a network namespace does not scope path sockets and AppArmor needs a profile that root loads. A target that needs one opts out with `"confinement": false` and the adopter's recorded reason. An exit 12 that names file-system confinement or its audit is a host or project condition: repair it as the run guide's `## Run confined` describes, and find each further cause (a failed step in building the target's private git repository, an observer that fails mid-run) in the [Evaluate CLI reference](https://github.com/bmad-code-org/bmad-method-test-architecture-enterprise/blob/main/docs/reference/tea-evaluate-confinement.md#file-system-confinement).
 
 ## Map AD-10 exits and classes to repairs
 

package/skills/bmad-tea-test-review/steps-c/step-03f-aggregate-scores.md

@@ -374,7 +374,9 @@ fs.writeFileSync(`/tmp/tea-test-review-summary-${runKey}-${timestamp}.json`, JSO
 ### 6. Display Summary to User
 
 ```text
-✅ Quality Evaluation Complete (Parallel Execution)
+✅ Quality Evaluation Complete
+
+⚙️ Execution Mode: {execution_mode}
 
 📊 Overall Quality Score: {roundedScore}/100 (Grade: {overallGrade})
 
@@ -393,8 +395,6 @@ fs.writeFileSync(`/tmp/tea-test-review-summary-${runKey}-${timestamp}.json`, JSO
 - LOW:      {low_count} violations
 - TOTAL:    {total_count} violations
 
-🚀 Performance: Parallel execution ~60% faster than sequential
-
 ✅ Ready for report generation (Step 4)
 ```
 

package/skills/bmod-tea/bmod.toml

@@ -2,7 +2,7 @@
 
 [bmod]
 code = "tea"
-version = "3.1.0"
+version = "3.3.0"
 update_source = "github:bmad-code-org/bmad-method-test-architecture-enterprise/skills"
 skills = [
   "bmad-tea",

package/skills/bmod-tea/help/help.md

@@ -9,7 +9,7 @@ This document covers the skills of the `tea` module: what each one gives the use
 - `bmad-tea-test-design`: risk-based test planning, producing a test design document. Recommend it during solutioning, before tests or a framework exist. Afterwards offer `bmad-tea-framework`.
 - `bmad-tea-framework`: owns production-ready framework and CI setup, with framework-only, CI-only, or both scope. It installs Playwright Utils or Pact.js Utils when enabled. Recommend it during solutioning, after test design. When CI scope cannot be inferred from the request, interactive runs ask whether to include CI. Unattended runs default to framework only and state in the summary that CI was excluded.
 - `bmad-tea-automate`: owns test generation in red or expand mode. Red mode writes failing acceptance tests and an ATDD checklist before implementation, then verifies the intended missing behavior. Expand mode writes tests for implemented code, runs them, heals test defects for up to three rounds by default, and reports product defects. Recommend red mode after a story is created and expand mode once code exists.
-- `bmad-tea-evaluate`: builds and runs a scored, evidence-backed evaluation of an agent, skill, workflow or tool-use system. Recommend it after automation when the target's behavior needs proof. Afterwards offer `bmad-tea-framework` with CI scope to enforce it.
+- `bmad-tea-evaluate`: builds and runs a scored, evidence-backed evaluation of an agent, skill, workflow or tool-use system. Recommend it after automation when the target's behavior needs proof. Afterwards offer `bmad-tea-framework` with CI scope to enforce it. On explicit request, Evaluate also analyzes saved native Test Design consistency offline or prepares a bounded fresh/configuration proposal. “Is it repeatable?” first clarifies saved records versus new runs. Native live experiments remain blocked by telemetry and enforceable-budget gaps; ordinary evaluation gains no extra experiment work.
 - `bmad-tea-test-review`: audits test quality with a 0-100 score and a review report. Recommend it after tests are written. Afterwards offer `bmad-tea-trace`.
 - `bmad-tea-nfr`: audits the evidence for non-functional requirements and writes an NFR report. Recommend it after automation, before a release.
 - `bmad-tea-trace`: maps requirements to tests in a traceability matrix and makes the PASS, CONCERNS, FAIL or WAIVED gate decision. Recommend it after test review, for the release gate.

package/skills/bmod-tea/knowledge/auth-session.md

@@ -90,39 +90,44 @@ test('authenticated request', async ({ authToken, request }) => {
 **Implementation**:
 
 ```typescript
+import { expect } from '@playwright/test';
 import { test } from '../support/auth/auth-fixture';
 
-// Option 1: Per-test user override
-test('admin actions', async ({ authToken, authOptions }) => {
-  // Override default user
-  authOptions.userIdentifier = 'admin';
+// The user is chosen before the test body runs: `authToken`, `context` and `page`
+// resolve from `authOptions` during fixture setup, so assigning
+// `authOptions.userIdentifier` inside a test changes nothing.
 
-  const { authToken: adminToken } = await test.step('Get admin token', async () => {
-    return { authToken }; // Re-fetches with new identifier
-  });
+// Option 1: Override the user for a group of tests
+test.describe('admin actions', () => {
+  test.use({ authOptions: { userIdentifier: 'admin' } });
 
-  // Use admin token
-  const response = await request.get('/api/admin/users', {
-    headers: { Authorization: `Bearer ${adminToken}` },
+  test('lists users', async ({ authToken, request }) => {
+    const response = await request.get('/api/admin/users', {
+      headers: { Authorization: `Bearer ${authToken}` }, // admin token
+    });
+    expect(response.ok()).toBe(true);
   });
 });
 
-// Option 2: Parallel execution with different users
-test.describe.parallel('multi-user tests', () => {
-  test('user 1 actions', async ({ authToken }) => {
-    // Uses default user (e.g., 'user1')
-  });
+// Option 2: Parallel execution with different users, one describe block per user
+test.describe('multi-user tests', () => {
+  test.describe.configure({ mode: 'parallel' });
 
-  test('user 2 actions', async ({ authToken, authOptions }) => {
-    authOptions.userIdentifier = 'user2';
-    // Uses different token for user2
-  });
+  for (const userIdentifier of ['user1', 'user2']) {
+    test.describe(`as ${userIdentifier}`, () => {
+      test.use({ authOptions: { userIdentifier } });
+
+      test('sees own profile', async ({ page }) => {
+        await page.goto('/profile'); // signed in as this user
+      });
+    });
+  }
 });
 ```
 
 **Key Points**:
 
-- Override `authOptions.userIdentifier` per test
+- Override `authOptions.userIdentifier` with `test.use()` inside a `test.describe()` block; the test body runs after the token resolves
 - Tokens cached separately per user identifier
 - Parallel tests isolated with different users
 - Worker-specific accounts possible

package/skills/bmod-tea/knowledge/test-healing-patterns.md

@@ -92,6 +92,8 @@ A parser error or unresolved import prevents the acceptance criterion from execu
 
 Permanent Playwright acceptance scaffolds keep `test.skip('title', ...)`. Verify a disposable copy, preserving original files and every business assertion. Prefer the available `tea-atdd-red-check` for compatible browserless loopback projects: it uses a minimal worker environment, refuses worker child processes including browser launches, and supports `*.spec.ts` scaffolds. Derive its per-file process bound from the project's existing execution budgets; the default 15 seconds cannot replace a longer declared test budget. Its successful exit means the report exists.
 
+Permanent Vitest and Jest scaffolds keep `test.skip('title', ...)` or `it.skip('title', ...)`. Prefer the available `tea-unit-check --mode red`: it copies the project with linked dependencies, blanks only the generated leaves' `.skip` in the copy, runs each file alone with the installed runner and project config, and reports every attempt's failure as `assertion`, `timeout` or `error`, plus unhandled errors Vitest's JSON omits. A headless run accepts only this report as Vitest or Jest red evidence.
+
 For browser tests, required project environment/services, non-loopback endpoints, another runner/format or a skill-only install with no verifier, use the actual project-native runner on the disposable activated copy with the original configuration and environment. Activate only the generated leaf scaffold wrappers. Preserve other skips and all test bodies. Run files separately when a load error would mask sibling results; capture fresh per-file JSON, every project/attempt's actual status, full assertion messages and load/setup errors. Compare production/config content and permissions before and after execution. Record the fallback reason and runner/budget in the summary. Keep the evaluation CLI's isolation unchanged.
 
 Map each result to the criterion's recorded failure signature and establish that the assertion executed. Skipped, empty, unmapped, timed-out and wrong-reason results cannot count as verified red. Repair wrong-reason syntax/import/locator/setup defects only until the expected missing-behavior failure is reached. Preserve that failure and keep production unchanged. A runner without usable per-test evidence reports could not measure.