maestro-flow 0.3.32 → 0.3.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/commands/learn-retro.md +1 -1
- package/.claude/commands/maestro-composer.md +2 -2
- package/.claude/commands/maestro-ralph-execute.md +1 -1
- package/.claude/commands/maestro-ralph.md +11 -11
- package/.claude/commands/maestro-verify.md +1 -1
- package/.claude/commands/maestro.md +1 -2
- package/.claude/commands/quality-auto-test.md +124 -0
- package/.claude/commands/quality-refactor.md +1 -1
- package/.claude/commands/quality-test.md +3 -3
- package/.codex/skills/learn-retro/SKILL.md +1 -1
- package/.codex/skills/maestro/SKILL.md +1 -1
- package/.codex/skills/maestro-link-coordinate/SKILL.md +1 -1
- package/.codex/skills/maestro-milestone-audit/SKILL.md +9 -1
- package/.codex/skills/maestro-plan/SKILL.md +72 -6
- package/.codex/skills/maestro-ralph/SKILL.md +159 -159
- package/.codex/skills/maestro-roadmap/SKILL.md +31 -5
- package/.codex/skills/maestro-verify/SKILL.md +1 -1
- package/.codex/skills/quality-auto-test/SKILL.md +547 -0
- package/.codex/skills/quality-debug/SKILL.md +18 -6
- package/.codex/skills/quality-refactor/SKILL.md +8 -1
- package/.codex/skills/quality-sync/SKILL.md +23 -4
- package/.codex/skills/quality-test/SKILL.md +498 -166
- package/dashboard/dist-server/src/config/cli-tools-config.js +5 -20
- package/dashboard/dist-server/src/config/cli-tools-config.js.map +1 -1
- package/dashboard/dist-server/src/config/cli-tools-defaults.json +17 -0
- package/dist/src/commands/config.d.ts.map +1 -1
- package/dist/src/commands/config.js +168 -32
- package/dist/src/commands/config.js.map +1 -1
- package/dist/src/commands/hooks.d.ts +9 -0
- package/dist/src/commands/hooks.d.ts.map +1 -1
- package/dist/src/commands/hooks.js +1 -1
- package/dist/src/commands/hooks.js.map +1 -1
- package/dist/src/commands/install.js +1 -1
- package/dist/src/commands/install.js.map +1 -1
- package/dist/src/commands/overlay.js +1 -1
- package/dist/src/commands/overlay.js.map +1 -1
- package/dist/src/commands/tools.d.ts.map +1 -1
- package/dist/src/commands/tools.js +7 -17
- package/dist/src/commands/tools.js.map +1 -1
- package/dist/src/commands/uninstall.js +1 -1
- package/dist/src/commands/uninstall.js.map +1 -1
- package/dist/src/config/cli-tools-config.d.ts.map +1 -1
- package/dist/src/config/cli-tools-config.js +5 -20
- package/dist/src/config/cli-tools-config.js.map +1 -1
- package/dist/src/config/cli-tools-defaults.json +17 -0
- package/dist/src/tui/config-ui/ConfigHub.d.ts +14 -0
- package/dist/src/tui/config-ui/ConfigHub.d.ts.map +1 -0
- package/dist/src/tui/config-ui/ConfigHub.js +125 -0
- package/dist/src/tui/config-ui/ConfigHub.js.map +1 -0
- package/dist/src/tui/config-ui/ConfigSourcesView.d.ts +6 -0
- package/dist/src/tui/config-ui/ConfigSourcesView.d.ts.map +1 -0
- package/dist/src/tui/config-ui/ConfigSourcesView.js +25 -0
- package/dist/src/tui/config-ui/ConfigSourcesView.js.map +1 -0
- package/dist/src/tui/config-ui/HooksPanel.d.ts +6 -0
- package/dist/src/tui/config-ui/HooksPanel.d.ts.map +1 -0
- package/dist/src/tui/config-ui/HooksPanel.js +92 -0
- package/dist/src/tui/config-ui/HooksPanel.js.map +1 -0
- package/dist/src/tui/config-ui/SkillConfigDashboard.d.ts +9 -0
- package/dist/src/tui/config-ui/SkillConfigDashboard.d.ts.map +1 -0
- package/dist/src/tui/config-ui/SkillConfigDashboard.js +64 -0
- package/dist/src/tui/config-ui/SkillConfigDashboard.js.map +1 -0
- package/dist/src/tui/config-ui/SkillParamEditor.d.ts +12 -0
- package/dist/src/tui/config-ui/SkillParamEditor.d.ts.map +1 -0
- package/dist/src/tui/config-ui/SkillParamEditor.js +162 -0
- package/dist/src/tui/config-ui/SkillParamEditor.js.map +1 -0
- package/dist/src/tui/config-ui/SkillsList.d.ts +12 -0
- package/dist/src/tui/config-ui/SkillsList.d.ts.map +1 -0
- package/dist/src/tui/config-ui/SkillsList.js +91 -0
- package/dist/src/tui/config-ui/SkillsList.js.map +1 -0
- package/dist/src/tui/config-ui/SpecPanel.d.ts +6 -0
- package/dist/src/tui/config-ui/SpecPanel.d.ts.map +1 -0
- package/dist/src/tui/config-ui/SpecPanel.js +79 -0
- package/dist/src/tui/config-ui/SpecPanel.js.map +1 -0
- package/dist/src/tui/config-ui/index.d.ts +23 -0
- package/dist/src/tui/config-ui/index.d.ts.map +1 -0
- package/dist/src/tui/config-ui/index.js +34 -0
- package/dist/src/tui/config-ui/index.js.map +1 -0
- package/dist/src/tui/index.d.ts +7 -0
- package/dist/src/tui/index.d.ts.map +1 -0
- package/dist/src/tui/index.js +10 -0
- package/dist/src/tui/index.js.map +1 -0
- package/dist/src/tui/install-ui/BackupConfig.d.ts +10 -0
- package/dist/src/tui/install-ui/BackupConfig.d.ts.map +1 -0
- package/dist/src/tui/install-ui/BackupConfig.js +46 -0
- package/dist/src/tui/install-ui/BackupConfig.js.map +1 -0
- package/dist/src/tui/install-ui/BlueprintPreview.d.ts +9 -0
- package/dist/src/tui/install-ui/BlueprintPreview.d.ts.map +1 -0
- package/dist/src/tui/install-ui/BlueprintPreview.js +65 -0
- package/dist/src/tui/install-ui/BlueprintPreview.js.map +1 -0
- package/dist/src/tui/install-ui/BlueprintPreview.logic.d.ts +33 -0
- package/dist/src/tui/install-ui/BlueprintPreview.logic.d.ts.map +1 -0
- package/dist/src/tui/install-ui/BlueprintPreview.logic.js +51 -0
- package/dist/src/tui/install-ui/BlueprintPreview.logic.js.map +1 -0
- package/dist/src/tui/install-ui/ComponentGrid.d.ts +13 -0
- package/dist/src/tui/install-ui/ComponentGrid.d.ts.map +1 -0
- package/dist/src/tui/install-ui/ComponentGrid.js +75 -0
- package/dist/src/tui/install-ui/ComponentGrid.js.map +1 -0
- package/dist/src/tui/install-ui/ComponentGrid.logic.d.ts +31 -0
- package/dist/src/tui/install-ui/ComponentGrid.logic.d.ts.map +1 -0
- package/dist/src/tui/install-ui/ComponentGrid.logic.js +58 -0
- package/dist/src/tui/install-ui/ComponentGrid.logic.js.map +1 -0
- package/dist/src/tui/install-ui/ConfigPanel.d.ts +11 -0
- package/dist/src/tui/install-ui/ConfigPanel.d.ts.map +1 -0
- package/dist/src/tui/install-ui/ConfigPanel.js +35 -0
- package/dist/src/tui/install-ui/ConfigPanel.js.map +1 -0
- package/dist/src/tui/install-ui/CyberItem.d.ts +18 -0
- package/dist/src/tui/install-ui/CyberItem.d.ts.map +1 -0
- package/dist/src/tui/install-ui/CyberItem.js +33 -0
- package/dist/src/tui/install-ui/CyberItem.js.map +1 -0
- package/dist/src/tui/install-ui/CyberdeckBlueprint.d.ts +7 -0
- package/dist/src/tui/install-ui/CyberdeckBlueprint.d.ts.map +1 -0
- package/dist/src/tui/install-ui/CyberdeckBlueprint.js +91 -0
- package/dist/src/tui/install-ui/CyberdeckBlueprint.js.map +1 -0
- package/dist/src/tui/install-ui/ExecutionView.d.ts +12 -0
- package/dist/src/tui/install-ui/ExecutionView.d.ts.map +1 -0
- package/dist/src/tui/install-ui/ExecutionView.js +154 -0
- package/dist/src/tui/install-ui/ExecutionView.js.map +1 -0
- package/dist/src/tui/install-ui/GradientHeader.d.ts +8 -0
- package/dist/src/tui/install-ui/GradientHeader.d.ts.map +1 -0
- package/dist/src/tui/install-ui/GradientHeader.js +18 -0
- package/dist/src/tui/install-ui/GradientHeader.js.map +1 -0
- package/dist/src/tui/install-ui/HooksConfig.d.ts +8 -0
- package/dist/src/tui/install-ui/HooksConfig.d.ts.map +1 -0
- package/dist/src/tui/install-ui/HooksConfig.js +35 -0
- package/dist/src/tui/install-ui/HooksConfig.js.map +1 -0
- package/dist/src/tui/install-ui/InstallConfirm.d.ts +27 -0
- package/dist/src/tui/install-ui/InstallConfirm.d.ts.map +1 -0
- package/dist/src/tui/install-ui/InstallConfirm.js +25 -0
- package/dist/src/tui/install-ui/InstallConfirm.js.map +1 -0
- package/dist/src/tui/install-ui/InstallExecution.d.ts +21 -0
- package/dist/src/tui/install-ui/InstallExecution.d.ts.map +1 -0
- package/dist/src/tui/install-ui/InstallExecution.js +154 -0
- package/dist/src/tui/install-ui/InstallExecution.js.map +1 -0
- package/dist/src/tui/install-ui/InstallFlow.d.ts +14 -0
- package/dist/src/tui/install-ui/InstallFlow.d.ts.map +1 -0
- package/dist/src/tui/install-ui/InstallFlow.js +186 -0
- package/dist/src/tui/install-ui/InstallFlow.js.map +1 -0
- package/dist/src/tui/install-ui/InstallHub.d.ts +33 -0
- package/dist/src/tui/install-ui/InstallHub.d.ts.map +1 -0
- package/dist/src/tui/install-ui/InstallHub.js +97 -0
- package/dist/src/tui/install-ui/InstallHub.js.map +1 -0
- package/dist/src/tui/install-ui/InstallResult.d.ts +7 -0
- package/dist/src/tui/install-ui/InstallResult.d.ts.map +1 -0
- package/dist/src/tui/install-ui/InstallResult.js +15 -0
- package/dist/src/tui/install-ui/InstallResult.js.map +1 -0
- package/dist/src/tui/install-ui/McpConfig.d.ts +12 -0
- package/dist/src/tui/install-ui/McpConfig.d.ts.map +1 -0
- package/dist/src/tui/install-ui/McpConfig.js +69 -0
- package/dist/src/tui/install-ui/McpConfig.js.map +1 -0
- package/dist/src/tui/install-ui/ResultDashboard.d.ts +8 -0
- package/dist/src/tui/install-ui/ResultDashboard.d.ts.map +1 -0
- package/dist/src/tui/install-ui/ResultDashboard.js +17 -0
- package/dist/src/tui/install-ui/ResultDashboard.js.map +1 -0
- package/dist/src/tui/install-ui/ReviewPanel.d.ts +19 -0
- package/dist/src/tui/install-ui/ReviewPanel.d.ts.map +1 -0
- package/dist/src/tui/install-ui/ReviewPanel.js +25 -0
- package/dist/src/tui/install-ui/ReviewPanel.js.map +1 -0
- package/dist/src/tui/install-ui/ShortcutFooter.d.ts +7 -0
- package/dist/src/tui/install-ui/ShortcutFooter.d.ts.map +1 -0
- package/dist/src/tui/install-ui/ShortcutFooter.js +15 -0
- package/dist/src/tui/install-ui/ShortcutFooter.js.map +1 -0
- package/dist/src/tui/install-ui/StatuslineConfig.d.ts +11 -0
- package/dist/src/tui/install-ui/StatuslineConfig.d.ts.map +1 -0
- package/dist/src/tui/install-ui/StatuslineConfig.js +39 -0
- package/dist/src/tui/install-ui/StatuslineConfig.js.map +1 -0
- package/dist/src/tui/install-ui/StepSelector.d.ts +14 -0
- package/dist/src/tui/install-ui/StepSelector.d.ts.map +1 -0
- package/dist/src/tui/install-ui/StepSelector.js +46 -0
- package/dist/src/tui/install-ui/StepSelector.js.map +1 -0
- package/dist/src/tui/install-ui/index.d.ts +8 -0
- package/dist/src/tui/install-ui/index.d.ts.map +1 -0
- package/dist/src/tui/install-ui/index.js +10 -0
- package/dist/src/tui/install-ui/index.js.map +1 -0
- package/dist/src/tui/install-ui/types.d.ts +35 -0
- package/dist/src/tui/install-ui/types.d.ts.map +1 -0
- package/dist/src/tui/install-ui/types.js +25 -0
- package/dist/src/tui/install-ui/types.js.map +1 -0
- package/dist/src/tui/overlay-ui/OverlayList.d.ts +37 -0
- package/dist/src/tui/overlay-ui/OverlayList.d.ts.map +1 -0
- package/dist/src/tui/overlay-ui/OverlayList.js +146 -0
- package/dist/src/tui/overlay-ui/OverlayList.js.map +1 -0
- package/dist/src/tui/overlay-ui/index.d.ts +18 -0
- package/dist/src/tui/overlay-ui/index.d.ts.map +1 -0
- package/dist/src/tui/overlay-ui/index.js +204 -0
- package/dist/src/tui/overlay-ui/index.js.map +1 -0
- package/dist/src/tui/render.d.ts +13 -0
- package/dist/src/tui/render.d.ts.map +1 -0
- package/dist/src/tui/render.js +31 -0
- package/dist/src/tui/render.js.map +1 -0
- package/dist/src/tui/tools-ui/CommandReference.d.ts +7 -0
- package/dist/src/tui/tools-ui/CommandReference.d.ts.map +1 -0
- package/dist/src/tui/tools-ui/CommandReference.js +33 -0
- package/dist/src/tui/tools-ui/CommandReference.js.map +1 -0
- package/dist/src/tui/tools-ui/ConfigSources.d.ts +6 -0
- package/dist/src/tui/tools-ui/ConfigSources.d.ts.map +1 -0
- package/dist/src/tui/tools-ui/ConfigSources.js +27 -0
- package/dist/src/tui/tools-ui/ConfigSources.js.map +1 -0
- package/dist/src/tui/tools-ui/RegisterSettings.d.ts +8 -0
- package/dist/src/tui/tools-ui/RegisterSettings.d.ts.map +1 -0
- package/dist/src/tui/tools-ui/RegisterSettings.js +125 -0
- package/dist/src/tui/tools-ui/RegisterSettings.js.map +1 -0
- package/dist/src/tui/tools-ui/RoleMappings.d.ts +9 -0
- package/dist/src/tui/tools-ui/RoleMappings.d.ts.map +1 -0
- package/dist/src/tui/tools-ui/RoleMappings.js +104 -0
- package/dist/src/tui/tools-ui/RoleMappings.js.map +1 -0
- package/dist/src/tui/tools-ui/ToolsDashboard.d.ts +8 -0
- package/dist/src/tui/tools-ui/ToolsDashboard.d.ts.map +1 -0
- package/dist/src/tui/tools-ui/ToolsDashboard.js +60 -0
- package/dist/src/tui/tools-ui/ToolsDashboard.js.map +1 -0
- package/dist/src/tui/tools-ui/ToolsOverview.d.ts +9 -0
- package/dist/src/tui/tools-ui/ToolsOverview.d.ts.map +1 -0
- package/dist/src/tui/tools-ui/ToolsOverview.js +84 -0
- package/dist/src/tui/tools-ui/ToolsOverview.js.map +1 -0
- package/dist/src/tui/tools-ui/index.d.ts +3 -0
- package/dist/src/tui/tools-ui/index.d.ts.map +1 -0
- package/dist/src/tui/tools-ui/index.js +6 -0
- package/dist/src/tui/tools-ui/index.js.map +1 -0
- package/dist/src/tui/uninstall-ui/UninstallFlow.d.ts +7 -0
- package/dist/src/tui/uninstall-ui/UninstallFlow.d.ts.map +1 -0
- package/dist/src/tui/uninstall-ui/UninstallFlow.js +189 -0
- package/dist/src/tui/uninstall-ui/UninstallFlow.js.map +1 -0
- package/dist/src/tui/uninstall-ui/index.d.ts +3 -0
- package/dist/src/tui/uninstall-ui/index.d.ts.map +1 -0
- package/dist/src/tui/uninstall-ui/index.js +6 -0
- package/dist/src/tui/uninstall-ui/index.js.map +1 -0
- package/package.json +1 -1
- package/workflows/auto-test.md +682 -0
- package/workflows/business-test.md +570 -0
- package/workflows/integration-test.md +1 -1
- package/workflows/maestro.codex.md +4 -3
- package/workflows/maestro.md +5 -4
- package/workflows/status.md +2 -2
- package/workflows/test.md +1 -1
- package/workflows/verify.md +1 -1
- package/.claude/commands/quality-business-test.md +0 -110
- package/.claude/commands/quality-integration-test.md +0 -67
- package/.claude/commands/quality-test-gen.md +0 -68
- package/.codex/skills/maestro-ralph-execute/SKILL.md +0 -219
- package/.codex/skills/quality-business-test/SKILL.md +0 -218
- package/.codex/skills/quality-integration-test/SKILL.md +0 -427
- package/.codex/skills/quality-test-gen/SKILL.md +0 -352
|
@@ -0,0 +1,682 @@
|
|
|
1
|
+
# Auto-Test Workflow (Unified)
|
|
2
|
+
|
|
3
|
+
Unified automated testing with intelligent state-based routing. Merges test generation (gap-forward), business testing (PRD-forward), and integration testing (code-forward) into a single pipeline.
|
|
4
|
+
|
|
5
|
+
**Core idea: read project state → auto-select scenario source → shared pipeline from plan through iteration.**
|
|
6
|
+
|
|
7
|
+
Pipeline: Parse → Route → Source Scenarios → Discover Infrastructure → Plan → Write Tests → Execute → Iterate → Report
|
|
8
|
+
Only Step 2 diverges (scenario source). Everything else is shared.
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
### Step 0: Parse Input & Load Specs
|
|
13
|
+
|
|
14
|
+
**Parse arguments:**
|
|
15
|
+
|
|
16
|
+
| Input | Result |
|
|
17
|
+
|-------|--------|
|
|
18
|
+
| No arguments | Error E001 |
|
|
19
|
+
| Phase number | Resolve phase dir from artifact registry |
|
|
20
|
+
| `--max-iter N` | Set MAX_ITER = N (default 5). **1 = single-pass** (no iteration loop) |
|
|
21
|
+
| `--layer L` | Start from / restrict to specific layer (L0\|L1\|L2\|L3) |
|
|
22
|
+
| `--dry-run` | Generate plan only, do not execute |
|
|
23
|
+
| `--re-run` | Re-run only previously failed scenarios |
|
|
24
|
+
|
|
25
|
+
**Resolve phase dir:** from `state.json` artifact registry (`type='execute'`, matching phase). Error E002 if not found.
|
|
26
|
+
|
|
27
|
+
**Load specs:**
|
|
28
|
+
```
|
|
29
|
+
specs_test = maestro spec load --category test
|
|
30
|
+
specs_arch = maestro spec load --category arch
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
`specs_test` for test conventions (Steps 3-5); `specs_arch` for module boundaries (Step 2).
|
|
34
|
+
|
|
35
|
+
---
|
|
36
|
+
|
|
37
|
+
### Step 1: Read State & Route
|
|
38
|
+
|
|
39
|
+
Read project state signals and auto-select scenario source. This is the **sole branch point** in the pipeline.
|
|
40
|
+
|
|
41
|
+
```
|
|
42
|
+
Priority: Resume > Re-run > Spec > Gap > Code
|
|
43
|
+
|
|
44
|
+
1. RESUME:
|
|
45
|
+
Check: ${PHASE_DIR}/.tests/auto-test/state.json exists AND status == "running"
|
|
46
|
+
Compat: also check ${PHASE_DIR}/.tests/integration/state.json (old format)
|
|
47
|
+
Action: offer resume or restart
|
|
48
|
+
If resume: restore session, jump to Step 6 at saved iteration/layer
|
|
49
|
+
If restart: archive to .history/, continue as new
|
|
50
|
+
|
|
51
|
+
2. RE-RUN:
|
|
52
|
+
Check: --re-run flag AND .tests/auto-test/report.json has failed/blocked scenarios
|
|
53
|
+
Compat: also check .tests/business/business-test-report.json (old format)
|
|
54
|
+
Action: load failed/blocked scenarios with status reset to pending
|
|
55
|
+
Skip to Step 4 (scenarios pre-loaded, plan confirmation)
|
|
56
|
+
|
|
57
|
+
3. SPEC:
|
|
58
|
+
Check: .workflow/.spec/SPEC-*/requirements/REQ-*.md exists
|
|
59
|
+
Resolve: SPEC_DIR from index.json.spec_ref or most recent SPEC-*/
|
|
60
|
+
If SPEC_DIR found: set ROUTE = "spec", SPEC_MODE = "full"
|
|
61
|
+
If no spec but has success_criteria: set ROUTE = "spec", SPEC_MODE = "degraded"
|
|
62
|
+
|
|
63
|
+
4. GAP:
|
|
64
|
+
Check: ${PHASE_DIR}/verification.json has gaps[] with status MISSING or PARTIAL
|
|
65
|
+
OR: ${PHASE_DIR}/.tests/coverage-report.json has requirements_uncovered[]
|
|
66
|
+
Action: set ROUTE = "gap"
|
|
67
|
+
|
|
68
|
+
5. CODE:
|
|
69
|
+
Default fallback when no spec package and no gaps detected.
|
|
70
|
+
Action: set ROUTE = "code"
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
**Display route selection:**
|
|
74
|
+
```
|
|
75
|
+
=== AUTO-TEST ===
|
|
76
|
+
来源: {ROUTE} ({reason})
|
|
77
|
+
阶段: {phase_name}
|
|
78
|
+
{IF spec: "Spec: {SPEC_DIR} (mode: {SPEC_MODE})"}
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
---
|
|
82
|
+
|
|
83
|
+
### Step 2: Source Scenarios
|
|
84
|
+
|
|
85
|
+
Execute the route-specific scenario extraction, then normalize ALL scenarios into the unified format. **After this step, the pipeline is identical regardless of source.**
|
|
86
|
+
|
|
87
|
+
#### Route A: `spec` source (PRD-forward)
|
|
88
|
+
|
|
89
|
+
1. Load spec package:
|
|
90
|
+
- `requirements/REQ-*.md` — functional requirements with acceptance criteria
|
|
91
|
+
- `requirements/NFR-*.md` — non-functional requirements
|
|
92
|
+
- `architecture/_index.md` — API endpoints, data model, state machines
|
|
93
|
+
- `epics/EPIC-*.md` — user stories for E2E scenarios
|
|
94
|
+
|
|
95
|
+
2. For each `REQ-NNN-{slug}.md`, parse `## Acceptance Criteria`:
|
|
96
|
+
- Extract each numbered criterion as a scenario seed
|
|
97
|
+
- Map RFC 2119 keywords to priority:
|
|
98
|
+
|
|
99
|
+
| Keyword | Priority | Failure Severity |
|
|
100
|
+
|---------|----------|-----------------|
|
|
101
|
+
| MUST / MUST NOT / SHALL / SHALL NOT | critical | blocker |
|
|
102
|
+
| SHOULD / SHOULD NOT / RECOMMENDED | high | major |
|
|
103
|
+
| MAY / OPTIONAL | medium | minor |
|
|
104
|
+
|
|
105
|
+
3. Classify scenario into layer:
|
|
106
|
+
|
|
107
|
+
| Source | Layer | Category |
|
|
108
|
+
|--------|-------|----------|
|
|
109
|
+
| Architecture API endpoints + REQ AC about request/response | L1 | api_contract |
|
|
110
|
+
| REQ AC about business logic, validation, state changes | L2 | business_rule |
|
|
111
|
+
| REQ AC about state transitions (from architecture state machines) | L2 | state_transition |
|
|
112
|
+
| Epic user stories (multi-step flows) | L3 | user_flow |
|
|
113
|
+
| NFR performance/security constraints | L2 | non_functional |
|
|
114
|
+
|
|
115
|
+
4. **Generate Fixtures** (conditional sub-step):
|
|
116
|
+
|
|
117
|
+
**Tier 1: Schema-derived fixtures**
|
|
118
|
+
```
|
|
119
|
+
FOR each entity in REQ data model:
|
|
120
|
+
valid: object satisfying all field constraints
|
|
121
|
+
invalid: one variant per constraint violation (null_required, empty_string, overflow, wrong_type)
|
|
122
|
+
boundary: edge value variants (min_value, max_value, min_minus_one, max_plus_one)
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
**Tier 2: Acceptance-criteria-derived expectations**
|
|
126
|
+
```
|
|
127
|
+
FOR each "MUST return X when Y": fixture { input: Y, expected: X }
|
|
128
|
+
FOR each "MUST validate Z": fixture { input: invalid_Z, expected: validation_error }
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
**Tier 3: Business-scenario-derived data sets (L3 only)**
|
|
132
|
+
```
|
|
133
|
+
FOR each Epic user story:
|
|
134
|
+
scenario_pack: coordinated data set across story steps
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
**Mock contracts:**
|
|
138
|
+
```
|
|
139
|
+
FOR each API endpoint in architecture/_index.md:
|
|
140
|
+
mock_contract: { request_pattern, response_fixture }
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
5. **Degraded mode** (SPEC_MODE = "degraded"):
|
|
144
|
+
- Extract from `index.json.success_criteria` (each → one L2 scenario)
|
|
145
|
+
- Extract from `plan.json` task convergence criteria (each → one L1/L2 scenario)
|
|
146
|
+
- Extract from `.summaries/TASK-*-summary.md` (each → one L1 scenario)
|
|
147
|
+
- All default to priority: "high". No L3 in degraded mode.
|
|
148
|
+
|
|
149
|
+
6. Convert to unified scenario format.
|
|
150
|
+
|
|
151
|
+
#### Route B: `gap` source (coverage-forward)
|
|
152
|
+
|
|
153
|
+
1. Read gap sources:
|
|
154
|
+
- `verification.json` → `gaps[]` with status MISSING or PARTIAL
|
|
155
|
+
- `coverage-report.json` → `requirements_uncovered[]`
|
|
156
|
+
- Task summaries → modified files list
|
|
157
|
+
|
|
158
|
+
2. Priority: MISSING or uncovered → HIGH; PARTIAL → MEDIUM.
|
|
159
|
+
|
|
160
|
+
3. Classify each changed file:
|
|
161
|
+
|
|
162
|
+
| File Type | Category | Layer |
|
|
163
|
+
|-----------|----------|-------|
|
|
164
|
+
| Pure function / utility | unit | L1 |
|
|
165
|
+
| React component | unit + e2e | L1 + L3 |
|
|
166
|
+
| API route / handler | integration | L2 |
|
|
167
|
+
| Database model / query | integration | L2 |
|
|
168
|
+
| CLI command | e2e | L3 |
|
|
169
|
+
| Config / types / constants / CSS / test files | skip | — |
|
|
170
|
+
|
|
171
|
+
4. **Optional: CLI supplementary analysis** (skip if no CLI tools enabled or all files "skip"):
|
|
172
|
+
```
|
|
173
|
+
Bash({
|
|
174
|
+
command: 'maestro delegate "PURPOSE: Analyze source files for test-worthy edge cases
|
|
175
|
+
TASK: Identify error handling | boundary conditions | state transitions | external dependencies
|
|
176
|
+
MODE: analysis
|
|
177
|
+
CONTEXT: @${target_files}
|
|
178
|
+
EXPECTED: JSON array of { file, edge_cases: [{ description, type, priority }] }
|
|
179
|
+
CONSTRAINTS: Non-obvious cases only | Max 5 per file
|
|
180
|
+
" --role analyze --mode analysis',
|
|
181
|
+
run_in_background: true
|
|
182
|
+
})
|
|
183
|
+
```
|
|
184
|
+
On callback: merge edge_cases into scenarios, mark `source: "cli-analysis"`.
|
|
185
|
+
|
|
186
|
+
5. Convert to unified scenario format.
|
|
187
|
+
|
|
188
|
+
#### Route C: `code` source (exploration-forward)
|
|
189
|
+
|
|
190
|
+
1. Explore codebase for testable integration points:
|
|
191
|
+
- Module boundaries and cross-module calls
|
|
192
|
+
- API endpoints and their handlers
|
|
193
|
+
- Database interactions and queries
|
|
194
|
+
- External service integrations
|
|
195
|
+
- Event flows and message passing
|
|
196
|
+
|
|
197
|
+
2. Scan for cross-module imports, API route definitions, database calls.
|
|
198
|
+
|
|
199
|
+
3. Map integration points: which modules communicate through what interfaces.
|
|
200
|
+
|
|
201
|
+
4. Infer layer from integration type:
|
|
202
|
+
- Isolated function → L1
|
|
203
|
+
- Cross-module call / API handler → L2
|
|
204
|
+
- Full user flow → L3
|
|
205
|
+
|
|
206
|
+
5. Convert to unified scenario format.
|
|
207
|
+
|
|
208
|
+
#### Unified Scenario Format
|
|
209
|
+
|
|
210
|
+
All routes produce this identical structure:
|
|
211
|
+
|
|
212
|
+
```json
|
|
213
|
+
{
|
|
214
|
+
"id": "AT-{NNN}",
|
|
215
|
+
"source": "spec|gap|code|re-run",
|
|
216
|
+
"layer": "L0|L1|L2|L3",
|
|
217
|
+
"priority": "critical|high|medium",
|
|
218
|
+
"category": "api_contract|business_rule|state_transition|user_flow|non_functional|unit|integration|e2e|static",
|
|
219
|
+
"name": "descriptive scenario name",
|
|
220
|
+
"target_file": "src/path/to/file.ts",
|
|
221
|
+
"test_file": "src/path/__tests__/file.test.ts",
|
|
222
|
+
"req_ref": "REQ-NNN:AC-N | gap-id | null",
|
|
223
|
+
"description": "what this scenario validates",
|
|
224
|
+
"test_cases": ["case 1", "case 2"],
|
|
225
|
+
"input": { "$fixture_ref": "..." },
|
|
226
|
+
"expected": {
|
|
227
|
+
"status": 200,
|
|
228
|
+
"behavior": "description of expected outcome"
|
|
229
|
+
},
|
|
230
|
+
"preconditions": [],
|
|
231
|
+
"postconditions": [],
|
|
232
|
+
"mock_services": [],
|
|
233
|
+
"fixtures": {}
|
|
234
|
+
}
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
**Field population by route:**
|
|
238
|
+
|
|
239
|
+
| Field | spec | gap | code |
|
|
240
|
+
|-------|------|-----|------|
|
|
241
|
+
| `req_ref` | REQ-NNN:AC-N | gap ID from verification.json | null |
|
|
242
|
+
| `fixtures` | Tier 1-3 generated | empty (inferred in Step 5) | empty (inferred in Step 5) |
|
|
243
|
+
| `mock_services` | from architecture/_index.md | empty | discovered from imports |
|
|
244
|
+
| `preconditions` | from REQ AC text | empty | empty |
|
|
245
|
+
|
|
246
|
+
---
|
|
247
|
+
|
|
248
|
+
### Step 3: Discover Test Infrastructure
|
|
249
|
+
|
|
250
|
+
Detect existing test framework and patterns:
|
|
251
|
+
|
|
252
|
+
1. **Config files**: `jest.config.*`, `vitest.config.*`, `pytest.ini`, `pyproject.toml`, `.mocharc.*`
|
|
253
|
+
2. **Existing tests**: `*.test.*`, `*.spec.*`, `test_*` (exclude node_modules, .git)
|
|
254
|
+
3. **Utilities**: `test-utils.*`, `testHelper*`, `conftest.py`, `setup.*`
|
|
255
|
+
|
|
256
|
+
Extract: framework, directory structure, naming convention, test utilities, run command.
|
|
257
|
+
|
|
258
|
+
Read 2-3 existing test files to learn: import style, describe/it nesting, assertion library, mock patterns, setup/teardown.
|
|
259
|
+
|
|
260
|
+
**Detect tech stack for generation:**
|
|
261
|
+
|
|
262
|
+
| Stack | L1 | L2 | L3 |
|
|
263
|
+
|-------|----|----|-----|
|
|
264
|
+
| Java/Spring Boot | MockMvc | JUnit 5 + WireMock | TestContainers |
|
|
265
|
+
| TypeScript/Node | vitest | supertest + nock | playwright/cypress |
|
|
266
|
+
| Python | pytest | httpx + responses | selenium |
|
|
267
|
+
|
|
268
|
+
If no test framework detected: Error E003.
|
|
269
|
+
|
|
270
|
+
Output: `infrastructure` object passed to Steps 5-6.
|
|
271
|
+
|
|
272
|
+
---
|
|
273
|
+
|
|
274
|
+
### Step 4: Generate Test Plan & Confirm
|
|
275
|
+
|
|
276
|
+
1. **Merge pre-existing tests** from `.tests/test-gen-report.json` (if exists):
|
|
277
|
+
- Mark as "pre-existing" so Step 5 skips writing them
|
|
278
|
+
- Step 6 includes them in execution
|
|
279
|
+
|
|
280
|
+
2. Apply `--layer` filter if specified.
|
|
281
|
+
|
|
282
|
+
3. **Archive previous plan** to `.history/` if exists.
|
|
283
|
+
|
|
284
|
+
4. Write `.tests/auto-test/test-plan.json`:
|
|
285
|
+
```json
|
|
286
|
+
{
|
|
287
|
+
"phase": "{phase}",
|
|
288
|
+
"source_route": "{ROUTE}",
|
|
289
|
+
"spec_ref": "{SPEC_DIR name or null}",
|
|
290
|
+
"spec_mode": "full|degraded|null",
|
|
291
|
+
"generated_at": "{ISO timestamp}",
|
|
292
|
+
"infrastructure": { "framework": "...", "run_command": "..." },
|
|
293
|
+
"layers": {
|
|
294
|
+
"L0": { "scenario_count": N, "commands": ["tsc --noEmit", "eslint src/"] },
|
|
295
|
+
"L1": { "scenario_count": N, "priority_distribution": { "critical": X, "high": Y, "medium": Z } },
|
|
296
|
+
"L2": { "scenario_count": N, "priority_distribution": { ... } },
|
|
297
|
+
"L3": { "scenario_count": N, "priority_distribution": { ... } }
|
|
298
|
+
},
|
|
299
|
+
"scenarios": [ "...unified format..." ],
|
|
300
|
+
"fixtures": { "...if spec source..." },
|
|
301
|
+
"requirement_coverage_plan": { "requirements_targeted": [], "requirements_skipped": [] }
|
|
302
|
+
}
|
|
303
|
+
```
|
|
304
|
+
|
|
305
|
+
5. Display and confirm:
|
|
306
|
+
```
|
|
307
|
+
=== AUTO-TEST PLAN ===
|
|
308
|
+
来源: {ROUTE}
|
|
309
|
+
阶段: {phase_name}
|
|
310
|
+
Spec: {spec_ref or "N/A"}
|
|
311
|
+
|
|
312
|
+
层级分布:
|
|
313
|
+
L0 Static: {N} checks
|
|
314
|
+
L1 Unit/API: {N} scenarios ({X} critical, {Y} high)
|
|
315
|
+
L2 Integration: {N} scenarios ({X} critical, {Y} high)
|
|
316
|
+
L3 E2E: {N} scenarios ({X} critical, {Y} high)
|
|
317
|
+
|
|
318
|
+
Total: {N} scenarios, {M} test cases
|
|
319
|
+
Max iterations: {max_iter}
|
|
320
|
+
|
|
321
|
+
Proceed? (yes/edit/cancel)
|
|
322
|
+
```
|
|
323
|
+
|
|
324
|
+
- `--dry-run`: stop here, report plan
|
|
325
|
+
- User "edit": modify plan interactively
|
|
326
|
+
- User "cancel": abort
|
|
327
|
+
|
|
328
|
+
---
|
|
329
|
+
|
|
330
|
+
### Step 5: Write Tests (RED-GREEN) via CSV Parallel
|
|
331
|
+
|
|
332
|
+
**Parallel strategy**: Build `layer-L{N}-write.csv` per layer, execute via `spawn_agents_on_csv`. Each agent writes one test file independently.
|
|
333
|
+
|
|
334
|
+
#### 5a. Build Write CSV
|
|
335
|
+
|
|
336
|
+
For each layer (L1, L2, L3 — sequential):
|
|
337
|
+
|
|
338
|
+
```
|
|
339
|
+
Extract pending scenarios for this layer from test-plan.json
|
|
340
|
+
Build layer-L{N}-write.csv:
|
|
341
|
+
Columns: id, name, target_file, test_file, description, test_cases, fixtures, req_ref, infrastructure_hints, prev_context
|
|
342
|
+
|
|
343
|
+
prev_context = findings from completed prior-layer scenarios (cross-layer propagation)
|
|
344
|
+
```
|
|
345
|
+
|
|
346
|
+
#### 5b. Parallel Test Writing via spawn_agents_on_csv
|
|
347
|
+
|
|
348
|
+
```javascript
|
|
349
|
+
spawn_agents_on_csv({
|
|
350
|
+
csv_path: `.tests/auto-test/.csv-session/layer-L${N}-write.csv`,
|
|
351
|
+
id_column: "id",
|
|
352
|
+
instruction: `
|
|
353
|
+
Write ONE test file for the given scenario using RED-GREEN methodology.
|
|
354
|
+
|
|
355
|
+
Rules:
|
|
356
|
+
- Read target_file to understand module under test
|
|
357
|
+
- Write test at test_file path following infrastructure_hints patterns
|
|
358
|
+
- Each test_case → one it() block, include id in describe("AT-NNN: {name}")
|
|
359
|
+
- Use fixtures column (infer from source if empty)
|
|
360
|
+
- Run test once after writing: report red_result
|
|
361
|
+
- NEVER modify source code — only write/fix test files
|
|
362
|
+
|
|
363
|
+
RED results:
|
|
364
|
+
- pass: test passes immediately (may need strengthening)
|
|
365
|
+
- expected_fail: test correctly targets real behavior
|
|
366
|
+
- unexpected_fail: setup/import error — fix test, re-run
|
|
367
|
+
|
|
368
|
+
Read discoveries.ndjson for shared patterns. Append if you find reusable ones.
|
|
369
|
+
`,
|
|
370
|
+
max_concurrency: 5,
|
|
371
|
+
max_runtime_seconds: 1800,
|
|
372
|
+
output_csv_path: `.tests/auto-test/.csv-session/layer-L${N}-write-results.csv`,
|
|
373
|
+
output_schema: { id, status: [written|failed], red_result: [expected_fail|unexpected_fail|pass], findings, error }
|
|
374
|
+
})
|
|
375
|
+
```
|
|
376
|
+
|
|
377
|
+
#### 5c. Merge & Continue
|
|
378
|
+
|
|
379
|
+
Merge write-results into master state. Delete temp CSV. Proceed to next layer or Step 6.
|
|
380
|
+
|
|
381
|
+
**If `--max-iter 1`:** After all layers written and run once, jump directly to Step 8 (single-pass mode, replaces test-gen behavior). Skip Steps 6-7.
|
|
382
|
+
|
|
383
|
+
---
|
|
384
|
+
|
|
385
|
+
### Step 6: Execute (Progressive Layers)
|
|
386
|
+
|
|
387
|
+
Run tests progressively through layers with fail-fast on critical:
|
|
388
|
+
|
|
389
|
+
- L0: `tsc --noEmit` + `eslint src/` (static analysis)
|
|
390
|
+
- L1: unit tests (`--testPathPattern="unit|__tests__"`)
|
|
391
|
+
- L2: integration tests (`--testPathPattern="integration"`)
|
|
392
|
+
- L3: E2E tests (`--testPathPattern="e2e"`)
|
|
393
|
+
|
|
394
|
+
**Fail-fast rule:**
|
|
395
|
+
- L0 must pass before L1
|
|
396
|
+
- If ANY "critical" priority failure in L1 → do NOT proceed to L2
|
|
397
|
+
- If ANY "critical" priority failure in L2 → do NOT proceed to L3
|
|
398
|
+
- "high" and "medium" failures do NOT block next layer
|
|
399
|
+
|
|
400
|
+
Record per-scenario results: `{ status, actual_response, duration_ms, error_detail, classification: null }`.
|
|
401
|
+
|
|
402
|
+
Write iteration results to `.tests/auto-test/results-iter-{N}.json`.
|
|
403
|
+
|
|
404
|
+
---
|
|
405
|
+
|
|
406
|
+
### Step 7: Reflect & Adjust (Unified Iteration Engine)
|
|
407
|
+
|
|
408
|
+
Single engine that subsumes both Generator-Critic (per-layer inner) and Reflect-Adjust (global outer). Uses `spawn_agents_on_csv` for parallel failure diagnosis.
|
|
409
|
+
|
|
410
|
+
```
|
|
411
|
+
OUTER LOOP (max_iter iterations):
|
|
412
|
+
|
|
413
|
+
FOR each active layer (L0 through current):
|
|
414
|
+
|
|
415
|
+
INNER LOOP (max 3 iterations per layer — Generator-Critic):
|
|
416
|
+
|
|
417
|
+
1. Build diagnosis CSV from failed scenarios:
|
|
418
|
+
|
|
419
|
+
Build diagnosis-iter-{N}.csv:
|
|
420
|
+
Columns: id, scenario_id, layer, test_file, error_detail, expected, actual, target_file, source_context
|
|
421
|
+
|
|
422
|
+
spawn_agents_on_csv({
|
|
423
|
+
csv_path: `.tests/auto-test/.csv-session/diagnosis-iter-${iter}.csv`,
|
|
424
|
+
id_column: "id",
|
|
425
|
+
instruction: `
|
|
426
|
+
Classify ONE test failure and provide fix if applicable.
|
|
427
|
+
|
|
428
|
+
Classifications:
|
|
429
|
+
- test_defect: Test wrong (bad import, endpoint, fixture, assertion)
|
|
430
|
+
- code_defect: Source violates business rule (actual != expected)
|
|
431
|
+
- env_issue: Environment problem (service down, config missing)
|
|
432
|
+
|
|
433
|
+
If test_defect: provide fix_code (old → new)
|
|
434
|
+
If code_defect/env_issue: leave fix_code empty, provide evidence
|
|
435
|
+
|
|
436
|
+
Rules:
|
|
437
|
+
- NEVER suggest source code changes
|
|
438
|
+
- A test correctly catching a real bug = code_defect, not test_defect
|
|
439
|
+
- When uncertain: prefer code_defect (conservative)
|
|
440
|
+
`,
|
|
441
|
+
max_concurrency: 5,
|
|
442
|
+
max_runtime_seconds: 1200,
|
|
443
|
+
output_csv_path: `.tests/auto-test/.csv-session/diagnosis-iter-${iter}-results.csv`,
|
|
444
|
+
output_schema: { id, classification, fix_code, evidence, error }
|
|
445
|
+
})
|
|
446
|
+
|
|
447
|
+
2. Merge diagnosis results:
|
|
448
|
+
- test_defect with fix_code → apply fix to test file
|
|
449
|
+
- code_defect → mark as confirmed failure (stop retrying)
|
|
450
|
+
- env_issue → mark as blocked
|
|
451
|
+
|
|
452
|
+
3. IF test_defects found:
|
|
453
|
+
Re-run ALL scenarios in this layer (catch regressions)
|
|
454
|
+
|
|
455
|
+
4. IF no test_defects remain: break inner loop
|
|
456
|
+
|
|
457
|
+
END INNER
|
|
458
|
+
|
|
459
|
+
Record final layer results
|
|
460
|
+
IF critical code_defects in this layer: stop layer progression (fail-fast)
|
|
461
|
+
|
|
462
|
+
END FOR
|
|
463
|
+
|
|
464
|
+
REFLECT:
|
|
465
|
+
Analyze: which tests failed, pass rate improving/plateauing/regressing,
|
|
466
|
+
failures clustered by component, strategy effectiveness.
|
|
467
|
+
Append to reflection-log.md:
|
|
468
|
+
iteration, strategy, pass_rate delta, what worked/failed,
|
|
469
|
+
detected patterns, strategy assessment (effective/ineffective + recommendation)
|
|
470
|
+
|
|
471
|
+
ADJUST (Adaptive Strategy Engine):
|
|
472
|
+
|
|
473
|
+
| Condition | Strategy | Behavior |
|
|
474
|
+
|-----------|----------|----------|
|
|
475
|
+
| Iteration 1-2 | Conservative | Fix obvious test_defects only, don't refactor |
|
|
476
|
+
| Pass rate >80% AND failures similar | Aggressive | Batch-fix related failures together |
|
|
477
|
+
| New regressions appeared | Surgical | Revert last changes, fix regression only |
|
|
478
|
+
| Stuck 3+ iterations (rate not improving) | Reflective | Step back, re-analyze root cause pattern |
|
|
479
|
+
|
|
480
|
+
Transitions:
|
|
481
|
+
Conservative --(>80%)--> Aggressive
|
|
482
|
+
Aggressive --(regression)--> Surgical --(fixed)--> Aggressive
|
|
483
|
+
Any --(stuck 3+ iters)--> Reflective --(insight)--> Conservative
|
|
484
|
+
|
|
485
|
+
CONVERGENCE CHECK:
|
|
486
|
+
pass_rate >= threshold (95%) → Step 8 (converged)
|
|
487
|
+
iteration >= max_iter → Step 8 (max_iter_reached)
|
|
488
|
+
all remaining failures = code_defect → Step 8 (confirmed_defects)
|
|
489
|
+
ELSE → next outer iteration (back to Execute)
|
|
490
|
+
|
|
491
|
+
END OUTER
|
|
492
|
+
```
|
|
493
|
+
|
|
494
|
+
**Degenerate cases:**
|
|
495
|
+
- `max_iter=1`: Step 5 writes tests, Step 6 executes once, Step 7 runs Reflect only (log results), no Adjust, no loop → Step 8. Equivalent to test-gen single pass.
|
|
496
|
+
- `max_iter=3`: Inner loop cleans test_defects. Up to 3 outer passes. Similar to business-test.
|
|
497
|
+
- `max_iter=5`: Full adaptive strategy progression. Similar to integration-test.
|
|
498
|
+
|
|
499
|
+
---
|
|
500
|
+
|
|
501
|
+
### Step 8: Complete & Write Artifacts
|
|
502
|
+
|
|
503
|
+
1. Update session state:
|
|
504
|
+
```json
|
|
505
|
+
// .tests/auto-test/state.json
|
|
506
|
+
{
|
|
507
|
+
"session_id": "auto-test-{YYYYMMDD-HHmmss}",
|
|
508
|
+
"phase": "{phase}",
|
|
509
|
+
"phase_dir": "{PHASE_DIR}",
|
|
510
|
+
"source_route": "spec|gap|code|re-run",
|
|
511
|
+
"spec_ref": "SPEC-001 | null",
|
|
512
|
+
"spec_mode": "full|degraded|null",
|
|
513
|
+
"status": "converged|max_iter_reached|confirmed_defects|single_pass",
|
|
514
|
+
"flags": { "max_iter": 5, "layer": null, "dry_run": false, "re_run": false },
|
|
515
|
+
"iteration": 2,
|
|
516
|
+
"strategy": "conservative",
|
|
517
|
+
"strategy_history": ["conservative", "aggressive"],
|
|
518
|
+
"threshold": 95,
|
|
519
|
+
"current_layer": "L2",
|
|
520
|
+
"layer_state": {
|
|
521
|
+
"L0": { "inner_iter": 1, "pass_rate": 100.0, "status": "passed" },
|
|
522
|
+
"L1": { "inner_iter": 2, "pass_rate": 95.0, "status": "passed" },
|
|
523
|
+
"L2": { "inner_iter": 1, "pass_rate": 87.5, "status": "completed" },
|
|
524
|
+
"L3": { "inner_iter": 0, "pass_rate": 0, "status": "pending" }
|
|
525
|
+
},
|
|
526
|
+
"pass_rate_history": [72.0, 85.0, 95.0],
|
|
527
|
+
"scenario_count": 30,
|
|
528
|
+
"infrastructure": { "framework": "vitest", "run_command": "npm test" },
|
|
529
|
+
"started_at": "{ISO}",
|
|
530
|
+
"updated_at": "{ISO}"
|
|
531
|
+
}
|
|
532
|
+
```
|
|
533
|
+
|
|
534
|
+
2. Archive previous report to `.history/` if exists.
|
|
535
|
+
|
|
536
|
+
3. Write `.tests/auto-test/report.json`:
|
|
537
|
+
```json
|
|
538
|
+
{
|
|
539
|
+
"phase": "{phase}",
|
|
540
|
+
"source_route": "{ROUTE}",
|
|
541
|
+
"spec_ref": "{spec ref or null}",
|
|
542
|
+
"spec_mode": "full|degraded|null",
|
|
543
|
+
"completed_at": "{ISO timestamp}",
|
|
544
|
+
"convergence": {
|
|
545
|
+
"status": "converged|max_iter_reached|confirmed_defects|single_pass",
|
|
546
|
+
"iterations": N,
|
|
547
|
+
"final_pass_rate": 95.0,
|
|
548
|
+
"threshold": 95,
|
|
549
|
+
"strategy_history": ["conservative", "aggressive"]
|
|
550
|
+
},
|
|
551
|
+
"infrastructure": { "framework": "vitest", "test_dir": "__tests__/", "run_command": "npm test" },
|
|
552
|
+
"layers": {
|
|
553
|
+
"L0": { "total": N, "passed": P, "failed": F, "blocked": B, "pass_rate": 100.0 },
|
|
554
|
+
"L1": { "total": N, "passed": P, "failed": F, "blocked": B, "pass_rate": 95.0 },
|
|
555
|
+
"L2": { ... },
|
|
556
|
+
"L3": { ... }
|
|
557
|
+
},
|
|
558
|
+
"scenarios": [
|
|
559
|
+
{
|
|
560
|
+
"id": "AT-001", "source": "spec", "layer": "L1", "name": "...",
|
|
561
|
+
"test_file": "...", "req_ref": "REQ-001:AC-1",
|
|
562
|
+
"status": "passed|failed|blocked",
|
|
563
|
+
"classification": "null|test_defect|code_defect|env_issue",
|
|
564
|
+
"iterations_to_pass": 2
|
|
565
|
+
}
|
|
566
|
+
],
|
|
567
|
+
"failures": [
|
|
568
|
+
{
|
|
569
|
+
"id": "AF-001", "scenario_id": "AT-005", "req_ref": "REQ-001:AC-3",
|
|
570
|
+
"layer": "L1", "severity": "critical", "classification": "code_defect",
|
|
571
|
+
"description": "...", "expected": "...", "actual": "...",
|
|
572
|
+
"fix_suggestion": { "file": "src/...", "line": 42, "direction": "..." }
|
|
573
|
+
}
|
|
574
|
+
],
|
|
575
|
+
"requirement_coverage": [],
|
|
576
|
+
"summary": {
|
|
577
|
+
"source_route": "{ROUTE}",
|
|
578
|
+
"total_scenarios": 30, "total_passed": 26, "total_failed": 3, "total_blocked": 1,
|
|
579
|
+
"bugs_discovered": 3, "test_defects_fixed": 5, "coverage_pct": 85.0
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
```
|
|
583
|
+
|
|
584
|
+
4. Update `validation.json`: if gap source, change MISSING → COVERED for passing tests.
|
|
585
|
+
5. Update `index.json` with `auto_test` section.
|
|
586
|
+
|
|
587
|
+
**Artifact file mapping (old → new):**
|
|
588
|
+
|
|
589
|
+
| Old file | New file |
|
|
590
|
+
|----------|----------|
|
|
591
|
+
| `.tests/test-gen-report.json` | `.tests/auto-test/report.json` (source_route: "gap") |
|
|
592
|
+
| `.tests/integration/state.json` | `.tests/auto-test/state.json` |
|
|
593
|
+
| `.tests/integration/summary.json` | Merged into report.json |
|
|
594
|
+
| `.tests/integration/reflection-log.md` | `.tests/auto-test/reflection-log.md` |
|
|
595
|
+
| `.tests/business/business-test-report.json` | `.tests/auto-test/report.json` (source_route: "spec") |
|
|
596
|
+
| `.tests/business/business-test-summary.md` | `.tests/auto-test/traceability.md` (conditional) |
|
|
597
|
+
|
|
598
|
+
---
|
|
599
|
+
|
|
600
|
+
### Step 9: Post-Processing & Routing
|
|
601
|
+
|
|
602
|
+
#### Conditional: Traceability Matrix (spec source only)
|
|
603
|
+
|
|
604
|
+
Build REQ → AC → scenario → result mapping:
|
|
605
|
+
|
|
606
|
+
```
|
|
607
|
+
FOR each REQ in requirements targeted:
|
|
608
|
+
FOR each AC in REQ:
|
|
609
|
+
scenarios_for_ac = filter scenarios where req_ref == "REQ-NNN:AC-{N}"
|
|
610
|
+
ac_status = "passed" if ALL passed
|
|
611
|
+
"failed" if ANY failed
|
|
612
|
+
"blocked" if ANY blocked and none failed
|
|
613
|
+
"untested" if no scenarios mapped
|
|
614
|
+
|
|
615
|
+
req.coverage_pct = passed_criteria / total_criteria * 100
|
|
616
|
+
req.verdict = "verified" if all MUST+SHOULD passed
|
|
617
|
+
"partial" if some failed
|
|
618
|
+
"unverified" if all failed or untested
|
|
619
|
+
```
|
|
620
|
+
|
|
621
|
+
Populate `requirement_coverage[]` in report.json.
|
|
622
|
+
Write `.tests/auto-test/traceability.md` (human-readable table).
|
|
623
|
+
|
|
624
|
+
#### Conditional: Issue Creation (when failures exist)
|
|
625
|
+
|
|
626
|
+
```
|
|
627
|
+
FOR each failure in report.failures WHERE classification == "code_defect":
|
|
628
|
+
issue = {
|
|
629
|
+
id: "ISS-{YYYYMMDD}-{counter:03d}",
|
|
630
|
+
title: "Auto-Test: " + failure.req_ref + " - " + failure.description,
|
|
631
|
+
status: "registered",
|
|
632
|
+
priority: severity_to_priority(failure.severity),
|
|
633
|
+
source: "auto-test",
|
|
634
|
+
phase_ref: PHASE_NUM,
|
|
635
|
+
description: "Expected: " + failure.expected + ". Actual: " + failure.actual,
|
|
636
|
+
fix_direction: failure.fix_suggestion.direction,
|
|
637
|
+
context: { location: failure.fix_suggestion.file + ":" + failure.fix_suggestion.line },
|
|
638
|
+
tags: ["auto-test", failure.layer]
|
|
639
|
+
}
|
|
640
|
+
Append to .workflow/issues/issues.jsonl
|
|
641
|
+
```
|
|
642
|
+
|
|
643
|
+
#### Report Display
|
|
644
|
+
|
|
645
|
+
```
|
|
646
|
+
=== AUTO-TEST RESULTS ===
|
|
647
|
+
阶段: {phase_name}
|
|
648
|
+
来源: {ROUTE}
|
|
649
|
+
迭代: {N} (策略: {strategy_history})
|
|
650
|
+
收敛: {status} ({final_pass_rate}%)
|
|
651
|
+
|
|
652
|
+
层级结果:
|
|
653
|
+
L0 Static: {pass_rate}% ({passed}/{total})
|
|
654
|
+
L1 Unit/API: {pass_rate}% ({passed}/{total})
|
|
655
|
+
L2 Integration: {pass_rate}% ({passed}/{total})
|
|
656
|
+
L3 E2E: {pass_rate}% ({passed}/{total})
|
|
657
|
+
|
|
658
|
+
场景: {passed} passed, {failed} failed, {blocked} blocked
|
|
659
|
+
Bugs: {N} discovered
|
|
660
|
+
{IF spec: "需求覆盖: {pct}% | 已验证: {n}/{total}"}
|
|
661
|
+
|
|
662
|
+
Files:
|
|
663
|
+
.tests/auto-test/state.json
|
|
664
|
+
.tests/auto-test/test-plan.json
|
|
665
|
+
.tests/auto-test/report.json
|
|
666
|
+
.tests/auto-test/reflection-log.md
|
|
667
|
+
{IF spec: ".tests/auto-test/traceability.md"}
|
|
668
|
+
```
|
|
669
|
+
|
|
670
|
+
#### Next-step routing
|
|
671
|
+
|
|
672
|
+
| Result | Suggestion |
|
|
673
|
+
|--------|------------|
|
|
674
|
+
| Converged (>=threshold) | `/maestro-verify {phase}` to update validation |
|
|
675
|
+
| All requirements verified (spec) | `/maestro-milestone-audit` |
|
|
676
|
+
| Bugs discovered (code_defects) | `/quality-debug --from-auto-test {phase}` |
|
|
677
|
+
| Max iter, >80% | `/quality-test {phase}` for manual UAT on remaining gaps |
|
|
678
|
+
| Max iter, <80% | `/quality-debug {phase}` for deep investigation |
|
|
679
|
+
| Coverage still low | `/quality-auto-test {phase} --layer {missing}` |
|
|
680
|
+
| Re-run all pass | `/maestro-verify {phase}` |
|
|
681
|
+
| Single pass (max_iter=1), bugs found | `/quality-debug --from-auto-test {phase}` |
|
|
682
|
+
| Single pass, all pass | `/quality-test {phase}` |
|