gaia-framework 1.105.1 → 1.127.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/commands/gaia-bridge-disable.md +18 -0
- package/.claude/commands/gaia-bridge-enable.md +18 -0
- package/.claude/commands/gaia-fill-test-gaps.md +17 -0
- package/CLAUDE.md +64 -1
- package/_gaia/_config/gaia-help.csv +2 -0
- package/_gaia/_config/global.yaml +14 -1
- package/_gaia/_config/lifecycle-sequence.yaml +23 -3
- package/_gaia/_config/skill-manifest.csv +1 -0
- package/_gaia/_config/workflow-manifest.csv +2 -0
- package/_gaia/core/agents/orchestrator.md +1 -1
- package/_gaia/core/protocols/review-gate-check.xml +45 -5
- package/_gaia/core/validators/test-environment-validator.js +191 -0
- package/_gaia/core/workflows/bridge-toggle/checklist.md +11 -0
- package/_gaia/core/workflows/bridge-toggle/instructions.xml +69 -0
- package/_gaia/core/workflows/bridge-toggle/workflow.yaml +27 -0
- package/_gaia/dev/skills/_skill-index.yaml +13 -0
- package/_gaia/dev/skills/code-review-standards.md +50 -0
- package/_gaia/dev/skills/edge-cases.md +201 -0
- package/_gaia/lifecycle/knowledge/brownfield/ci-test-detection.md +194 -0
- package/_gaia/lifecycle/knowledge/brownfield/test-execution-scan.md +13 -0
- package/_gaia/lifecycle/skills/document-rulesets.md +93 -3
- package/_gaia/lifecycle/templates/story-template.md +7 -7
- package/_gaia/lifecycle/templates/test-gap-analysis-template.md +221 -0
- package/_gaia/lifecycle/workflows/4-implementation/check-review-gate/checklist.md +1 -1
- package/_gaia/lifecycle/workflows/4-implementation/check-review-gate/instructions.xml +11 -11
- package/_gaia/lifecycle/workflows/4-implementation/code-review/instructions.xml +1 -1
- package/_gaia/lifecycle/workflows/4-implementation/create-story/instructions.xml +73 -2
- package/_gaia/lifecycle/workflows/4-implementation/dev-story/instructions.xml +25 -2
- package/_gaia/lifecycle/workflows/4-implementation/retrospective/instructions.xml +1 -1
- package/_gaia/lifecycle/workflows/4-implementation/run-all-reviews/instructions.xml +132 -9
- package/_gaia/lifecycle/workflows/4-implementation/security-review/checklist.md +2 -0
- package/_gaia/lifecycle/workflows/4-implementation/sprint-planning/instructions.xml +13 -0
- package/_gaia/lifecycle/workflows/4-implementation/sprint-planning/workflow.yaml +8 -0
- package/_gaia/lifecycle/workflows/4-implementation/validate-story/checklist.md +1 -0
- package/_gaia/lifecycle/workflows/4-implementation/validate-story/instructions.xml +11 -0
- package/_gaia/lifecycle/workflows/5-deployment/deployment-checklist/instructions.xml +11 -0
- package/_gaia/lifecycle/workflows/anytime/brownfield-onboarding/instructions.xml +48 -1
- package/_gaia/lifecycle/workflows/anytime/brownfield-onboarding/workflow.yaml +10 -0
- package/_gaia/testing/agents/test-architect.md +2 -0
- package/_gaia/testing/workflows/fill-test-gaps/checklist.md +16 -0
- package/_gaia/testing/workflows/fill-test-gaps/instructions.xml +128 -0
- package/_gaia/testing/workflows/fill-test-gaps/workflow.yaml +30 -0
- package/_gaia/testing/workflows/test-gap-analysis/instructions.xml +47 -14
- package/_gaia/testing/workflows/test-gap-analysis/workflow.yaml +1 -0
- package/gaia-install.sh +37 -0
- package/package.json +3 -3
|
@@ -5,6 +5,8 @@
|
|
|
5
5
|
<mandate>Only stories with status 'ready-for-dev' are selectable for a sprint</mandate>
|
|
6
6
|
</critical>
|
|
7
7
|
<step n="1" title="Load Epics, Stories, Action Items, and Previous Retro">
|
|
8
|
+
<!-- E19-S28 AC3 / FR-315: parse optional --skip-gap-prompt flag and bind to local variable -->
|
|
9
|
+
<action>FLAG PARSING (E19-S28 AC3): Inspect the invocation arguments passed to /gaia-sprint-plan. If the flag `--skip-gap-prompt` is present, set the local variable `skip_gap_prompt = true`. Otherwise set `skip_gap_prompt = false` (the default declared in workflow.yaml). This variable is consulted by Step 3b to suppress the FR-237 gap analysis tip. The flag is scoped to sprint-planning only and does NOT affect the independent invocation path of /gaia-fill-test-gaps.</action>
|
|
8
10
|
<action>Read {implementation_artifacts}/action-items.yaml if it exists. Check for HIGH priority items open for 2+ sprints (escalation_count >= 2 AND priority == "high" AND status == "open").
|
|
9
11
|
If any found: HALT — "Cannot start sprint planning. {count} HIGH priority action item(s) have been open for {escalation_count}+ sprints:
|
|
10
12
|
{for each: A-{NNN}: {title} (open since {source_sprint}) → Run: /gaia-action-items A-{NNN}}
|
|
@@ -41,6 +43,17 @@
|
|
|
41
43
|
<ask>Team velocity estimate (story points)?</ask>
|
|
42
44
|
<ask>Sprint number (for multi-sprint tracking)?</ask>
|
|
43
45
|
</step>
|
|
46
|
+
<step n="3b" title="Gap Analysis Tip">
|
|
47
|
+
<critical>
|
|
48
|
+
<mandate>This step implements FR-237 (E19-S18). It surfaces a gap-analysis reminder to the scrum master after the velocity check and before story selection so that testing gaps are visible when prioritizing stories. The tip is advisory only — it never blocks sprint planning.</mandate>
|
|
49
|
+
</critical>
|
|
50
|
+
<action>SKIP FLAG CHECK (AC4 / E19-S28 AC3): Consult the local variable `skip_gap_prompt` bound in Step 1 from the `--skip-gap-prompt` flag parsed per workflow.yaml flag declaration. If `skip_gap_prompt == true`, skip this step entirely — emit no prompt, no file check, no wait. Proceed directly to Step 4 (Story Selection). This gives the scrum master an explicit opt-out for teams that do not use gap analysis. The flag does NOT affect the independent invocation path of /gaia-fill-test-gaps — it is scoped solely to suppressing this sprint-planning tip.</action>
|
|
51
|
+
<action>RESOLVE SPRINT START DATE: Derive {sprint_start_date} as the ISO date (YYYY-MM-DD) the current sprint is starting. Prefer the value from {implementation_artifacts}/sprint-status.yaml if it exists (field: started), otherwise fall back to today's date. This is the date used to locate the gap-analysis file for the current sprint.</action>
|
|
52
|
+
<action>GAP ANALYSIS FILE CHECK (AC2): Check whether {test_artifacts}/test-gap-analysis-{sprint_start_date}.md exists (this resolves to docs/test-artifacts/test-gap-analysis-{sprint_start_date}.md under the project root). If the file is present, a gap analysis has already been run for this sprint — skip the tip entirely and proceed to Step 4 (the prompt is only shown when the file is absent). If the file does not exist, fall through to the display action below.</action>
|
|
53
|
+
<action>DISPLAY TIP (AC1): Emit this exact one-line tip to the user: "Tip: Run `/gaia-test-gap-analysis` before selecting stories to identify untested areas." This wording is verbatim from FR-237 / AC1 and must not be paraphrased. The tip is informational — it does NOT gate sprint planning, and the scrum master may continue without running gap analysis.</action>
|
|
54
|
+
<action>YOLO AUTO-CONTINUE (AC3): In YOLO mode, display the tip and auto-continue after 2 seconds without waiting for user input. In normal mode, display the tip and wait for the user to press continue via the ask below.</action>
|
|
55
|
+
<ask if="not yolo_mode">Continue to story selection? [continue]</ask>
|
|
56
|
+
</step>
|
|
44
57
|
<step n="4" title="Story Selection">
|
|
45
58
|
<action>Read sizing_map from {project-root}/_gaia/_config/global.yaml. Use these canonical point values for T-shirt sizes (S/M/L/XL → points). Display the mapping to the user before selection.</action>
|
|
46
59
|
<action>Select stories for this sprint based on priority and dependencies — only from stories classified as SELECTABLE in Step 1 (status = 'ready-for-dev').</action>
|
|
@@ -4,6 +4,14 @@ module: lifecycle
|
|
|
4
4
|
agent: sm
|
|
5
5
|
val_validate_output: true
|
|
6
6
|
template_output_prompt: "auto"
|
|
7
|
+
# E19-S28 AC3 / FR-315: optional flag declarations
|
|
8
|
+
# --skip-gap-prompt suppresses the FR-237 / Step 3b gap analysis tip for the remainder of the run.
|
|
9
|
+
# The flag is scoped to sprint-planning only and does NOT affect independent invocation of /gaia-fill-test-gaps.
|
|
10
|
+
flags:
|
|
11
|
+
skip-gap-prompt:
|
|
12
|
+
type: boolean
|
|
13
|
+
default: false
|
|
14
|
+
description: "Suppress the FR-237 gap analysis tip (Step 3b). Does not affect /gaia-fill-test-gaps."
|
|
7
15
|
config_resolved: "{installed_path}/.resolved/sprint-planning.yaml"
|
|
8
16
|
config_source: "{project-root}/_gaia/lifecycle/config.yaml"
|
|
9
17
|
installed_path: "{project-root}/_gaia/lifecycle/workflows/4-implementation/sprint-planning"
|
|
@@ -9,6 +9,7 @@ validation-target: 'Story validation'
|
|
|
9
9
|
- [ ] Semantic quality — duplicate detection, ADR cross-references
|
|
10
10
|
- [ ] Dependencies — all declared dependencies exist and resolve
|
|
11
11
|
- [ ] Factual verification — claims verified against filesystem and ground truth
|
|
12
|
+
- [ ] Review Gate vocabulary — all rows in {UNVERIFIED, PASSED, FAILED}
|
|
12
13
|
## Fix Loop (Step 3)
|
|
13
14
|
- [ ] SM fixes findings, Val re-validates
|
|
14
15
|
- [ ] Hard limit: 3 validation attempts max
|
|
@@ -56,6 +56,17 @@
|
|
|
56
56
|
- If origin is non-null, origin_ref must be non-empty (not null, not empty string).
|
|
57
57
|
A non-null origin with empty or null origin_ref is a WARNING finding.
|
|
58
58
|
If origin is null or absent, origin_ref is not validated (orphaned refs are acceptable).
|
|
59
|
+
|
|
60
|
+
(h) Review Gate Vocabulary — parse the story's Review Gate table and verify
|
|
61
|
+
each row's Status column value is one of the canonical three values:
|
|
62
|
+
UNVERIFIED, PASSED, FAILED. Match is case-sensitive and exact. Any other
|
|
63
|
+
value (e.g., PENDING, pending, -, blank, APPROVED, REQUEST_CHANGES) is a
|
|
64
|
+
WARNING finding. Report the field path as "Review Gate > {review name} > Status"
|
|
65
|
+
and include the found value in the finding description. The canonical set
|
|
66
|
+
is enforced: UNVERIFIED is the default for reviews that have not yet run,
|
|
67
|
+
PASSED indicates the review passed, FAILED indicates the review failed.
|
|
68
|
+
Code Review's internal verdict keywords (APPROVE/REQUEST_CHANGES) belong in
|
|
69
|
+
the report body and must be mapped to PASSED/FAILED in the gate row.
|
|
59
70
|
</action>
|
|
60
71
|
</step>
|
|
61
72
|
<step n="3" title="Validation Fix Loop">
|
|
@@ -51,6 +51,17 @@
|
|
|
51
51
|
<ask>Stakeholder approval: confirm release scope is approved. Enter CONFIRMED.</ask>
|
|
52
52
|
<check if="stakeholder_approval != CONFIRMED">HALT: Stakeholder approval required before deployment.</check>
|
|
53
53
|
</step>
|
|
54
|
+
<step n="6e" title="Gap Analysis Gate">
|
|
55
|
+
<critical>
|
|
56
|
+
<mandate>This step implements FR-238 (E19-S19, ADR-030 §10.22). It enforces that a test gap analysis has been run for the current sprint before deployment is allowed. The gate is HALT-enforced when the gap analysis file is missing — it is not advisory. A bypass flag (`--skip-gap-gate`) exists for exceptional releases and must log a warning plus record the bypass in the checklist output. Critical gaps surface as WARNINGs only — they do not block deployment.</mandate>
|
|
57
|
+
</critical>
|
|
58
|
+
<action>SKIP FLAG CHECK (AC4): Inspect the invocation arguments passed to /gaia-deploy-checklist. If the flag `--skip-gap-gate` is present: log the warning "gap gate bypassed by --skip-gap-gate" to the console, set a gap_gate_bypassed flag on the workflow state so Step 7 records the bypass in the generated checklist output (e.g., "Gap Analysis Gate: BYPASSED via --skip-gap-gate"), and proceed directly to Step 7 without performing the file check. This is the only path that skips the file check.</action>
|
|
59
|
+
<action>RESOLVE SPRINT WINDOW (AC2): Read {implementation_artifacts}/sprint-status.yaml and extract the current sprint's `sprint_id` and `started` (ISO date YYYY-MM-DD) fields. These define the sprint window — from the sprint start date to today. If sprint-status.yaml does not exist, HALT with "Deploy blocked: sprint-status.yaml missing — cannot determine sprint window. Run /gaia-sprint-plan or pass --skip-gap-gate if intentional."</action>
|
|
60
|
+
<action>GAP ANALYSIS FILE CHECK (AC1, AC2, FR-238): Check {test_artifacts}/ (which resolves to docs/test-artifacts/ under the project root) for any file matching the pattern `test-gap-analysis-{YYYY-MM-DD}.md` where the date is within the sprint window (sprint_start_date ≤ date ≤ today, inclusive). Preferred match: `test-gap-analysis-{sprint_start_date}.md`. Any dated file within the window satisfies the gate. If one or more matching files exist, capture the newest one as {gap_file} and proceed to the CRITICAL GAP CHECK below. If no matching file exists, fall through to the FAIL path.</action>
|
|
61
|
+
<action>FAIL PATH (AC3): If no test-gap-analysis file exists within the sprint window, HALT with this exact message: "Deploy blocked: no gap analysis found for sprint {sprint_id}. Run `/gaia-test-gap-analysis` first." Do NOT advance to Step 7. The release manager must either run /gaia-test-gap-analysis or re-run with --skip-gap-gate to bypass the gate explicitly.</action>
|
|
62
|
+
<action>CRITICAL GAP CHECK (AC5): If {gap_file} exists, read its YAML frontmatter for a `gap_count` field and scan the "Gap Table" section of the markdown for rows with `severity: critical` (or a "critical" column value). Count the critical gaps. If one or more critical gaps are found: emit a WARNING (not a HALT) with this exact message (E19-S28 AC2 / FR-315): "Deploy blocked: {N} critical gaps found in {gap_file_basename}. Run `/gaia-fill-test-gaps` to address them, or manually update gate status." — where {N} is the critical gap count and {gap_file_basename} is the base filename of {gap_file} (e.g., `test-gap-analysis-2026-04-08.md`). Follow the message with a bulleted list of each critical gap's description. Set a critical_gaps_warned flag on the workflow state so Step 7 includes the warning in the generated checklist output. Then proceed to Step 7. This is a non-blocking warning — the deployment continues (the "Deploy blocked" wording surfaces severity, not HALT semantics).</action>
|
|
63
|
+
<action>PASS PATH: If {gap_file} exists and has no critical gaps, emit "Gap Analysis Gate: PASSED — gap analysis {gap_file} found, no critical gaps." and proceed to Step 7.</action>
|
|
64
|
+
</step>
|
|
54
65
|
<step n="7" title="Generate Checklist">
|
|
55
66
|
<template-output file="{implementation_artifacts}/deployment-checklist-{date}.md">
|
|
56
67
|
Generate deployment checklist with: pre-deployment checks, dependency status, rollback readiness, approval gates, go/no-go recommendation.
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
<action>API Detection: scan for route definitions, controllers, OpenAPI/Swagger specs. Set {has_apis} flag (true/false)</action>
|
|
15
15
|
<action>Event Detection: scan for Kafka, RabbitMQ, SNS-SQS, Redis pub-sub, NATS, or other messaging patterns. Set {has_events} flag (true/false)</action>
|
|
16
16
|
<action>Dependency Detection: scan for outbound HTTP clients, SDKs, service URLs, database connections. Set {has_external_deps} flag (true/false)</action>
|
|
17
|
-
<action>Frontend Detection:
|
|
17
|
+
<action>Frontend Detection: use the shared detectProjectType module at scripts/lib/project-type-detection.js — call detectProjectType({project-path}) and check the result. If result.type is 'frontend', 'fullstack', or 'mobile', set {has_frontend} = true. Otherwise set {has_frontend} = false. The evidence array provides the matched signals for documentation.</action>
|
|
18
18
|
<action>Testing infrastructure: identify test framework, coverage config, test count, test patterns</action>
|
|
19
19
|
<action>CI/CD: identify GitHub Actions, Jenkins, Docker, Terraform, or other pipeline files</action>
|
|
20
20
|
<action>Project Type Detection (ADR-022 Template Discriminator Pattern): scan {project-path} for infrastructure file patterns across 6 marker categories to determine {project_type}. Each category is independently detected:
|
|
@@ -105,6 +105,53 @@
|
|
|
105
105
|
<action>If the subagent failed to write its output file: log a warning and continue — test execution scan failures should not block the overall brownfield workflow.</action>
|
|
106
106
|
</step>
|
|
107
107
|
|
|
108
|
+
<step n="2.8" title="Auto-Generate test-environment.yaml from Detected Infrastructure">
|
|
109
|
+
<!--
|
|
110
|
+
E19-S16 / FR-235 — Aggregates the four brownfield test-infrastructure
|
|
111
|
+
detectors (E19-S12 test-runner, E19-S13 ci-test, E19-S14 docker-test,
|
|
112
|
+
E19-S15 browser-matrix) into a single test-environment.yaml file at
|
|
113
|
+
docs/test-artifacts/test-environment.yaml. The generated file is
|
|
114
|
+
compatible with the E17-S7 schema consumed by the Test Execution
|
|
115
|
+
Bridge (ADR-028, FR-196) so that the bridge is immediately usable in
|
|
116
|
+
brownfield projects without manual authoring.
|
|
117
|
+
|
|
118
|
+
This step runs AFTER Step 2.75 so that any runners discovered by the
|
|
119
|
+
test execution scanner are already known. It is non-blocking for the
|
|
120
|
+
overall workflow: generation failures are logged as warnings and do
|
|
121
|
+
NOT halt brownfield onboarding.
|
|
122
|
+
|
|
123
|
+
Risk: HIGH. Merge conflict resolution is the critical path — detected
|
|
124
|
+
values MUST NEVER silently overwrite user customizations. The safe
|
|
125
|
+
default in YOLO mode is [m]erge, which fills only null / missing
|
|
126
|
+
fields and preserves every field the user has already set.
|
|
127
|
+
-->
|
|
128
|
+
<action>Invoke the four brownfield detectors at {project-path} and collect their results into a single detections object:
|
|
129
|
+
- testRunners ← detectTestRunners({project-path}) [E19-S12, src/brownfield/test-runner-detector.js]
|
|
130
|
+
- ciTestExecution ← detectCITestExecution({project-path}) [E19-S13, src/brownfield/ci-test-detector.js]
|
|
131
|
+
- dockerTestConfig ← detectDockerTestConfig({project-path}) [E19-S14, src/brownfield/docker-test-detector.js]
|
|
132
|
+
- browserMatrix ← detectBrowserMatrix({project-path}) [E19-S15, src/brownfield/browser-matrix-detector.js]
|
|
133
|
+
Each detector is best-effort: wrap each call in a try/catch and record null on failure so one detector crashing cannot block the others.</action>
|
|
134
|
+
|
|
135
|
+
<action>Call hasDetectedInfrastructure(detections) from src/brownfield/test-environment-generator.js. If it returns false, log "No test infrastructure detected — skipping test-environment.yaml generation (AC6 gate is also skipped)" and proceed to the next step. This keeps pristine greenfield-ish projects from being blocked by a gate they cannot satisfy.</action>
|
|
136
|
+
|
|
137
|
+
<action>Otherwise, call generateTestEnvironmentYaml(detections) to build the document. The document contains the six story-required metadata fields — test_runner (from E19-S12), ci_provider (from E19-S13), docker_test_config (from E19-S14), browser_matrix (from E19-S15), generated_by: brownfield, and generated_date — alongside the E17-S7 schema-required `version` and `runners` fields (AC1, AC2).</action>
|
|
138
|
+
|
|
139
|
+
<action>Resolve the target path as {project-root}/docs/test-artifacts/test-environment.yaml. Check whether the file already exists at this path (AC3).</action>
|
|
140
|
+
|
|
141
|
+
<action if="file does not exist">Call writeTestEnvironmentYaml(targetPath, doc, "merge") — the mode is irrelevant for fresh writes and the helper treats a missing file as a creation. Log "Created test-environment.yaml from detected infrastructure."</action>
|
|
142
|
+
|
|
143
|
+
<action if="file exists AND execution_mode == 'yolo'">YOLO mode always uses the safe default: merge. Detected values fill only null or missing fields; every non-null user-supplied field is preserved byte-for-byte. Call writeTestEnvironmentYaml(targetPath, doc, "merge"). Log "Merged detected values into existing test-environment.yaml (YOLO safe default)." This satisfies AC4.</action>
|
|
144
|
+
|
|
145
|
+
<action if="file exists AND execution_mode != 'yolo'">Prompt the user: "test-environment.yaml already exists — [m]erge detected values (safe, default) / [s]kip (leave file unchanged) / [o]verwrite (REPLACE entire file — destructive)". Wait for the user to choose one of the three options (AC3).
|
|
146
|
+
- [m] → writeTestEnvironmentYaml(targetPath, doc, "merge")
|
|
147
|
+
- [s] → writeTestEnvironmentYaml(targetPath, doc, "skip"); log "Skipped — existing file preserved."
|
|
148
|
+
- [o] → writeTestEnvironmentYaml(targetPath, doc, "overwrite"); log "Overwritten — previous content discarded."</action>
|
|
149
|
+
|
|
150
|
+
<action>After writing, validate the resulting file against the E17-S7 schema by calling validateTestEnvironment(readFileSync(targetPath, 'utf8')) from {project-root}/_gaia/core/validators/test-environment-validator.js (AC5). If the validator returns warnings, log each warning as a WARN-level message listing the specific failing field and continue — the file is still written but the user is notified so they can refine it. Never halt the overall brownfield workflow on schema warnings.</action>
|
|
151
|
+
|
|
152
|
+
<action>Record the detection results and the chosen conflict-resolution action in the brownfield onboarding report for traceability: include {generated,merged,skipped,overwritten} action, the count of runners detected, the ci_provider, and whether docker_test_config / browser_matrix were populated.</action>
|
|
153
|
+
</step>
|
|
154
|
+
|
|
108
155
|
<step n="3" title="NFR Assessment & Performance Test Plan">
|
|
109
156
|
<action>Spawn a subagent — "Analyze the codebase at {project-path} for non-functional requirements. Read {installed_path}/../../templates/nfr-assessment-template.md for the output format. Assess: code quality (linting, complexity, duplication), security posture (dependency vulnerabilities, secrets handling, auth quality — load security-basics skill section vulnerability-scanning if needed), performance (bundle size if frontend, query patterns, caching, resource management), accessibility (ARIA, semantic HTML, keyboard nav if frontend), test coverage (framework, count, coverage %, untested areas, quality), CI/CD (build pipeline, deploy strategy, environments, IaC). Create NFR Baseline Summary Table with measured values (not placeholders). Output to {test_artifacts}/nfr-assessment.md. Then create a performance test plan: load {project-root}/_gaia/testing/knowledge/performance/k6-patterns.md for load testing patterns. If {has_frontend}: also load {project-root}/_gaia/testing/knowledge/performance/lighthouse-ci.md. Define performance budgets (P50/P95/P99) based on NFR baselines, load test scenarios (gradual, spike, soak), backend profiling targets (slow queries, N+1, connection pools), CI performance gates. If {has_frontend}: define Core Web Vitals targets (LCP under 2.5s, INP under 200ms, CLS under 0.1). Output to {test_artifacts}/performance-test-plan-{date}.md"</action>
|
|
110
157
|
<action>When subagent returns: verify both {test_artifacts}/nfr-assessment.md and {test_artifacts}/performance-test-plan-{date}.md exist. If the subagent failed to write files, the orchestrator MUST write them to {test_artifacts}/ as declared in workflow.yaml output.artifacts — do NOT fall back to {planning_artifacts}/.</action>
|
|
@@ -13,6 +13,16 @@ quality_gates:
|
|
|
13
13
|
on_fail: "HALT: NFR assessment not found at {test_artifacts}/nfr-assessment.md."
|
|
14
14
|
- check: "performance_test_plan_exists"
|
|
15
15
|
on_fail: "HALT: Performance test plan not found at {test_artifacts}/. Run /gaia-perf-testing."
|
|
16
|
+
# E19-S16 / FR-235 — NFR onboarding gate: when any test infrastructure is
|
|
17
|
+
# detected during Steps 2.5/2.75 (E19-S12–S15 detectors), the brownfield
|
|
18
|
+
# workflow MUST emit test-environment.yaml in test-artifacts/ so that the
|
|
19
|
+
# Test Execution Bridge (E17) is immediately usable. The gate is
|
|
20
|
+
# CONDITIONAL — a project with no detected test infrastructure is not
|
|
21
|
+
# blocked by a missing file (AC6). When infrastructure IS detected but
|
|
22
|
+
# the file is absent, the workflow halts with a clear remediation path.
|
|
23
|
+
- check: "test_environment_yaml_required_when_infra_detected"
|
|
24
|
+
description: "If any of E19-S12/S13/S14/S15 detected test infrastructure, docs/test-artifacts/test-environment.yaml must exist (AC6, FR-235)."
|
|
25
|
+
on_fail: "HALT: Brownfield detected test infrastructure but test-environment.yaml was not generated. Re-run step 2.8 or run /gaia-brownfield again."
|
|
16
26
|
on_error:
|
|
17
27
|
missing_file: "ask_user"
|
|
18
28
|
unresolved_variable: "halt"
|
|
@@ -107,6 +107,7 @@ You must fully embody this agent's persona and follow the activation protocol EX
|
|
|
107
107
|
<item cmd="7" label="Test Review" description="Review test quality and flakiness" />
|
|
108
108
|
<item cmd="8" label="NFR Assessment" description="Non-functional requirements assessment" />
|
|
109
109
|
<item cmd="9" label="Traceability" description="Requirements-to-tests traceability" />
|
|
110
|
+
<item cmd="10" label="Test Gap Analysis" description="Scan test suite against requirements to identify coverage gaps" />
|
|
110
111
|
</menu>
|
|
111
112
|
|
|
112
113
|
<greeting>
|
|
@@ -123,6 +124,7 @@ I see testing through a risk lens — not everything needs the same depth of cov
|
|
|
123
124
|
7. **Test Review** — audit existing test quality
|
|
124
125
|
8. **NFR Assessment** — performance, security, reliability
|
|
125
126
|
9. **Traceability** — requirements-to-tests matrix
|
|
127
|
+
10. **Test Gap Analysis** — scan tests against requirements to identify coverage gaps
|
|
126
128
|
|
|
127
129
|
What's the risk profile we're working with?
|
|
128
130
|
</greeting>
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# Fill Test Gaps Checklist
|
|
2
|
+
|
|
3
|
+
## Pre-Start Gates
|
|
4
|
+
- [ ] At least one `test-gap-analysis-*.md` file exists in `docs/test-artifacts/`
|
|
5
|
+
- [ ] Story files or sprint-status.yaml available for status resolution
|
|
6
|
+
|
|
7
|
+
## Post-Complete Gates
|
|
8
|
+
- [ ] Gap report loaded and parsed without errors (Step 1)
|
|
9
|
+
- [ ] Severity filter applied — filtered_out_count recorded in triage header (Step 2)
|
|
10
|
+
- [ ] Triage map built with one row per story_key, gap types aggregated (Step 3)
|
|
11
|
+
- [ ] Action proposal rules from `gap-triage-rules.js` applied to every row (Step 4)
|
|
12
|
+
- [ ] Skip statuses (in-progress, review, blocked) correctly handled with reason recorded
|
|
13
|
+
- [ ] Unknown gap types marked as skip with reason `unknown_gap_type`
|
|
14
|
+
- [ ] Stories not found in any source marked as skip with reason `story_not_found`
|
|
15
|
+
- [ ] Triage table rendered as markdown and saved to template-output path (Step 5)
|
|
16
|
+
- [ ] Performance: 50-gap report completes Steps 1-5 in under 30 seconds
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
<workflow name="fill-test-gaps">
|
|
2
|
+
<critical>
|
|
3
|
+
<mandate>Read the latest gap analysis report, triage gaps by severity and story, and produce a remediation action table — ADR-039 §10.22.8</mandate>
|
|
4
|
+
<mandate>Action proposal rules MUST be applied from scripts/lib/gap-triage-rules.js — the single source of truth for the §10.22.8.2 rule table</mandate>
|
|
5
|
+
<mandate>Execute each approved remediation action by invoking the appropriate bundled sub-workflow (/gaia-add-stories, /gaia-triage-findings, /gaia-test-automate) via <invoke-workflow> with mode="yolo" — ADR-039 §10.22.8.3 / FR-314</mandate>
|
|
6
|
+
<mandate>Logs-and-continues error handling: a single sub-workflow failure MUST NOT halt the parent workflow — FR-314</mandate>
|
|
7
|
+
<mandate>Emit a remediation report at {test_artifacts}/gap-remediation-report-{date}.md conforming to the §10.22.8.3 schema after all approved actions complete</mandate>
|
|
8
|
+
<mandate>Retry-only-failed semantics: before building the triage table, consult the most recent prior remediation report for this gap report (within 24 hours) and skip rows whose story_key previously succeeded — AC6, AC7</mandate>
|
|
9
|
+
<!-- Dev Note (E19-S27): the bridging ADR-037 adapter in scripts/lib/adr037-return-adapter.js
|
|
10
|
+
normalizes sub-workflow returns while /gaia-add-stories, /gaia-triage-findings, and
|
|
11
|
+
/gaia-test-automate remain pre-E24. Update the adapter in lockstep whenever one of
|
|
12
|
+
those sub-workflows migrates to return ADR-037 natively. See ADR-039 Consequences
|
|
13
|
+
(architecture.md §5347) and story dev notes on E24 soft dependency (§10.22.8.5). -->
|
|
14
|
+
</critical>
|
|
15
|
+
|
|
16
|
+
<!-- ============================== -->
|
|
17
|
+
<!-- Step 1: Load Gap Report (AC2) -->
|
|
18
|
+
<!-- ============================== -->
|
|
19
|
+
|
|
20
|
+
<step n="1" title="Load Gap Report">
|
|
21
|
+
<action>Glob {test_artifacts}/test-gap-analysis-*.md to find all gap analysis reports</action>
|
|
22
|
+
<action>Sort glob results deterministically (lexicographic on filename) before processing</action>
|
|
23
|
+
<action>For each matched file, parse the YAML frontmatter and extract the `date` field</action>
|
|
24
|
+
<action>If a file has missing or malformed frontmatter, HALT with error: "Parse error in {filepath} — malformed or missing frontmatter. Fix the file or regenerate with /gaia-test-gap-analysis" (EC3)</action>
|
|
25
|
+
<action>Select the file with the most recent `date` value (ISO date string comparison)</action>
|
|
26
|
+
<action>If multiple files share the same most-recent date, tie-break on filename (lexicographic ascending — pick the last), and log a warning: "Multiple gap reports share date {date} — using {filename}" (EC2)</action>
|
|
27
|
+
<action>If zero files match the glob, HALT with message: "No gap analysis report found — run `/gaia-test-gap-analysis` first" (EC1)</action>
|
|
28
|
+
<action>Parse the selected report's markdown body to extract the gap table rows. Each row has at minimum: story_key, gap_type, severity, and description</action>
|
|
29
|
+
<action>Store the loaded gaps list and report metadata (filepath, date, total gap count) in workflow context</action>
|
|
30
|
+
</step>
|
|
31
|
+
|
|
32
|
+
<!-- ============================== -->
|
|
33
|
+
<!-- Step 2: Severity Filter (AC3) -->
|
|
34
|
+
<!-- ============================== -->
|
|
35
|
+
|
|
36
|
+
<step n="2" title="Severity Filter">
|
|
37
|
+
<action>Default severity filter: critical + high</action>
|
|
38
|
+
<action>In YOLO mode: apply the default filter without prompting</action>
|
|
39
|
+
<action>In normal mode: present the default filter and prompt the user to adjust (options: critical-only, critical+high, critical+high+medium, all). If the user selects "none", ABORT with message: "No severities selected — nothing to triage" (EC6)</action>
|
|
40
|
+
<action>Apply the selected filter to the gap list</action>
|
|
41
|
+
<action>Record `filtered_out_count` = (total gaps) - (filtered gaps) in the triage header context</action>
|
|
42
|
+
<action>If 0 gaps remain after filtering (EC7): continue to Step 3 with an empty list — the triage table will render with a header note "No gaps match the selected severity filter" and still emit the template-output</action>
|
|
43
|
+
<action>If filtered gap count exceeds 50 (EC8): record an INFO note in the triage header: "Report size {N} exceeds 50-gap perf budget — expect >30s runtime"</action>
|
|
44
|
+
</step>
|
|
45
|
+
|
|
46
|
+
<!-- ============================================ -->
|
|
47
|
+
<!-- Step 3: Group by story_key (AC4) -->
|
|
48
|
+
<!-- ============================================ -->
|
|
49
|
+
|
|
50
|
+
<step n="3" title="Group by story_key">
|
|
51
|
+
<action>Build a triage map from the filtered gap list, keyed by story_key</action>
|
|
52
|
+
<action>Each triage row contains: story_key, gap_count, gap_types (deduplicated array), proposed_action (empty — filled in Step 4), status (pending)</action>
|
|
53
|
+
<action>Stories with multiple gap types collapse into a single triage row with all gap types aggregated</action>
|
|
54
|
+
<action>Sort the triage map deterministically by story_key (lexicographic ascending)</action>
|
|
55
|
+
<action>Retry-only-failed preload (AC6, AC7): import loadPrior from {project-path}/scripts/lib/prior-remediation-loader.js. Call loadPrior with sourceGapReportPath = selected gap report path (from Step 1) and testArtifactsDir = {test_artifacts}. If a non-null prior is returned, store prior.skipMap in workflow context as prior_succeeded_map. If null is returned (no prior or >24h stale), set prior_succeeded_map = {} — every row will be presented fresh.</action>
|
|
56
|
+
<action>For each row in the triage map, if row.story_key exists in prior_succeeded_map, mark the row status = "skip_prior_success", set skip_reason = "succeeded in prior remediation report {prior.reportPath}", and exclude it from the Step 4 rule application (but keep it in the triage table header note so the user sees why it is missing)</action>
|
|
57
|
+
</step>
|
|
58
|
+
|
|
59
|
+
<!-- ============================================== -->
|
|
60
|
+
<!-- Step 4: Action Proposal Rules (AC5) -->
|
|
61
|
+
<!-- ============================================== -->
|
|
62
|
+
|
|
63
|
+
<step n="4" title="Action Proposal">
|
|
64
|
+
<action>For each triage row, resolve the story's current status:
|
|
65
|
+
1. First, try to read the story file at {implementation_artifacts}/{story_key}-*.md and parse its frontmatter `status` field (source of truth per CLAUDE.md Sprint-Status Write Safety)
|
|
66
|
+
2. If the story file does not exist, fall back to sprint-status.yaml and log a warning: "Story file not found for {story_key} — falling back to sprint-status.yaml"
|
|
67
|
+
3. If neither source has the story, mark the row as `skip` with reason `story_not_found` (EC4) and continue to next row</action>
|
|
68
|
+
<action>Import and invoke the `proposeAction` function from {project-path}/scripts/lib/gap-triage-rules.js for each gap in the row</action>
|
|
69
|
+
<action>The rule table (§10.22.8.2):
|
|
70
|
+
- uncovered-ac + backlog/ready-for-dev → action: append_ac, sub_workflow: /gaia-add-stories
|
|
71
|
+
- missing-test + done → action: new_story, sub_workflow: /gaia-triage-findings
|
|
72
|
+
- missing-edge-case + backlog/ready-for-dev → action: append_edge_case, sub_workflow: /gaia-add-stories
|
|
73
|
+
- unexecuted + any status → action: expand_automation, sub_workflow: /gaia-test-automate
|
|
74
|
+
- any gap + in-progress/review/blocked → action: skip, skip_reason: "story is {status} — defer remediation"
|
|
75
|
+
- unknown gap_type → action: skip, skip_reason: "unknown_gap_type" (EC5)</action>
|
|
76
|
+
<action>If a row has multiple gap types, apply the rules to each gap individually and collect all proposed actions for the row. The row's proposed_action becomes the highest-priority non-skip action (or skip if all gaps resolve to skip)</action>
|
|
77
|
+
<action>Mark each row's status as `proposed` (or `skip` if all actions are skip)</action>
|
|
78
|
+
</step>
|
|
79
|
+
|
|
80
|
+
<!-- ================================================ -->
|
|
81
|
+
<!-- Step 5: Triage Table Template-Output (AC6) -->
|
|
82
|
+
<!-- ================================================ -->
|
|
83
|
+
|
|
84
|
+
<step n="5" title="Triage Table Output">
|
|
85
|
+
<action>Render the triage map as a markdown table with columns: story_key | gap_count | gap_types | proposed_action | sub_workflow | status | skip_reason</action>
|
|
86
|
+
<action>Add a triage header containing: report source file, report date, severity filter applied, total gaps, filtered gaps, filtered_out_count, and any INFO notes (perf budget, empty filter, etc.)</action>
|
|
87
|
+
<action>Save the rendered triage table to {test_artifacts}/fill-test-gaps-triage-{date}.md</action>
|
|
88
|
+
<action>In normal mode: present the triage table for user review — the user can approve or modify per-row actions before the file is finalized</action>
|
|
89
|
+
<action>In YOLO mode: auto-accept the default proposals and save immediately</action>
|
|
90
|
+
<template-output path="{test_artifacts}/fill-test-gaps-triage-{date}.md" />
|
|
91
|
+
</step>
|
|
92
|
+
|
|
93
|
+
<!-- ================================================ -->
|
|
94
|
+
<!-- Step 6: Execute Approved Actions (AC1..AC4) -->
|
|
95
|
+
<!-- ================================================ -->
|
|
96
|
+
|
|
97
|
+
<step n="6" title="Execute Approved Actions">
|
|
98
|
+
<action>Initialize an in-memory tracking map as an empty array — rows are appended in Step 6 execution order and consumed by Step 7</action>
|
|
99
|
+
<action>Import normalizeReturn from {project-path}/scripts/lib/adr037-return-adapter.js. Define the allowed bundled refs as the constant array ['add-stories', 'triage-findings', 'test-automate']</action>
|
|
100
|
+
<action>For each triage row where the user approved the proposed action (approved: true in the triage table saved by Step 5):
|
|
101
|
+
1. If action_type is "skip" or the row was marked skip_prior_success in Step 3, append { story_key, action_type, sub_workflow, status: "skipped", artifacts: [], error: null } to the tracking map and continue to the next row — do NOT invoke any sub-workflow for skipped rows
|
|
102
|
+
2. Resolve the bundled ref by stripping the leading "/gaia-" from row.sub_workflow (e.g., "/gaia-add-stories" → "add-stories")
|
|
103
|
+
3. Runtime assertion (AC2): if the resolved ref is not in the allowed bundled refs array, HALT with a workflow-author error: "invoke-workflow ref='{row.sub_workflow}' is not one of the 3 bundled framework sub-workflows (add-stories, triage-findings, test-automate). This is a workflow-author error — the rule table must not propose other refs."
|
|
104
|
+
4. Build structured params from the triage row: for add-stories pass { story_key, appended_ac } (AC/edge-case text derived from the gap row); for triage-findings pass { finding, story_key }; for test-automate pass { story_key, tier }
|
|
105
|
+
5. Emit <invoke-workflow ref="{resolved_ref}" mode="yolo" params="{built_params}" /> — the engine binds the return to a local variable unique to this iteration
|
|
106
|
+
6. Call normalizeReturn(raw_return, resolved_ref) to coerce the return into the ADR-037 schema { status, summary, artifacts, findings, next }
|
|
107
|
+
7. Branch on normalized.status (AC3):
|
|
108
|
+
- "ok" → append row with status: "succeeded", artifacts: normalized.artifacts, error: null
|
|
109
|
+
- "error" → append row with status: "failed", error: normalized.summary (AC4 — logs-and-continues: DO NOT halt, proceed to next row)
|
|
110
|
+
- "halted" → append row with status: "failed", error: "sub-workflow halted: {normalized.summary}"
|
|
111
|
+
- "needs_user" → append row with status: "failed", error: "sub-workflow requested user input despite mode=yolo (unexpected)"; log a warning
|
|
112
|
+
</action>
|
|
113
|
+
<action>At the end of the loop, log the execution summary: "{succeeded_count} succeeded, {failed_count} failed, {skipped_count} skipped"</action>
|
|
114
|
+
</step>
|
|
115
|
+
|
|
116
|
+
<!-- ================================================ -->
|
|
117
|
+
<!-- Step 7: Emit Remediation Report (AC5) -->
|
|
118
|
+
<!-- ================================================ -->
|
|
119
|
+
|
|
120
|
+
<step n="7" title="Emit Remediation Report">
|
|
121
|
+
<action>Import writeReport from {project-path}/scripts/lib/gap-remediation-report-writer.js</action>
|
|
122
|
+
<action>Call writeReport({ trackingMap: (from Step 6), sourceGapReport: (selected gap report path from Step 1), sourceFeatureId: (null unless a feature origin was supplied by caller), outputDir: {test_artifacts}, executionDate: {date} })</action>
|
|
123
|
+
<action>The writer emits {test_artifacts}/gap-remediation-report-{date}.md with frontmatter fields source_gap_report, execution_date, total_actions, succeeded, failed, skipped, source_feature_id and a per-action detail table conforming to architecture §10.22.8.3</action>
|
|
124
|
+
<action>If writeReport throws, HALT with the error message — the report must always be written so retry-only-failed semantics (AC6) can operate on the next run</action>
|
|
125
|
+
<template-output path="{test_artifacts}/gap-remediation-report-{date}.md" />
|
|
126
|
+
</step>
|
|
127
|
+
|
|
128
|
+
</workflow>
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
name: fill-test-gaps
|
|
2
|
+
description: 'Read the latest gap analysis report, filter by severity, group by story, and present a triage table of proposed remediation actions'
|
|
3
|
+
module: testing
|
|
4
|
+
agent: test-architect
|
|
5
|
+
execution_mode: planning
|
|
6
|
+
config_resolved: "{installed_path}/.resolved/fill-test-gaps.yaml"
|
|
7
|
+
config_source: "{project-root}/_gaia/testing/config.yaml"
|
|
8
|
+
installed_path: "{project-root}/_gaia/testing/workflows/fill-test-gaps"
|
|
9
|
+
instructions: "{installed_path}/instructions.xml"
|
|
10
|
+
validation: "{installed_path}/checklist.md"
|
|
11
|
+
traces_to:
|
|
12
|
+
- FR-313
|
|
13
|
+
- ADR-039
|
|
14
|
+
performance_constraints:
|
|
15
|
+
max_duration_seconds: 30 # AC7 — 50-gap report must complete Steps 1-5 in under 30s
|
|
16
|
+
input_file_patterns:
|
|
17
|
+
gap_report:
|
|
18
|
+
whole: "{test_artifacts}/test-gap-analysis-*.md"
|
|
19
|
+
load_strategy: "LATEST_BY_DATE"
|
|
20
|
+
story_files:
|
|
21
|
+
whole: "{implementation_artifacts}/*.md"
|
|
22
|
+
load_strategy: "INDEX_GUIDED"
|
|
23
|
+
sprint_status:
|
|
24
|
+
whole: "{implementation_artifacts}/sprint-status.yaml"
|
|
25
|
+
load_strategy: "FULL_LOAD"
|
|
26
|
+
output:
|
|
27
|
+
primary: "{test_artifacts}/fill-test-gaps-triage-{date}.md"
|
|
28
|
+
on_error:
|
|
29
|
+
missing_file: "halt"
|
|
30
|
+
unresolved_variable: "halt"
|
|
@@ -35,26 +35,53 @@
|
|
|
35
35
|
<action>For each story's acceptance criteria, check if a corresponding test case exists in the test plan</action>
|
|
36
36
|
<action>Flag each AC as covered (has test case) or uncovered-ac (no test case found)</action>
|
|
37
37
|
<action>Calculate coverage rate per story: covered_ACs / total_ACs</action>
|
|
38
|
-
<action>Calculate overall coverage percentage across all stories
|
|
38
|
+
<action>Calculate overall coverage percentage across all stories. Note: the authoritative calculation logic lives in `scripts/lib/coverage-calc.js` (E19-S25). Step 4c uses the shared helper for per-module and aggregate percentages.</action>
|
|
39
|
+
</step>
|
|
40
|
+
|
|
41
|
+
<step n="4b" title="Frontend Dimensions Analysis (conditional)">
|
|
42
|
+
<action>Frontend detection via shared module — call detectProjectType({project-path}) from scripts/lib/project-type-detection.js. If result.type is 'frontend', 'fullstack', or 'mobile', set is_frontend=true and project_type=result.type. If result.type is any other value, set is_frontend=false. The evidence array provides the matched signals for documentation. (E19-S22: extracted from inline duplication to shared module.)</action>
|
|
43
|
+
<action if="is_frontend == false">Skip this step entirely. Do NOT emit a "## Frontend Dimensions" section in the output artifact. Proceed directly to Step 5. No error, no warning, no empty table. Non-frontend projects must see no trace of this section.</action>
|
|
44
|
+
<action if="is_frontend == true">For each of the six architectural dimensions (per ADR-030 §10.22.3 and FR-224), scan the project and compute (gap_count, coverage_score 0-100%, top_3_uncovered):
|
|
45
|
+
1. **Unit Tests** — glob `src/**/*.{test,spec}.{ts,tsx,js,jsx}`. If an LCOV report is present at `coverage/lcov.info` or `coverage/lcov-report/`, parse line-coverage percentage. Gap if coverage < 60% or no spec files found. top_3_uncovered = first 3 source files with lowest coverage (or "no spec files" when absent).
|
|
46
|
+
2. **E2E Tests** — glob `tests/**/*.spec.{ts,js}`, `e2e/**/*.spec.{ts,js}`, and `cypress/e2e/**/*.cy.{ts,js}`. Gap if no E2E spec files found. top_3_uncovered = names of 3 largest source components lacking an E2E spec.
|
|
47
|
+
3. **Cross-browser** — read `playwright.config.{ts,js}` or `cypress.config.{ts,js}` and extract configured browser projects. Gap if only chromium/electron present, or if config file is absent. top_3_uncovered = missing browsers from the set {chromium, firefox, webkit}.
|
|
48
|
+
4. **Accessibility** — read `package.json` dependencies and devDependencies. Gap if none of `axe-core`, `jest-axe`, `@testing-library/jest-axe`, `@axe-core/playwright`, or `cypress-axe` are present. top_3_uncovered = the missing canonical a11y packages for the detected test runner.
|
|
49
|
+
5. **Visual Regression** — read `package.json` dependencies. Gap if none of `@percy/cli`, `@percy/playwright`, `@chromatic-com/playwright`, `chromatic`, or `playwright-visual-comparisons` are present. top_3_uncovered = 3 of the missing canonical visual-regression packages.
|
|
50
|
+
6. **Responsive/Viewport** — grep test files for any of: `setViewportSize`, `cy.viewport`, `--viewport`, `viewport:` in Playwright config. Gap if no matches. top_3_uncovered = names of 3 top-level pages/components with no responsive test.</action>
|
|
51
|
+
<action if="is_frontend == true">Populate the "## Frontend Dimensions" section of the output template with 6 rows, one per dimension, each row containing (gap_count, coverage_score, top_3_uncovered). The per-dimension coverage_score feeds E19-S6's overall coverage aggregation.</action>
|
|
52
|
+
</step>
|
|
53
|
+
|
|
54
|
+
<step n="4c" title="Per-Module Coverage Calculation (E19-S6, FR-225)">
|
|
55
|
+
<action>Group all scanned story files by epic key — parse the `epic:` frontmatter field from each story file. Stories sharing the same epic prefix (e.g., `E19`) are aggregated into a single module row. Story files whose `epic:` field is missing or empty are grouped under a synthetic `ungrouped` module key.</action>
|
|
56
|
+
<action>For each epic group, count `total_acs` — the sum of acceptance criteria items (`AC1`, `AC2`, ...) across all story files in the group, reusing the story→AC map built in Step 3.</action>
|
|
57
|
+
<action>For each epic group, count `tested_acs` — the number of acceptance criteria in the group that have at least one matching test case in the test plan map built in Step 2. An AC is considered tested when a test case explicitly references either the story key or the `{story_key}/ACn` identifier.</action>
|
|
58
|
+
<action>For each epic group, count `gap_count` — the number of ACs in the group classified as `uncovered-ac` by Step 4.</action>
|
|
59
|
+
<action>Calculate per-epic and aggregate coverage percentages using the shared helper `scripts/lib/coverage-calc.js` as the single source of truth (E19-S25). Build the input array with one entry per epic group: `{ name: epicKey, covered: tested_acs, total: total_acs }`. Call `calculateCoverage({ modules })` to obtain `perModule` (per-epic percentages) and `overall` (aggregate percentage). The helper applies banker's rounding to one decimal place and returns 0 for division-by-zero cases. See the module JSDoc for the full contract.</action>
|
|
60
|
+
<action>Exclude epics with zero story files from the per-module table. Count the number excluded and emit a footnote beneath the table: "{N} epics with no story files were excluded." When N is 0, omit the footnote entirely.</action>
|
|
61
|
+
<action>Determinism requirement (AC5): the computation MUST be deterministic — given identical inputs (same story files, same test plan), the workflow MUST produce byte-identical per-module rows, aggregate percentage, and sort order. The `calculateCoverage` helper guarantees deterministic output (sorted keys, banker's rounding, no side effects). Sort the per-module table rows by `coverage_pct` ascending (lowest coverage first); break ties by epic key lexicographical ascending.</action>
|
|
62
|
+
<action>Populate the "## Per-Module Coverage" section of the output template with one row per included epic, columns: `module`, `total_acs`, `tested_acs`, `coverage_pct`, `gap_count`. The `module` column renders as the epic key (e.g., `E19`). The `coverage_pct` column renders as `{n}.{d}%` (one decimal + percent sign).</action>
|
|
39
63
|
</step>
|
|
40
64
|
|
|
41
65
|
<step n="5" title="Generate Output (FR-223 Schema)">
|
|
42
66
|
<action>Generate the output artifact at {test_artifacts}/test-gap-analysis-{date}.md</action>
|
|
43
67
|
<action>Output must include the following FR-223 schema sections:
|
|
44
|
-
- Summary section with total count of stories analyzed, ACs scanned, gaps found, and overall coverage percentage
|
|
68
|
+
- Summary section with total count of stories analyzed, ACs scanned, gaps found, and overall coverage percentage (from Step 4c `aggregate_coverage_pct`)
|
|
69
|
+
- Per-module coverage table (from Step 4c) with columns module, total_acs, tested_acs, coverage_pct, gap_count
|
|
45
70
|
- Per-story gap table listing each story, its ACs, and their coverage status
|
|
46
71
|
- Coverage rate breakdown showing covered vs uncovered acceptance criteria
|
|
47
72
|
</action>
|
|
48
73
|
<action>If zero gaps are detected (all ACs have corresponding test cases):
|
|
49
74
|
Output "No coverage gaps detected" in the summary section with a gap count of 0 and coverage rate of 100%</action>
|
|
50
75
|
<template-output file="{test_artifacts}/test-gap-analysis-{date}.md">
|
|
51
|
-
Gap analysis report following FR-223 schema with summary count, per-story gap table, and coverage percentage.
|
|
76
|
+
Gap analysis report following FR-223 schema with summary count, per-module coverage table, per-story gap table, and aggregate coverage percentage.
|
|
52
77
|
</template-output>
|
|
53
78
|
</step>
|
|
54
79
|
|
|
55
80
|
<step n="6" title="Performance Validation (Coverage Mode)">
|
|
56
81
|
<action>Verify workflow completed within the NFR-040 constraint of under 60 seconds</action>
|
|
57
82
|
<action>Log total execution time in the output footer</action>
|
|
83
|
+
<!-- E19-S28 AC1 / FR-315: completion nudge — display-only, never auto-executes, emitted in both normal and YOLO modes -->
|
|
84
|
+
<action if="total_gaps > 0">Emit the completion nudge to the user verbatim: "Run `/gaia-fill-test-gaps` to remediate these gaps now." Do NOT auto-invoke the remediation workflow — the user retains full control. Emit in both normal and YOLO execution modes.</action>
|
|
58
85
|
</step>
|
|
59
86
|
|
|
60
87
|
<!-- ================================= -->
|
|
@@ -63,10 +90,10 @@
|
|
|
63
90
|
<!-- ================================= -->
|
|
64
91
|
|
|
65
92
|
<step n="7" title="Scan Generated Test Cases (Verification Mode)">
|
|
66
|
-
<action>Scan docs/test-artifacts/ for all generated test case files and
|
|
93
|
+
<action>Scan docs/test-artifacts/ for all generated test case files matching `{story_key}-*.md` and ATDD files `atdd-{story_key}*.md` (E19-S7 Task 1.1)</action>
|
|
67
94
|
<action>Build a map of generated test cases per story: story_key -> [test_case_id, test_file, test_name]</action>
|
|
68
95
|
<action>Cross-reference generated test cases with story files in docs/implementation-artifacts/</action>
|
|
69
|
-
<action>Count total generated test cases per story and in aggregate
|
|
96
|
+
<action>Count total generated test cases per story (`generated` count — E19-S7 Task 1.2) and in aggregate (`total_generated`). The per-story `generated` count is the number of distinct test case IDs (TC IDs) attributed to that story across all scanned test-artifacts files.</action>
|
|
70
97
|
</step>
|
|
71
98
|
|
|
72
99
|
<step n="8" title="Detect and Parse Execution Results (Verification Mode)">
|
|
@@ -86,27 +113,33 @@
|
|
|
86
113
|
<step n="9" title="Cross-Reference Generated vs Executed (Verification Mode)">
|
|
87
114
|
<action>For each story, cross-reference its generated test cases against the executed tests map</action>
|
|
88
115
|
<action>Classify each generated test case as: EXECUTED (found in results with pass/fail status) or UNEXECUTED (not found in any result file)</action>
|
|
89
|
-
<action>Calculate per
|
|
90
|
-
<action>
|
|
91
|
-
<action>
|
|
116
|
+
<action>Calculate per-story counts: `generated` (count of generated test cases for the story) and `executed` (count of generated test cases found in execution results for the story). Per E19-S7 Task 2.1/2.2, `executed` is sourced from evidence JSON (E17-S10 format) and JUnit XML; Task 2.3 requires deduplication across result sources — a single test case counted in both evidence JSON and JUnit XML is counted once.</action>
|
|
117
|
+
<action>Deduplicate executed test cases across result sources: if the same test identifier appears in both JUnit XML and evidence JSON, count it exactly once in the per-story `executed` tally. Deduplication key is the canonical test case ID (TC-ID) when present, otherwise the fully-qualified test name (`suite.test_name`).</action>
|
|
118
|
+
<action>Calculate per-story `exec_ratio = (executed / generated) * 100`, rounded to one decimal place. Division-by-zero handling (E19-S7 Task 3.3): when `generated == 0`, set `exec_ratio = 0.0` and render as `0.0%` with the note `0/0 (no generated tests)`. Never raise a division-by-zero error.</action>
|
|
119
|
+
<action>Calculate aggregate totals: `total_generated` = sum of per-story `generated` across all stories; `total_executed` = sum of per-story `executed` across all stories. Compute `aggregate_exec_ratio = (total_executed / total_generated) * 100`, rounded to one decimal place. When `total_generated == 0`, set `aggregate_exec_ratio = 0.0`.</action>
|
|
120
|
+
<action>Flag stories with zero executed tests (where `executed == 0` and `generated > 0`) as HIGH priority gaps — these stories have generated test cases but none have been run. The flag is reflected in the Gap Table row severity (`high`) and in the Per-Story Detail remediation line. Per E19-S7 Task 3.2, the HIGH flag is mandatory whenever `executed count is 0` for a story with at least one generated test.</action>
|
|
92
121
|
</step>
|
|
93
122
|
|
|
94
123
|
<step n="10" title="Generate Verification Output (FR-226 Schema)">
|
|
95
|
-
<action>Generate the output artifact at {test_artifacts}/test-gap-analysis-{date}.md in verification mode format
|
|
124
|
+
<action>Generate the output artifact at {test_artifacts}/test-gap-analysis-{date}.md in verification mode format, using the FR-223 schema template at `_gaia/lifecycle/templates/test-gap-analysis-template.md`.</action>
|
|
125
|
+
<action>Populate the Executive Summary section with the aggregate **Generated vs Executed** row (E19-S7, AC3) in the exact format `Generated vs Executed: {total_executed}/{total_generated} ({aggregate_exec_ratio}%)`. This row sits alongside `Stories analyzed`, `Gaps detected`, and `Overall coverage rate`. When `total_generated == 0`, render `0/0 (0.0%)`.</action>
|
|
126
|
+
<action>Populate the Per-Story Detail section for each story with the `generated`, `executed`, and `exec_ratio` fields (E19-S7, AC2, AC5). The `exec_ratio` field is rendered as a percentage with one decimal place (e.g., `60.0%`). When `generated == 0`, render `exec_ratio` as `0.0%` with the inline note `(0/0 — no generated tests)` — this is the canonical division-by-zero rendering.</action>
|
|
96
127
|
<action>Output must include the following FR-226 schema sections:
|
|
97
|
-
- Summary
|
|
98
|
-
- Per-story generated-vs-executed count table listing each story with its generated
|
|
99
|
-
- Aggregate generated-vs-executed count showing overall totals
|
|
128
|
+
- Executive Summary with the aggregate `Generated vs Executed: {total_executed}/{total_generated} ({aggregate_exec_ratio}%)` row
|
|
129
|
+
- Per-story generated-vs-executed count table listing each story with its `generated`, `executed`, and `exec_ratio` fields
|
|
130
|
+
- Aggregate generated-vs-executed count showing overall totals (`total_generated`, `total_executed`, `aggregate_exec_ratio`)
|
|
100
131
|
- Unverified tests table listing test files with UNVERIFIED status and story key
|
|
101
132
|
</action>
|
|
102
|
-
<action>For stories
|
|
133
|
+
<action>For stories where `executed count is 0` (zero executed tests) and `generated > 0`: set gap priority to HIGH in the per-story table (E19-S7, AC4). The HIGH flag MUST also appear as the severity of the corresponding Gap Table row when the workflow emits an `unexecuted` gap for the story.</action>
|
|
103
134
|
<template-output file="{test_artifacts}/test-gap-analysis-{date}.md">
|
|
104
|
-
Verification mode gap analysis report with per-story and aggregate generated-vs-executed counts, HIGH priority flags for zero-executed stories.
|
|
135
|
+
Verification mode gap analysis report with per-story and aggregate generated-vs-executed counts, exec_ratio percentages, and HIGH priority flags for zero-executed stories.
|
|
105
136
|
</template-output>
|
|
106
137
|
</step>
|
|
107
138
|
|
|
108
139
|
<step n="11" title="Performance Validation (Verification Mode)">
|
|
109
140
|
<action>Verify workflow completed within the NFR-040 constraint of under 60 seconds</action>
|
|
110
141
|
<action>Log total execution time in the output footer</action>
|
|
142
|
+
<!-- E19-S28 AC1 / FR-315: completion nudge — display-only, never auto-executes, emitted in both normal and YOLO modes -->
|
|
143
|
+
<action if="total_gaps > 0">Emit the completion nudge to the user verbatim: "Run `/gaia-fill-test-gaps` to remediate these gaps now." Do NOT auto-invoke the remediation workflow — the user retains full control. Emit in both normal and YOLO execution modes.</action>
|
|
111
144
|
</step>
|
|
112
145
|
</workflow>
|
|
@@ -12,6 +12,7 @@ config_source: "{project-root}/_gaia/testing/config.yaml"
|
|
|
12
12
|
installed_path: "{project-root}/_gaia/testing/workflows/test-gap-analysis"
|
|
13
13
|
instructions: "{installed_path}/instructions.xml"
|
|
14
14
|
validation: "{installed_path}/checklist.md"
|
|
15
|
+
template: "{project-root}/_gaia/lifecycle/templates/test-gap-analysis-template.md"
|
|
15
16
|
traces_to:
|
|
16
17
|
- FR-221
|
|
17
18
|
- FR-222
|
package/gaia-install.sh
CHANGED
|
@@ -809,6 +809,43 @@ cmd_update() {
|
|
|
809
809
|
fi
|
|
810
810
|
echo ""
|
|
811
811
|
info "Run /gaia-build-configs in the Claude Code terminal to regenerate resolved configs."
|
|
812
|
+
|
|
813
|
+
# ─── E19-S17: Post-upgrade gap-analysis suggestion (FR-236) ───────────────
|
|
814
|
+
# When docs/test-artifacts/test-plan.md exists in the target project, suggest
|
|
815
|
+
# running /gaia-test-gap-analysis after the upgrade. The suggestion is
|
|
816
|
+
# recommended (NOT required), is logged to stdout only, and is NEVER
|
|
817
|
+
# auto-executed by the installer. The same suggestion is mirrored as a
|
|
818
|
+
# 'suggestion' entry in .post-upgrade-hook.yaml under the recommended group
|
|
819
|
+
# so the Claude Code runtime can offer it during the post-upgrade gate.
|
|
820
|
+
if [[ "$OPT_DRY_RUN" != true ]]; then
|
|
821
|
+
local test_plan_file="$TARGET/docs/test-artifacts/test-plan.md"
|
|
822
|
+
if [[ -f "$test_plan_file" ]]; then
|
|
823
|
+
echo ""
|
|
824
|
+
info "[recommended] Run \`/gaia-test-gap-analysis\` to check for new test gaps introduced by the upgrade."
|
|
825
|
+
|
|
826
|
+
local hook_file="$TARGET/_gaia/_config/.post-upgrade-hook.yaml"
|
|
827
|
+
mkdir -p "$(dirname "$hook_file")"
|
|
828
|
+
if [[ ! -f "$hook_file" ]]; then
|
|
829
|
+
cat > "$hook_file" <<'YAML'
|
|
830
|
+
schema_version: 1
|
|
831
|
+
status: pending
|
|
832
|
+
recommended: []
|
|
833
|
+
suggestions: []
|
|
834
|
+
YAML
|
|
835
|
+
fi
|
|
836
|
+
# Append the gap-analysis suggestion if not already present.
|
|
837
|
+
if ! grep -q "/gaia-test-gap-analysis" "$hook_file"; then
|
|
838
|
+
cat >> "$hook_file" <<'YAML'
|
|
839
|
+
- id: test-gap-analysis-post-upgrade
|
|
840
|
+
type: suggestion
|
|
841
|
+
command: /gaia-test-gap-analysis
|
|
842
|
+
condition: test_plan_exists
|
|
843
|
+
message: "Run /gaia-test-gap-analysis to check for new test gaps introduced by the upgrade."
|
|
844
|
+
YAML
|
|
845
|
+
fi
|
|
846
|
+
fi
|
|
847
|
+
fi
|
|
848
|
+
|
|
812
849
|
echo ""
|
|
813
850
|
}
|
|
814
851
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "gaia-framework",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.127.2",
|
|
4
4
|
"description": "GAIA — Generative Agile Intelligence Architecture installer",
|
|
5
5
|
"bin": {
|
|
6
6
|
"gaia-framework": "./bin/gaia-framework.js"
|
|
@@ -16,8 +16,8 @@
|
|
|
16
16
|
"test:all": "vitest run && node -e \"if(process.platform==='win32'){console.log('BATS tests skipped on Windows');process.exit(0)}\" && bats test/shell/",
|
|
17
17
|
"test:tier2": "vitest run --config vitest.config.tier2.js",
|
|
18
18
|
"fixtures:check": "node test/fixtures/sync-check.js",
|
|
19
|
-
"lint": "eslint bin/**/*.js test/**/*.js",
|
|
20
|
-
"lint:fix": "eslint --fix bin/**/*.js test/**/*.js",
|
|
19
|
+
"lint": "eslint bin/**/*.js test/**/*.js scripts/lib/**/*.js",
|
|
20
|
+
"lint:fix": "eslint --fix bin/**/*.js test/**/*.js scripts/lib/**/*.js",
|
|
21
21
|
"format:check": "prettier --check \"bin/**/*.js\" \"test/**/*.js\"",
|
|
22
22
|
"format": "prettier --write \"bin/**/*.js\" \"test/**/*.js\"",
|
|
23
23
|
"bench": "node scripts/benchmark-runner.mjs",
|