@tea-agent/loop-agent 0.13.0-beta.0 → 0.14.0

package/docs/templates/backend-test-dag.json

@@ -7,15 +7,26 @@
     "agentRuntime": "pi-only",
     "repairWriterProtocol": "explicit-node-v1"
   },
-  "objective": "End-to-end backend functional testing pipeline: analyze requirements as Backend Test Analysis v1 JSON → validate analysis contract → generate functional test cases → review cases → generate pytest automation → execute pytest → retrospective with maturity rating. Covers the full chain from requirement analysis to test maturity assessment.",
+  "objective": "End-to-end backend functional testing pipeline: analyze requirements as Backend Test Analysis v1 JSON → validate analysis contract → environment scout + execution contract → generate functional test cases → emit+validate Case Manifest v1 → review cases → generate pytest automation → traceability gate → preflight + execute pytest → parse Result v1 → classify failures → retrospective → outcome gate. Covers the full chain from requirement analysis to test maturity assessment with always-run retrospect on assertion failures and deterministic AC→case→symbol traceability.",
   "successCriteria": [
     "analyze-inputs-pi returns pure Backend Test Analysis v1 JSON (schema docs/templates/backend-test-analysis.schema.json) with no Markdown prose",
     "backend-test-analysis-contract-shell validates schemaId backend-test-analysis-v1 and materializes run-owned contracts/backend-test-analysis.json",
-    "generate-backend-functional-cases-pi consumes the validated analysis artifact and produces structured test cases with BE-<MODULE>-<NNN> IDs under testcase/md/",
+    "backend-test-environment-scout-pi returns pure Backend Test Execution Contract v1 JSON (schema docs/templates/backend-test-execution.schema.json)",
+    "backend-test-execution-contract-shell validates schemaId backend-test-execution-v1 and materializes run-owned contracts/backend-test-execution.json",
+    "generate-backend-functional-cases-pi consumes validated contracts and produces structured test cases with BE-<MODULE>-<NNN> IDs under testcase/md/",
+    "emit-backend-case-manifest-pi returns pure Backend Test Case Manifest v1 JSON (schema docs/templates/backend-test-case-manifest.schema.json) with no Markdown prose",
+    "backend-test-case-manifest-shell validates schemaId backend-test-case-manifest-v1 and materializes run-owned contracts/backend-test-case-manifest.json with deterministic AC coverage",
+    "backend-test-traceability-gate-shell verifies generated file/symbol existence after pytest generation and before execute",
+    "test-retrospect-pi uses Case Manifest coverageSummary + Result v1 only for coverage/pass rate inputs (model must not invent percentages)",
     "review-backend-cases-pi returns VERDICT: pass or request-revision with coverage assessment",
     "review-backend-cases-gate-shell blocks pytest generation unless the review verdict is VERDICT: pass",
-    "generate-backend-pytest-pi converts reviewed cases into pytest code with 1:1 traceability under testcase/",
-    "execute-backend-pytest-shell runs pytest with a read-only worktree and writes JUnit XML under the current HARNESS_DAG_RUN_DIR/reports/** only",
+    "generate-backend-pytest-pi converts reviewed cases into pytest code using verified analysis + execution contracts under testcase/",
+    "execute-backend-pytest-shell runs deterministic preflight against the execution contract, then pytest with frozen Adapter testRoot (default testcase/), writing JUnit XML under the current $HARNESS_DAG_RUN_DIR/reports/** only",
+    "execute-backend-pytest-shell maps pytest exit 0/1 to node success only when non-empty JUnit exists; raw pytestExitCode is side-channeled under reports/",
+    "parse-backend-test-result-shell materializes run-owned contracts/backend-test-result.json (Backend Test Result v1) from JUnit + pytestExitCode",
+    "classify-backend-test-result-pi returns structured failure classification JSON (ProductBug|TestBug|EnvFailure|ContractMismatch|FlakyTest|Unknown) without writing files",
+    "test-retrospect-pi runs after parse+classify on both pass and assertion-fail paths; stats come from Result v1 only",
+    "backend-test-outcome-gate-shell exits non-zero unless result.outcome=passed (shell facts only; retrospective cannot override)",
     "test-retrospect-pi generates retrospective report with A/B/C/D maturity rating under docs/test-reports/",
     "Full traceability from acceptance criteria → functional test case ID → pytest function name"
   ],
@@ -36,7 +47,10 @@
     "execute-backend-pytest-shell must not modify test assertions or production code to make tests pass; test failures indicate potential implementation issues and must be reported honestly.",
     "Prompt templates must not ask main session to write artifacts; node output is the artifact and runner archives it under .harness/dag-runs/.",
     "Actual file operation paths must be macOS/Windows compatible; use / only for stable repo refs, JSON/Markdown evidence refs, and glob conventions.",
-    "Same-rank exclusive writeSet entries must be disjoint."
+    "Same-rank exclusive writeSet entries must be disjoint.",
+    "parse-backend-test-result-shell materializes Backend Test Result v1 from JUnit + pytestExitCode; classify/retrospect run on pass and assertion-fail; backend-test-outcome-gate-shell uses result.outcome only.",
+    "backend-test-case-manifest-shell validates schemaId backend-test-case-manifest-v1 and materializes contracts/backend-test-case-manifest.json; AC coverage is fail-closed and deterministic.",
+    "backend-test-traceability-gate-shell verifies generated file/symbol existence after pytest generation and before execute; models must not invent coverage percentages."
   ],
   "defaults": {
     "executor": "pi",
@@ -136,15 +150,76 @@
       }
     },
     {
-      "id": "generate-backend-functional-cases-pi",
+      "id": "backend-test-environment-scout-pi",
       "depends_on": [
         "backend-test-analysis-contract-shell"
       ],
       "complexity": "MED",
       "executor": "pi",
+      "role": "scout",
+      "writePolicy": "read-only",
+      "allowedPaths": [
+        "REPLACE/WITH/SOURCE/PATH/**"
+      ],
+      "forbiddenPaths": [
+        ".harness/**",
+        "artifacts/**"
+      ],
+      "outputContract": "Pure Backend Test Execution Contract v1 JSON object matching docs/templates/backend-test-execution.schema.json. No Markdown prose and no file writes.",
+      "retryPolicy": {
+        "maxAttempts": 3,
+        "backoff": "exponential",
+        "initialDelayMs": 2000,
+        "maxDelayMs": 30000,
+        "retryCategories": [
+          "timeout",
+          "network",
+          "rate-limit",
+          "unavailable"
+        ]
+      },
+      "subtask_prompt": "Read-only environment scout for backend-test pytest MVP.\n\nReturn exactly one JSON object matching Backend Test Execution Contract v1 (schema docs/templates/backend-test-execution.schema.json).\n\nPrefer pure JSON; a single fenced json block is tolerated; no trailing prose.\n\nDiscover only non-secret evidence: pytest config files (pytest.ini / pyproject.toml / setup.cfg test paths), candidate test roots, existing fixtures/clients, documented run commands, and env *names* (not values).\n\nDo NOT search the whole repo for secrets, .env values, tokens, private keys, or production credentials.\n\nframework must be \"pytest\". Default targetMode to \"in-process\" unless evidence clearly shows an external service base URL env name or documented managed start/stop with sourceRef.\n\nDo NOT select targetMode \"managed-command\" unless task source documents a safe start/stop command with an explicit sourceRef; otherwise leave managedCommand absent and record the gap in evidenceGaps.\n\ntestRoot and workingDirectory must be repo-relative posix paths without .. or absolute form. Adapter default testRoot is testcase when evidence is incomplete.\n\nrunner must not include secret values. report.format must be junit with a relativeHint under the run (e.g. reports/backend-test-junit.xml).\n\nrequiredEnvNames lists env NAMES only. baseUrlEnvName is required only for external-running-service and must match ^[A-Z_][A-Z0-9_]*$.\n\nRecord incomplete discovery in evidenceGaps. Populate evidenceRefs with repo-relative paths actually read.\n\nRequired top-level keys: schemaVersion, framework, runner, testRoot, workingDirectory, report, targetMode, existingFixtures, authenticationMode, requiredEnvNames, dataIsolation, evidenceGaps, evidenceRefs.\n\nRead-only: do not modify code, docs, artifacts, or repository files."
+    },
+    {
+      "id": "backend-test-execution-contract-shell",
+      "depends_on": [
+        "backend-test-environment-scout-pi"
+      ],
+      "complexity": "LOW",
+      "executor": "shell",
+      "role": "verifier",
+      "writePolicy": "read-only",
+      "allowedPaths": [
+        "REPLACE/WITH/SOURCE/PATH/**"
+      ],
+      "forbiddenPaths": [
+        ".harness/**",
+        "artifacts/**"
+      ],
+      "outputContract": "Validated run-owned Backend Test Execution Contract v1 artifact pointer, schema ID, and SHA-256.",
+      "subtask_prompt": "Materialize and validate the backend-test execution contract under the current DAG run.",
+      "shell": {
+        "commands": [],
+        "jsonArtifactGate": {
+          "fromNodeId": "backend-test-environment-scout-pi",
+          "schemaId": "backend-test-execution-v1",
+          "artifactName": "backend-test-execution.json",
+          "outputDir": "contracts"
+        },
+        "cwd": ".",
+        "timeoutMs": 60000
+      }
+    },
+    {
+      "id": "generate-backend-functional-cases-pi",
+      "depends_on": [
+        "backend-test-execution-contract-shell"
+      ],
+      "complexity": "MED",
+      "executor": "pi",
       "role": "implementer",
-      "toolProfile": "write",
       "writePolicy": "exclusive",
+      "toolProfile": "write",
       "writeSet": [
         "testcase/md/**"
       ],
@@ -155,15 +230,76 @@
         ".harness/**",
         "artifacts/**"
       ],
-      "outputContract": "Structured backend functional test cases in Markdown under testcase/md/. Each case uses BE-<MODULE>-<NNN> ID format.",
-      "subtask_prompt": "Read the validated structured artifact pointer from backend-test-analysis-contract-shell and generate cases only from that JSON contract.\n\n## Output Steps (do in order):\n1. First, output a brief summary: how many modules, how many cases planned per module\n2. Then write each test case file under testcase/md/\n\n## Format Rules:\n- Each test case ID: BE-<MODULE>-<NNN> (e.g. BE-ORDER-001)\n- Each file covers one module\n- Case structure: ID, Title, Precondition, Steps, Expected Result\n- Map each case to acceptance criteria (AC-xxx)\n\n## Coverage Requirements:\n- Positive paths: happy path for each acceptance criterion\n- Negative paths: error scenarios (invalid input, not found, state violations)\n\n## Conditional Coverage (include ONLY if mentioned in upstream analysis):\n- Boundary conditions: include ONLY if upstream analyze-inputs-pi mentions value ranges, length limits, numeric bounds, or format constraints\n- State transitions: include ONLY if upstream analyze-inputs-pi mentions state machine\n- Authentication scenarios: include ONLY if upstream analyze-inputs-pi mentions auth mechanism\n- Timeout scenarios: include ONLY if upstream analyze-inputs-pi mentions timeout handling\n- Concurrency scenarios: include ONLY if upstream analyze-inputs-pi mentions concurrency/idempotency rules\n- If not mentioned, do NOT generate these test cases\n\n## Constraints:\n- Stay within writeSet: testcase/md/**\n- Do NOT re-read source documents or fall back to free-form analysis — use the validated structured artifact only\n- Do not write root artifacts/**"
+      "subtask_prompt": "Read the validated structured artifact pointer from backend-test-analysis-contract-shell and generate cases only from that JSON contract.\n\n\n\n\n\n## Output Steps (do in order):\n\n1. First, output a brief summary: how many modules, how many cases planned per module\n\n2. Then write each test case file under testcase/md/\n\n\n\n## Format Rules:\n\n- Each test case ID: BE-<MODULE>-<NNN> (e.g. BE-ORDER-001)\n\n- Each file covers one module\n\n- Case structure: ID, Title, Precondition, Steps, Expected Result\n\n- Map each case to acceptance criteria (AC-xxx)\n\n\n\n## Coverage Requirements:\n\n- Positive paths: happy path for each acceptance criterion\n\n- Negative paths: error scenarios (invalid input, not found, state violations)\n\n\n\n## Conditional Coverage (include ONLY if mentioned in upstream analysis):\n\n- Boundary conditions: include ONLY if upstream analyze-inputs-pi mentions value ranges, length limits, numeric bounds, or format constraints\n\n- State transitions: include ONLY if upstream analyze-inputs-pi mentions state machine\n\n- Authentication scenarios: include ONLY if upstream analyze-inputs-pi mentions auth mechanism\n\n- Timeout scenarios: include ONLY if upstream analyze-inputs-pi mentions timeout handling\n\n- Concurrency scenarios: include ONLY if upstream analyze-inputs-pi mentions concurrency/idempotency rules\n\n- If not mentioned, do NOT generate these test cases\n\n\n\n## Constraints:\n\n- Stay within writeSet: testcase/md/**\n\n- Do NOT re-read source documents or fall back to free-form analysis — use the validated structured artifact only\n\n\n\n- Do not write root artifacts/**"
     },
     {
-      "id": "review-backend-cases-pi",
+      "id": "emit-backend-case-manifest-pi",
       "depends_on": [
         "generate-backend-functional-cases-pi",
         "backend-test-analysis-contract-shell"
       ],
+      "complexity": "MED",
+      "executor": "pi",
+      "role": "scout",
+      "writePolicy": "read-only",
+      "allowedPaths": [
+        "**"
+      ],
+      "forbiddenPaths": [
+        ".harness/**",
+        "artifacts/**"
+      ],
+      "outputContract": "Pure Backend Test Case Manifest v1 JSON (schema docs/templates/backend-test-case-manifest.schema.json). No file writes; model must not write .harness/**.",
+      "retryPolicy": {
+        "maxAttempts": 3,
+        "backoff": "exponential",
+        "initialDelayMs": 2000,
+        "maxDelayMs": 30000,
+        "retryCategories": [
+          "timeout",
+          "network",
+          "rate-limit",
+          "unavailable"
+        ]
+      },
+      "subtask_prompt": "Emit Backend Test Case Manifest v1 as pure JSON (or one fenced json block with no trailing text).\n\nRead-only: use validated contracts/backend-test-analysis.json pointer + testcase/md/** only. Do not write repository files or .harness/**.\n\nsourceBinding must match the analysis contract / DAG source binding exactly (taskId, requirementPath, requirementSha256, referencePaths, requirementIds).\n\nFor each functional case under testcase/md/: caseId BE-<MODULE>-<NNN>, acIds[], title, category, automationStatus.\n\nAfter case generation (pre-pytest), default automationStatus=planned. Use skipped/unsupported only with gapReason. Use generated only when file+symbol already exist.\n\nevidenceGaps: structured gaps for explicit AC-* that cannot be mapped to a case.\n\nDo NOT invent coverage percentages. Optional coverageSummary must match deterministic counts (gate recomputes/validates).\n\nNo secrets or credential-shaped fields."
+    },
+    {
+      "id": "backend-test-case-manifest-shell",
+      "depends_on": [
+        "emit-backend-case-manifest-pi"
+      ],
+      "complexity": "LOW",
+      "executor": "shell",
+      "role": "verifier",
+      "writePolicy": "read-only",
+      "allowedPaths": [
+        "**"
+      ],
+      "forbiddenPaths": [
+        ".harness/**",
+        "artifacts/**"
+      ],
+      "outputContract": "Validated run-owned Backend Test Case Manifest v1 at contracts/backend-test-case-manifest.json (schemaId backend-test-case-manifest-v1) with deterministic AC coverage.",
+      "subtask_prompt": "Materialize and validate Backend Test Case Manifest v1; fail closed on duplicate IDs, unknown AC, missing AC coverage without gap, or skipped without gapReason.",
+      "shell": {
+        "commands": [],
+        "jsonArtifactGate": {
+          "fromNodeId": "emit-backend-case-manifest-pi",
+          "schemaId": "backend-test-case-manifest-v1",
+          "artifactName": "backend-test-case-manifest.json",
+          "outputDir": "contracts"
+        },
+        "cwd": ".",
+        "timeoutMs": 60000
+      }
+    },
+    {
+      "id": "review-backend-cases-pi",
+      "depends_on": [
+        "backend-test-case-manifest-shell",
+        "backend-test-analysis-contract-shell"
+      ],
       "complexity": "HIGH",
       "executor": "pi",
       "role": "reviewer",
@@ -225,13 +361,14 @@
     {
       "id": "generate-backend-pytest-pi",
       "depends_on": [
-        "review-backend-cases-gate-shell"
+        "review-backend-cases-gate-shell",
+        "backend-test-execution-contract-shell"
       ],
       "complexity": "HIGH",
       "executor": "pi",
       "role": "implementer",
-      "toolProfile": "write",
       "writePolicy": "exclusive",
+      "toolProfile": "write",
       "writeSet": [
         "testcase/**/test_*.py",
         "testcase/**/helpers/**",
@@ -245,13 +382,40 @@
         ".harness/**",
         "artifacts/**"
       ],
-      "outputContract": "Pytest test files under testcase/ with 1:1 mapping to functional test case IDs; optional helpers/factories under testcase/**/helpers|factories. Summary lists generated files, test function count, and any skipped cases with reasons.",
       "subtask_prompt_markdown": "./backend-test-dag.generate-pytest.prompt.md"
     },
+    {
+      "id": "backend-test-traceability-gate-shell",
+      "depends_on": [
+        "generate-backend-pytest-pi",
+        "backend-test-case-manifest-shell"
+      ],
+      "complexity": "LOW",
+      "executor": "shell",
+      "role": "verifier",
+      "writePolicy": "read-only",
+      "allowedPaths": [
+        "**"
+      ],
+      "forbiddenPaths": [
+        ".harness/**",
+        "artifacts/**"
+      ],
+      "outputContract": "Deterministic traceability: generated cases have real file/symbol; skipped/unsupported have gapReason; convention symbols scanned under testcase/**/test_*.py.",
+      "subtask_prompt": "Fail closed when generated automation claims do not resolve to workspace pytest symbols, or skip/unsupported lacks gapReason.",
+      "shell": {
+        "commands": [
+          "backend-test-traceability-gate"
+        ],
+        "cwd": ".",
+        "timeoutMs": 60000
+      }
+    },
     {
       "id": "execute-backend-pytest-shell",
       "depends_on": [
-        "generate-backend-pytest-pi"
+        "backend-test-traceability-gate-shell",
+        "backend-test-execution-contract-shell"
       ],
       "complexity": "LOW",
       "executor": "shell",
@@ -264,11 +428,11 @@
         ".harness/**",
         "artifacts/**"
       ],
-      "outputContract": "Archived pytest stdout/stderr with exit codes; JUnit XML is runner-owned evidence at $HARNESS_DAG_RUN_DIR/reports/backend-test-junit.xml. Must not modify worktree files, testcase sources, production code, or assertions.",
-      "subtask_prompt": "Run pytest for the backend test suite; write JUnit evidence only under the current HARNESS_DAG_RUN_DIR/reports/.",
+      "outputContract": "Archived pytest stdout/stderr; raw pytestExitCode side-channel + JUnit at $HARNESS_DAG_RUN_DIR/reports/**. Exit 0/1 with non-empty JUnit finishes the node so parse/classify/retrospect can run; assertion failures remain recorded in exit file.",
+      "subtask_prompt": "Run pytest for the backend test suite; write JUnit + pytestExitCode evidence only under the current HARNESS_DAG_RUN_DIR/reports/.",
       "shell": {
         "commands": [
-          "test -n \"${HARNESS_DAG_RUN_DIR:-}\" || { echo \"missing HARNESS_DAG_RUN_DIR for backend pytest report\" >&2; exit 2; }; REPORT=\"${HARNESS_DAG_RUN_DIR}/reports/backend-test-junit.xml\"; mkdir -p \"$(dirname \"${REPORT}\")\"; PYTHONDONTWRITEBYTECODE=1 python -m pytest testcase/ -v -p no:cacheprovider --junitxml=\"${REPORT}\"; STATUS=$?; printf \"JUnit report: %s\\n\" \"${REPORT}\"; exit \"${STATUS}\""
+          "test -n \"${HARNESS_DAG_RUN_DIR:-}\" || { echo \"missing HARNESS_DAG_RUN_DIR for backend pytest preflight\" >&2; exit 2; }; CONTRACT=\"${HARNESS_DAG_RUN_DIR}/contracts/backend-test-execution.json\"; test -f \"${CONTRACT}\" || { echo \"missing backend-test execution contract: ${CONTRACT}\" >&2; exit 2; }; node -e 'const fs=require(\"fs\");const path=require(\"path\");const contractPath=process.argv[1];const contract=JSON.parse(fs.readFileSync(contractPath,\"utf8\"));const expected=\"testcase\";const errors=[];if(contract.framework!==\"pytest\") errors.push(\"framework must be pytest\");const testRoot=String(contract.testRoot||\"\");if(!testRoot||testRoot.includes(\"..\")||path.isAbsolute(testRoot)) errors.push(\"unsafe testRoot\");if(testRoot.replace(/\\/+$/,\"\")!==expected.replace(/\\/+$/,\"\")) errors.push(\"testRoot mismatch vs frozen command: \"+testRoot+\" !== \"+expected);const rootAbs=path.resolve(process.cwd(),testRoot);if(!fs.existsSync(rootAbs)) errors.push(\"testRoot does not exist: \"+testRoot);if(Array.isArray(contract.evidenceGaps)&&contract.evidenceGaps.length) errors.push(\"evidenceGaps present: \"+contract.evidenceGaps.length);if(contract.targetMode===\"in-process\"&&!(Array.isArray(contract.existingFixtures)&&contract.existingFixtures.length)) errors.push(\"in-process requires existingFixtures\");for (const name of (contract.requiredEnvNames||[])) { if(!process.env[name]) errors.push(\"required env missing: \"+name); }if(contract.targetMode===\"external-running-service\"){ const n=contract.baseUrlEnvName; if(!n||!process.env[n]) errors.push(\"external base URL env missing: \"+String(n||\"<empty>\")); }if(contract.targetMode===\"managed-command\" && !(contract.managedCommand&&contract.managedCommand.sourceRef)) errors.push(\"managed-command requires sourceRef evidence\");if(errors.length){ console.error(errors.join(\"; \")); process.exit(2);} console.log(\"backend-test preflight ok: framework=pytest testRoot=\"+testRoot+\" targetMode=\"+contract.targetMode);' \"${CONTRACT}\"; REPORT=\"${HARNESS_DAG_RUN_DIR}/reports/backend-test-junit.xml\"; EXIT_FILE=\"${HARNESS_DAG_RUN_DIR}/reports/backend-test-pytest-exit.txt\"; mkdir -p \"$(dirname \"${REPORT}\")\"; PYTHONDONTWRITEBYTECODE=1 python -m pytest testcase/ -v -p no:cacheprovider --junitxml=\"${REPORT}\"; STATUS=$?; printf \"%s\" \"${STATUS}\" > \"${EXIT_FILE}\"; printf \"JUnit report: %s\\n\" \"${REPORT}\"; printf \"pytestExitCode=%s\\n\" \"${STATUS}\"; if { [ \"${STATUS}\" -eq 0 ] || [ \"${STATUS}\" -eq 1 ]; } && [ -s \"${REPORT}\" ]; then exit 0; fi; exit \"${STATUS}\""
         ],
         "verifyEvidence": {
           "phase": "final",
@@ -276,7 +440,7 @@
           "commandSource": "inline",
           "commandCount": 1,
           "commandLabels": [
-            "backend pytest execution"
+            "test -n \"${HARNESS_DAG_RUN_DIR:-}\" || { echo \"missing HARNESS_DAG_RUN_DIR for backend pytest preflight\" >&2; exit 2; }; CONTRACT=\"${HARNESS_DAG_RUN_DIR}/contracts/backend-test-execution.json\"; test -f \"${CONTRACT}\" || { echo \"missing backend-test execution contract: ${CONTRACT}\" >&2; exit 2; }; node -e 'const fs=require(\"fs\");const path=require(\"path\");const contractPath=process.argv[1];const contract=JSON.parse(fs.readFileSync(contractPath,\"utf8\"));const expected=\"testcase\";const errors=[];if(contract.framework!==\"pytest\") errors.push(\"framework must be pytest\");const testRoot=String(contract.testRoot||\"\");if(!testRoot||testRoot.includes(\"..\")||path.isAbsolute(testRoot)) errors.push(\"unsafe testRoot\");if(testRoot.replace(/\\/+$/,\"\")!==expected.replace(/\\/+$/,\"\")) errors.push(\"testRoot mismatch vs frozen command: \"+testRoot+\" !== \"+expected);const rootAbs=path.resolve(process.cwd(),testRoot);if(!fs.existsSync(rootAbs)) errors.push(\"testRoot does not exist: \"+testRoot);if(Array.isArray(contract.evidenceGaps)&&contract.evidenceGaps.length) errors.push(\"evidenceGaps present: \"+contract.evidenceGaps.length);if(contract.targetMode===\"in-process\"&&!(Array.isArray(contract.existingFixtures)&&contract.existingFixtures.length)) errors.push(\"in-process requires existingFixtures\");for (const name of (contract.requiredEnvNames||[])) { if(!process.env[name]) errors.push(\"required env missing: \"+name); }if(contract.targetMode===\"external-running-service\"){ const n=contract.baseUrlEnvName; if(!n||!process.env[n]) errors.push(\"external base URL env missing: \"+String(n||\"<empty>\")); }if(contract.targetMode===\"managed-command\" && !(contract.managedCommand&&contract.managedCommand.sourceRef)) errors.push(\"managed-command requires sourceRef evidence\");if(errors.length){ console.error(errors.join(\"; \")); process.exit(2);} console.log(\"backend-test preflight ok: framework=pytest testRoot=\"+testRoot+\" targetMode=\"+contract.targetMode);' \"${CONTRACT}\"; REPORT=\"${HARNESS_DAG_RUN_DIR}/reports/backend-test-junit.xml\"; EXIT_FILE=\"${HARNESS_DAG_RUN_DIR}/reports/backend-test-pytest-exit.txt\"; mkdir -p \"$(dirname \"${REPORT}\")\"; PYTHONDONTWRITEBYTECODE=1 python -m pytest testcase/ -v -p no:cacheprovider --junitxml=\"${REPORT}\"; STATUS=$?; printf \"%s\" \"${STATUS}\" > \"${EXIT_FILE}\"; printf \"JUnit report: %s\\n\" \"${REPORT}\"; printf \"pytestExitCode=%s\\n\" \"${STATUS}\"; if { [ \"${STATUS}\" -eq 0 ] || [ \"${STATUS}\" -eq 1 ]; } && [ -s \"${REPORT}\" ]; then exit 0; fi; exit \"${STATUS}\""
           ],
           "finalFullRequired": true
         },
@@ -285,15 +449,64 @@
       }
     },
     {
-      "id": "test-retrospect-pi",
+      "id": "parse-backend-test-result-shell",
       "depends_on": [
         "execute-backend-pytest-shell"
       ],
+      "complexity": "LOW",
+      "executor": "shell",
+      "role": "verifier",
+      "writePolicy": "read-only",
+      "allowedPaths": [
+        "**"
+      ],
+      "forbiddenPaths": [
+        ".harness/**",
+        "artifacts/**"
+      ],
+      "outputContract": "Validated run-owned Backend Test Result v1 at contracts/backend-test-result.json (schemaId backend-test-result-v1) with outcome/counts/failures from deterministic JUnit parse.",
+      "subtask_prompt": "Materialize Backend Test Result v1 from JUnit + pytestExitCode under the current DAG run (fail-closed on missing/corrupt report).",
+      "shell": {
+        "commands": [],
+        "jsonArtifactGate": {
+          "fromNodeId": "execute-backend-pytest-shell",
+          "schemaId": "backend-test-result-v1",
+          "artifactName": "backend-test-result.json",
+          "outputDir": "contracts"
+        },
+        "cwd": ".",
+        "timeoutMs": 60000
+      }
+    },
+    {
+      "id": "classify-backend-test-result-pi",
+      "depends_on": [
+        "parse-backend-test-result-shell"
+      ],
+      "complexity": "MED",
+      "executor": "pi",
+      "role": "reviewer",
+      "writePolicy": "read-only",
+      "allowedPaths": [
+        "**"
+      ],
+      "forbiddenPaths": [
+        ".harness/**",
+        "artifacts/**"
+      ],
+      "outputContract": "Pure JSON classification: category in {ProductBug,TestBug,EnvFailure,ContractMismatch,FlakyTest,Unknown}, evidence[], confidence (capped), notes. No file writes.",
+      "subtask_prompt_markdown": "./backend-test-dag.classify.prompt.md"
+    },
+    {
+      "id": "test-retrospect-pi",
+      "depends_on": [
+        "classify-backend-test-result-pi"
+      ],
       "complexity": "MED",
       "executor": "pi",
       "role": "closeout",
-      "toolProfile": "write",
       "writePolicy": "exclusive",
+      "toolProfile": "write",
       "writeSet": [
         "docs/test-reports/**"
       ],
@@ -304,8 +517,43 @@
         ".harness/**",
         "artifacts/**"
       ],
-      "outputContract": "Markdown retrospective report under docs/test-reports/ with coverage summary, review findings, pytest results, and maturity rating (A/B/C/D).",
       "subtask_prompt_markdown": "./backend-test-dag.retrospect.prompt.md"
+    },
+    {
+      "id": "backend-test-outcome-gate-shell",
+      "depends_on": [
+        "test-retrospect-pi"
+      ],
+      "complexity": "LOW",
+      "executor": "shell",
+      "role": "verifier",
+      "writePolicy": "read-only",
+      "allowedPaths": [
+        "**"
+      ],
+      "forbiddenPaths": [
+        ".harness/**",
+        "artifacts/**"
+      ],
+      "outputContract": "Shell exit 0 only when Result v1 outcome=passed with failed=0 and error=0; non-zero otherwise. Ignores retrospective Markdown.",
+      "subtask_prompt": "Gate the backend-test DAG on run-owned Result v1 shell facts only (not retrospective prose).",
+      "shell": {
+        "commands": [
+          "test -n \"${HARNESS_DAG_RUN_DIR:-}\" || { echo \"missing HARNESS_DAG_RUN_DIR for backend-test outcome gate\" >&2; exit 2; }; RESULT=\"${HARNESS_DAG_RUN_DIR}/contracts/backend-test-result.json\"; test -f \"${RESULT}\" || { echo \"missing backend-test result: ${RESULT}\" >&2; exit 2; }; node -e 'const fs=require(\"fs\");const r=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\"));const outcome=String(r.outcome||\"\");const ok=outcome===\"passed\"&&Number(r.failed||0)===0&&Number(r.error||0)===0;console.log(\"backend-test outcome=\"+outcome+\" passed=\"+r.passed+\" failed=\"+r.failed+\" error=\"+r.error+\" executionStatus=\"+r.executionStatus);if(!ok){process.exit(1);}' \"${RESULT}\""
+        ],
+        "verifyEvidence": {
+          "phase": "final",
+          "quota": "full",
+          "commandSource": "inline",
+          "commandCount": 1,
+          "commandLabels": [
+            "test -n \"${HARNESS_DAG_RUN_DIR:-}\" || { echo \"missing HARNESS_DAG_RUN_DIR for backend-test outcome gate\" >&2; exit 2; }; RESULT=\"${HARNESS_DAG_RUN_DIR}/contracts/backend-test-result.json\"; test -f \"${RESULT}\" || { echo \"missing backend-test result: ${RESULT}\" >&2; exit 2; }; node -e 'const fs=require(\"fs\");const r=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\"));const outcome=String(r.outcome||\"\");const ok=outcome===\"passed\"&&Number(r.failed||0)===0&&Number(r.error||0)===0;console.log(\"backend-test outcome=\"+outcome+\" passed=\"+r.passed+\" failed=\"+r.failed+\" error=\"+r.error+\" executionStatus=\"+r.executionStatus);if(!ok){process.exit(1);}' \"${RESULT}\""
+          ],
+          "finalFullRequired": true
+        },
+        "cwd": ".",
+        "timeoutMs": 60000
+      }
     }
   ]
 }

package/docs/templates/backend-test-dag.retrospect.prompt.md

@@ -2,7 +2,7 @@
 
 ## Purpose
 
-Use this prompt for a **test retrospective** node: `executor: "pi"`, `role: "closeout"`, `toolProfile: "write"`, `writePolicy: "exclusive"`. The closeout agent reads upstream review reports and pytest execution results, then generates a retrospective report with an objective maturity rating.
+Use this prompt for a **test retrospective** node: `executor: "pi"`, `role: "closeout"`, `toolProfile: "write"`, `writePolicy: "exclusive"`. The closeout agent reads **Backend Test Result v1** (and classification), then generates a retrospective report with an objective maturity rating.
 
 Do **not** create a new executor type. This is a standard `executor: pi` writer node.
 
@@ -11,7 +11,7 @@ Do **not** create a new executor type. This is a standard `executor: pi` writer
 ```json
 {
   "id": "test-retrospect-pi",
-  "depends_on": ["execute-backend-pytest-shell"],
+  "depends_on": ["classify-backend-test-result-pi"],
   "complexity": "MED",
   "executor": "pi",
   "role": "closeout",
@@ -20,7 +20,7 @@ Do **not** create a new executor type. This is a standard `executor: pi` writer
   "writeSet": ["docs/test-reports/**"],
   "allowedPaths": ["docs/test-reports/**"],
   "forbiddenPaths": [".harness/**", "artifacts/**"],
-  "outputContract": "Markdown retrospective report under docs/test-reports/ with coverage summary, review findings, pytest results, and maturity rating (A/B/C/D).",
+  "outputContract": "Markdown retrospective report under docs/test-reports/ with coverage summary, review findings, Result v1 stats, classification, and maturity rating (A/B/C/D).",
   "subtask_prompt_markdown": "./backend-test-dag.retrospect.prompt.md"
 }
 ```
@@ -29,7 +29,9 @@ Do **not** create a new executor type. This is a standard `executor: pi` writer
 
 You are the Backend Test DAG **test retrospective** agent.
 
-Your job is to read upstream outputs (review report + pytest results) and generate a retrospective report with a maturity rating. Write the report under `docs/test-reports/` only. Stay within `writeSet`. Do not write root `artifacts/**`.
+Your job is to read upstream Result v1 + classification (+ review report) and generate a retrospective report with a maturity rating. Write the report under `docs/test-reports/` only. Stay within `writeSet`. Do not write root `artifacts/**`.
+
+This node runs on **both pass and assertion-fail** paths (after parse + classify). Final task success is decided later by `backend-test-outcome-gate-shell` using Result v1 shell facts only — **never** rewrite a failed result as passed in this report.
 
 ### Output Steps (do in order)
 
@@ -38,28 +40,40 @@ Your job is to read upstream outputs (review report + pytest results) and genera
 
 ### Inputs
 
-1. **Review report** — `review-backend-cases-pi` output (VERDICT, findings, coverage assessment).
-2. **Pytest output** — `execute-backend-pytest-shell` stdout/stderr and exit code.
-3. **Machine-readable report** — `$HARNESS_DAG_RUN_DIR/reports/backend-test-junit.xml` (runner-owned evidence; use the path reported by `execute-backend-pytest-shell`).
+1. **Result v1 (authoritative stats)** — `$HARNESS_DAG_RUN_DIR/contracts/backend-test-result.json`
+   Use `passed` / `failed` / `error` / `skipped` / `outcome` / `failures[]` / `pytestExitCode` only from this artifact.
+2. **Case Manifest v1 (authoritative AC coverage)** — `$HARNESS_DAG_RUN_DIR/contracts/backend-test-case-manifest.json`
+   Use `coverageSummary.acCoverageRatio`, `coveredAcCount`, `explicitAcCount`, case counts only from this artifact.
+3. **Classification** — `classify-backend-test-result-pi` JSON (`category`, `confidence`, `evidence`). Interpretive only; does not override outcome.
+4. **Review report** — `review-backend-cases-pi` output (VERDICT, findings, coverage assessment).
+5. Optional secondary: execute stdout markers / JUnit path (do not re-parse logs for counts when Result v1 exists).
 
 Do NOT re-read source documents. Use upstream outputs only.
 
+### Stats authority
+
+- Pass rate = `passed / (passed + failed + error)` when denominator > 0 (skipped excluded from denominator unless Result documents otherwise) — **Result v1 only**.
+- AC coverage = `coverageSummary.acCoverageRatio` from Case Manifest v1 only (do **not** recompute or invent percentages).
+- Failed case table rows must match `failures[]` from Result v1.
+- If Result v1 `outcome` is not `passed`, the retrospective **must not** claim overall success.
+
 ### Maturity Rating Criteria
 
-| Rating | Coverage | Pass Rate | Review Findings |
-|--------|----------|-----------|-----------------|
-| **A** | 100% acceptance criteria covered | 100% pytest pass | No Critical or Important findings |
-| **B** | ≥80% acceptance criteria covered | ≥90% pytest pass | Only Informational findings |
-| **C** | ≥60% acceptance criteria covered | ≥70% pytest pass | No Critical findings (Important allowed) |
+| Rating | Coverage (manifest) | Pass Rate (Result v1) | Review Findings |
+|--------|---------------------|------------------------|-----------------|
+| **A** | `acCoverageRatio` = 1 | 100% pytest pass | No Critical or Important findings |
+| **B** | `acCoverageRatio` ≥ 0.8 | ≥90% pytest pass | Only Informational findings |
+| **C** | `acCoverageRatio` ≥ 0.6 | ≥70% pytest pass | No Critical findings (Important allowed) |
 | **D** | Below C thresholds | Below C thresholds | Or any Critical finding unresolved |
 
 #### Rating Rules
 
-- **Coverage** = (acceptance criteria with ≥1 covering test case) / (total acceptance criteria) × 100%
-- **Pass rate** = (passed pytest functions) / (total non-skipped pytest functions) × 100%
+- **Coverage** from Case Manifest `coverageSummary` only (deterministic gate product).
+- **Pass rate** from Result v1 only (not guessed from logs).
 - If `review-backend-cases-pi` returned `VERDICT: request-revision` and revision was not completed, cap at **D**.
-- If pytest exit code is non-zero and >30% tests failed, cap at **D** regardless of coverage.
-- Skipped tests (`@pytest.mark.skip`) count as "not covered" for pass rate but not as failures.
+- If Result v1 shows >30% failed+error among executed tests, cap at **D** regardless of coverage.
+- Skipped tests count as "not covered" for pass rate but not as failures.
+- Collection/command/report errors → cap at **D** and record classification (not ProductBug by default).
 
 ### Report Structure
 
@@ -71,6 +85,8 @@ Write the report as a Markdown file named `backend-test-retrospect-<date>.md` un
 **Date:** <YYYY-MM-DD>
 **Task:** <task-id>
 **Maturity Rating:** <A|B|C|D>
+**Result outcome:** <from Result v1>
+**Classification:** <from classify JSON>
 
 ## 1. Test Coverage Summary
 
@@ -79,43 +95,41 @@ Write the report as a Markdown file named `backend-test-retrospect-<date>.md` un
 | Total acceptance criteria | N |
 | Covered by test cases | N (X%) |
 | Total functional test cases | N |
-| Positive path cases | N |
-| Negative path cases | N |
-| Boundary cases | N |
 
-## 2. Automation Results
+## 2. Automation Results (from Result v1)
 
 | Metric | Value |
 |--------|-------|
-| Total pytest functions | N |
+| Total tests | N |
 | Passed | N |
 | Failed | N |
+| Error | N |
 | Skipped | N |
 | Pass rate | X% |
 | Pytest exit code | N |
+| Outcome | … |
+| Execution status | … |
 
 ### Failed Test Analysis
 
-| Test Case ID | Function | Failure Reason | Root Cause |
-|--------------|----------|----------------|------------|
-| ... | ... | ... | ... |
+| Test Case / Function | Message (truncated) | Classification |
+|----------------------|-------------------|----------------|
+| ... | ... | ... |
 
 ## 3. Review Findings
 
 | Severity | Finding | Status |
 |----------|---------|--------|
-| Critical | ... | Resolved / Unresolved |
-| Important | ... | Resolved / Unresolved |
-| Informational | ... | Resolved / Unresolved |
+| … | … | … |
 
 ## 4. Maturity Rating Rationale
 
-Explain which threshold was met or missed, and why the specific rating was assigned.
+Explain which threshold was met or missed.
 
 ## 5. Recommendations
 
-- Actionable items for improving the rating in the next iteration.
-- Specific gaps to close (uncovered criteria, flaky tests, missing negative paths).
+- Actionable items for the next iteration.
+- Do not propose changing production code solely to greenwash tests.
 ```
 
 ### Output Shape (after rating line)

package/docs/templates/backend-test-dag.review-cases.prompt.md

@@ -11,7 +11,7 @@ Do **not** create `executor: reviewer`. Reviewer is a **role** on `executor: pi`
 ```json
 {
   "id": "review-backend-cases-pi",
-  "depends_on": ["generate-backend-functional-cases-pi", "backend-test-analysis-contract-shell"],
+  "depends_on": ["backend-test-case-manifest-shell", "backend-test-analysis-contract-shell"],
   "complexity": "HIGH",
   "executor": "pi",
   "role": "reviewer",
@@ -41,9 +41,10 @@ No preamble, heading, or blank lines before the verdict line.
 ### Inputs to Review
 
 1. **Acceptance criteria / analysis** — from the validated Backend Test Analysis v1 artifact materialized by `backend-test-analysis-contract-shell` (`contracts/backend-test-analysis.json` under the current DAG run). Do not treat free-form Markdown from `analyze-inputs-pi` as the contract.
-2. **Generated test cases** — files under `testcase/md/`.
+2. **Case Manifest v1** — `contracts/backend-test-case-manifest.json` (schemaId `backend-test-case-manifest-v1`). Prefer `coverageSummary` and caseId↔acIds from this artifact; do not invent coverage percentages.
+3. **Generated test cases** — files under `testcase/md/`.
 
-Do NOT re-read source documents. Use the validated analysis artifact and generated cases only.
+Do NOT re-read source documents. Use the validated analysis artifact, case manifest, and generated cases only.
 
 ### Review Checklist
 
@@ -54,7 +55,8 @@ Do NOT re-read source documents. Use the validated analysis artifact and generat
 | **Negative path coverage** | Error/exception scenarios (invalid input, not found, state violations) | Important |
 | **Boundary conditions** | Edge cases (empty input, max length, edge values) | Important |
 | **State transitions** | Illegal state changes covered | Important |
-| **Requirement traceability** | Each acceptance criterion (AC-xxx) maps to at least one test case ID | Critical |
+| **Requirement traceability** | Each acceptance criterion (AC-xxx) maps to at least one test case ID (manifest coverageSummary or evidenceGaps) | Critical |
+| **Manifest consistency** | Markdown cases align with Case Manifest v1 caseId/acIds | Critical |
 | **Case structure** | Each case has: ID, Title, Precondition, Steps, Expected Result | Important |
 | **No duplicate IDs** | All test case IDs are unique across files | Critical |