@tea-agent/loop-agent 0.13.0-alpha.0 → 0.13.0

package/docs/templates/backend-test-case-manifest.schema.json

@@ -0,0 +1,190 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://tea-agent.dev/schemas/backend-test-case-manifest-v1.json",
+  "title": "Backend Test Case Manifest v1",
+  "description": "Run-owned AC → BE-* case → automation mapping. Coverage ratios are deterministic gate outputs; models must not invent percentages. Conditional required fields mirror Zod: generated requires file+symbol; skipped|unsupported requires gapReason; each evidence gap requires acId and/or caseId.",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schemaVersion", "sourceBinding", "cases", "evidenceGaps"],
+  "properties": {
+    "schemaVersion": { "const": 1 },
+    "sourceBinding": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "taskId",
+        "requirementPath",
+        "requirementSha256",
+        "referencePaths",
+        "requirementIds"
+      ],
+      "properties": {
+        "taskId": { "type": "string", "minLength": 1 },
+        "requirementPath": { "type": "string", "minLength": 1 },
+        "requirementSha256": {
+          "type": "string",
+          "pattern": "^[a-f0-9]{64}$"
+        },
+        "referencePaths": {
+          "type": "array",
+          "items": { "type": "string", "minLength": 1 }
+        },
+        "requirementIds": {
+          "type": "array",
+          "items": {
+            "type": "string",
+            "pattern": "^(?:REQ|BR|AC)-[A-Z0-9]+(?:-[A-Z0-9]+)*$"
+          }
+        }
+      }
+    },
+    "cases": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": [
+          "caseId",
+          "acIds",
+          "title",
+          "category",
+          "automationStatus"
+        ],
+        "properties": {
+          "caseId": {
+            "type": "string",
+            "pattern": "^BE-[A-Z0-9]+(?:-[A-Z0-9]+)*-\\d{3}$"
+          },
+          "acIds": {
+            "type": "array",
+            "minItems": 1,
+            "items": {
+              "type": "string",
+              "pattern": "^AC-[A-Z0-9]+(?:-[A-Z0-9]+)*$"
+            }
+          },
+          "title": { "type": "string", "minLength": 1 },
+          "category": {
+            "type": "string",
+            "enum": [
+              "positive",
+              "negative",
+              "boundary",
+              "state-transition",
+              "auth",
+              "timeout",
+              "concurrency",
+              "other"
+            ]
+          },
+          "automationStatus": {
+            "type": "string",
+            "enum": ["planned", "generated", "skipped", "unsupported"]
+          },
+          "endpointRef": { "type": "string", "minLength": 1 },
+          "ruleRefs": {
+            "type": "array",
+            "items": { "type": "string", "minLength": 1 }
+          },
+          "file": {
+            "type": "string",
+            "minLength": 1,
+            "description": "Repo-relative posix path required when automationStatus=generated"
+          },
+          "symbol": {
+            "type": "string",
+            "minLength": 1,
+            "description": "pytest function name required when automationStatus=generated; must bind to caseId"
+          },
+          "gapReason": {
+            "type": "string",
+            "minLength": 1,
+            "description": "Required when automationStatus is skipped or unsupported"
+          },
+          "evidenceRef": { "type": "string", "minLength": 1 }
+        },
+        "allOf": [
+          {
+            "if": {
+              "properties": {
+                "automationStatus": { "const": "generated" }
+              },
+              "required": ["automationStatus"]
+            },
+            "then": {
+              "required": ["file", "symbol"]
+            }
+          },
+          {
+            "if": {
+              "properties": {
+                "automationStatus": {
+                  "enum": ["skipped", "unsupported"]
+                }
+              },
+              "required": ["automationStatus"]
+            },
+            "then": {
+              "required": ["gapReason"]
+            }
+          }
+        ]
+      }
+    },
+    "evidenceGaps": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["description"],
+        "properties": {
+          "acId": {
+            "type": "string",
+            "pattern": "^AC-[A-Z0-9]+(?:-[A-Z0-9]+)*$"
+          },
+          "caseId": {
+            "type": "string",
+            "pattern": "^BE-[A-Z0-9]+(?:-[A-Z0-9]+)*-\\d{3}$"
+          },
+          "description": { "type": "string", "minLength": 1 },
+          "evidenceRef": { "type": "string", "minLength": 1 }
+        },
+        "anyOf": [
+          { "required": ["acId"] },
+          { "required": ["caseId"] }
+        ]
+      }
+    },
+    "coverageSummary": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "explicitAcCount",
+        "coveredAcCount",
+        "gappedAcCount",
+        "caseCount",
+        "generatedCount",
+        "plannedCount",
+        "skippedCount",
+        "unsupportedCount",
+        "acCoverageRatio"
+      ],
+      "properties": {
+        "explicitAcCount": { "type": "integer", "minimum": 0 },
+        "coveredAcCount": { "type": "integer", "minimum": 0 },
+        "gappedAcCount": { "type": "integer", "minimum": 0 },
+        "caseCount": { "type": "integer", "minimum": 0 },
+        "generatedCount": { "type": "integer", "minimum": 0 },
+        "plannedCount": { "type": "integer", "minimum": 0 },
+        "skippedCount": { "type": "integer", "minimum": 0 },
+        "unsupportedCount": { "type": "integer", "minimum": 0 },
+        "acCoverageRatio": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1,
+          "description": "Deterministic coveredAcCount/explicitAcCount (or 1 when no explicit AC)"
+        }
+      }
+    }
+  }
+}

package/docs/templates/backend-test-dag.classify.prompt.md

@@ -0,0 +1,75 @@
+# Backend Test DAG Classify Prompt Template
+
+## Purpose
+
+Use this prompt for a **read-only classification** node: `executor: "pi"`, `role: "reviewer"`, `writePolicy: "read-only"`. The agent reads the run-owned Backend Test Result v1 artifact and returns structured failure classification JSON.
+
+Do **not** create a new executor type. Do **not** write repository files.
+
+## Recommended DAG Node Shape
+
+```json
+{
+  "id": "classify-backend-test-result-pi",
+  "depends_on": ["parse-backend-test-result-shell"],
+  "complexity": "MED",
+  "executor": "pi",
+  "role": "reviewer",
+  "writePolicy": "read-only",
+  "allowedPaths": ["**"],
+  "forbiddenPaths": [".harness/**", "artifacts/**"],
+  "outputContract": "Pure JSON classification: category in {ProductBug,TestBug,EnvFailure,ContractMismatch,FlakyTest,Unknown}, evidence[], confidence (capped), notes. No file writes.",
+  "subtask_prompt_markdown": "./backend-test-dag.classify.prompt.md"
+}
+```
+
+## Prompt Body
+
+You are the Backend Test DAG **result classifier** agent.
+
+Your job is to classify the structured Backend Test Result v1 produced by `parse-backend-test-result-shell`. Return **exactly one JSON object**. Prefer pure JSON; a single fenced `json` block is tolerated; no trailing prose. Read-only: do not modify code, docs, artifacts, or repository files.
+
+### Inputs (authoritative)
+
+1. **Result v1** — `$HARNESS_DAG_RUN_DIR/contracts/backend-test-result.json` (schemaId `backend-test-result-v1`).
+2. Optional: execute-node stdout markers (`pytestExitCode=…`, `JUnit report: …`) as secondary evidence only.
+
+Do **not** invent pass rates or failure lists from raw logs when Result v1 is present. Counts and `failures[]` come from the result artifact only.
+
+### Output JSON shape
+
+```json
+{
+  "schemaVersion": 1,
+  "category": "ProductBug",
+  "confidence": 0.0,
+  "evidence": ["result.outcome=completed-with-failures", "failures[0].name=…"],
+  "notes": "short rationale",
+  "forbiddenCategoriesHonored": ["FlakyTest"]
+}
+```
+
+### Categories
+
+| Category | When |
+|----------|------|
+| **ProductBug** | Assertion failures that indicate implementation/API behavior mismatch (only when collection/command/report are healthy). |
+| **TestBug** | Broken test code, wrong expectations, bad fixtures, or collection/import errors clearly in tests. |
+| **EnvFailure** | Missing env, service down, tooling/runtime failure, command-error. |
+| **ContractMismatch** | Execution/analysis contract assumptions violated (wrong testRoot/mode, missing readiness). |
+| **FlakyTest** | **Only** with multi-run historical evidence of intermittent pass/fail. |
+| **Unknown** | Insufficient evidence. |
+
+### Hard constraints (MUST)
+
+1. **Single-run failure MUST NOT use `FlakyTest`.** Prefer `Unknown`, `TestBug`, or `ProductBug`.
+2. If `executionStatus` or `outcome` is `collection-error`, `command-error`, or `report-error`, **MUST NOT** use `ProductBug`. Prefer `EnvFailure`, `TestBug`, or `Unknown`.
+3. If `outcome=passed` with `failed=0` and `error=0`, set `category` to `Unknown` (or omit product diagnosis) and note all-pass; do not invent bugs.
+4. `confidence` caps: ≤ `0.75` for assertion failures; ≤ `0.6` for env/collection/command/report errors; `1.0` only for all-pass with no issues.
+5. `evidence[]` must cite concrete result fields (`outcome`, `executionStatus`, `failed`, `failures[].name`, `pytestExitCode`).
+
+### Non-goals
+
+- Do not rewrite Result v1.
+- Do not decide final DAG success/failure (that is `backend-test-outcome-gate-shell`).
+- Do not implement M3 case manifest / Task Pool auto follow-up.

package/docs/templates/backend-test-dag.generate-pytest.prompt.md

@@ -11,16 +11,16 @@ Do **not** create a new executor type. This is a standard `executor: pi` writer
 ```json
 {
   "id": "generate-backend-pytest-pi",
-  "depends_on": ["review-backend-cases-gate-shell"],
+  "depends_on": ["review-backend-cases-gate-shell", "backend-test-execution-contract-shell"],
   "complexity": "HIGH",
   "executor": "pi",
   "role": "implementer",
   "toolProfile": "write",
   "writePolicy": "exclusive",
-  "writeSet": ["testcase/**/test_*.py"],
-  "allowedPaths": ["testcase/**/test_*.py"],
+  "writeSet": ["testcase/**/test_*.py", "testcase/**/helpers/**", "testcase/**/factories/**"],
+  "allowedPaths": ["**"],
   "forbiddenPaths": [".harness/**", "artifacts/**"],
-  "outputContract": "Pytest test files under tests/backend/ with 1:1 mapping to functional test case IDs. Summary lists generated files, test function count, and any skipped cases with reasons.",
+  "outputContract": "Pytest test files under testcase/ with 1:1 mapping to functional test case IDs; optional helpers/factories. Summary lists generated files, test function count, and any skipped cases with reasons.",
   "subtask_prompt_markdown": "./backend-test-dag.generate-pytest.prompt.md"
 }
 ```
@@ -38,10 +38,12 @@ Your job is to convert reviewed test cases under `testcase/md/` into pytest auto
 
 ### Inputs
 
-1. **Reviewed test cases** — files under `testcase/md/` (approved by `review-backend-cases-pi`).
-2. **Target project conventions** — read `conftest.py`, `pytest.ini` / `pyproject.toml` to understand conventions, but do NOT modify them.
+1. **Reviewed test cases** — files under `testcase/md/` (approved by `review-backend-cases-pi` / `review-backend-cases-gate-shell`).
+2. **Validated Backend Test Analysis v1** — run-owned `contracts/backend-test-analysis.json` from `backend-test-analysis-contract-shell`.
+3. **Validated Backend Test Execution Contract v1** — run-owned `contracts/backend-test-execution.json` from `backend-test-execution-contract-shell` (fixtures, env *names*, `testRoot`, `targetMode`, authenticationMode).
+4. **Target project conventions** — read `conftest.py`, `pytest.ini` / `pyproject.toml` to understand conventions, but do NOT modify them.
 
-Do NOT re-read source documents. Use the reviewed cases only.
+Do NOT re-read source documents for free-form analysis. Use only reviewed cases and the validated contracts. Use only fixture/env/testRoot facts already present in the execution contract; never invent production credentials or secret values.
 
 ### Conversion Rules
 
@@ -53,10 +55,14 @@ Do NOT re-read source documents. Use the reviewed cases only.
 
 #### Write Boundary
 
-- Only **create new** test script files under `testcase/`
+- Only **create new** files under writeSet:
+  - `testcase/**/test_*.py`
+  - `testcase/**/helpers/**` (optional pure helpers)
+  - `testcase/**/factories/**` (optional test data factories)
 - Do NOT modify existing files: `conftest.py`, `pytest.ini`, `pyproject.toml`, `setup.cfg`, `__init__.py`, or any other framework/config file
-- Reuse existing fixtures; if required fixtures do not exist, report the gap instead of creating or modifying framework files
+- Reuse existing fixtures; if required helpers are missing, create NEW helper/factory modules under the writeSet paths above — never edit root conftest
 - Read existing framework files to understand conventions, but treat them as immutable
+- Do NOT write production code, `.env`, secrets, or credential files
 
 #### Naming Conflict Resolution
 
@@ -86,8 +92,9 @@ def test_BE_ORDER_001_create_order_with_valid_data():
 
 #### Fixture Strategy
 
-- Reuse existing project fixtures from `conftest.py` when available
-- Do not create or modify fixture/configuration files in this node
+- Reuse existing project fixtures from `conftest.py` when available (read-only)
+- Do not create or modify root fixture/configuration files (`conftest.py`, pytest.ini, …)
+- Optional NEW helpers/factories may live under `testcase/**/helpers/**` or `testcase/**/factories/**` only
 - Prefer `@pytest.fixture(scope="function")` for test isolation
 - Use `@pytest.mark.parametrize` for boundary condition cases with multiple inputs
 
@@ -98,12 +105,70 @@ def test_BE_ORDER_001_create_order_with_valid_data():
 - Do NOT add workarounds, skips, or try/except blocks to hide failures without explicit justification
 - Report all failures honestly in the output; the downstream `execute-backend-pytest-shell` node captures exit codes and stdout/stderr as-is
 
-#### Assertions
+#### Test Data Preparation Rules (MUST follow)
+
+**When Setup is Needed**
+
+Setup phase is REQUIRED only when test cases need pre-existing data:
+- Query/Read APIs: need data to exist before querying
+- Update/Delete APIs: need data to exist before modifying
+- State transition tests: need data in specific state
+
+Setup phase is NOT needed for:
+- Create APIs: testing the creation itself
+- Validation tests: testing input validation with invalid data
+
+**Data Setup Strategy**
+
+When setup is needed:
+1. Use `@pytest.fixture(scope='module')` or `@pytest.fixture(scope='session')` to prepare shared test data
+2. All test cases in the file share the same pre-constructed data
+
+**Data Construction Priority**
+
+1. **API-first**: Use documented APIs from `analyze-inputs-pi` / reviewed cases
+2. **Reuse existing conftest fixtures** (read-only)
+3. **Direct DB writes are last resort** and only via safe test-DB fixtures with rollback/isolation
+4. If neither API nor safe DB fixture exists, **skip with an explicit gap note** — do not invent credentials or touch live data
+
+**API Data Construction**
+
+- Prefer the `analyze-inputs-pi` API Endpoints section and reviewed cases for method/path/fields
+- Chain API calls only when cases document multi-step preconditions
+- Store created resource IDs in fixtures for reuse
+- Do **not** broadly search host route/controller trees for secrets, `.env`, private keys, or production configs
+- Read host API definitions only when needed to resolve a field name already referenced by reviewed cases
+
+**Database Data Construction (restricted)**
+
+- Allowed only via existing `conftest.py` test-DB fixtures with transaction rollback or equivalent isolation
+- Never hardcode connection strings, passwords, tokens, or cloud credentials
+- Never target production/shared non-test databases
+- If isolation is unclear, report the gap instead of writing DB rows
+
+#### Assertion Rules (MUST follow)
+
+**Positive Path (成功场景)**
+
+MUST assert ALL of the following:
+1. HTTP status code: as defined in API spec (e.g. 200, 201)
+2. Response structure: key fields exist in response body
+3. Specific values: each field equals expected value from test case
+4. Data type: each field is correct type
+
+**Negative Path (异常场景)**
+
+MUST assert ALL of the following:
+1. HTTP status code: as defined in API spec (e.g. 400, 404, 500)
+2. Error code field: field name from API spec (e.g. code, error_code, errcode, ret)
+3. Error message field: field name from API spec (e.g. message, msg, errmsg, error)
+
+**Field Name Resolution**
+
+Field names MUST come from the upstream `analyze-inputs-pi` output (API Endpoints section), NOT hardcoded. For example:
+- If API spec defines `{"ret": 0, "msg": "success"}`, assert `response.json()["ret"]` and `response.json()["msg"]`
+- If API spec defines `{"code": 4001, "message": "error"}`, assert `response.json()["code"]` and `response.json()["message"]`
 
-- Use `assert` statements, not `unittest` assertions
-- Assert specific values, not just "no exception"
-- For API tests: assert status code, response body keys, and specific field values
-- For database tests: assert record state after operation
 
 #### Conditional Test Implementation (include ONLY if test cases exist)