@tea-agent/loop-agent 0.13.0-beta.0 → 0.13.0

package/dist/workflows/dag/backend-test-case-manifest.js

@@ -0,0 +1,503 @@
+import { createHash } from "node:crypto";
+import { readdir, readFile, stat } from "node:fs/promises";
+import path from "node:path";
+import { z } from "zod";
+import { writeDagRunJsonArtifact } from "../../infrastructure/harness/artifact-store.js";
+export const BACKEND_TEST_CASE_MANIFEST_SCHEMA_ID = "backend-test-case-manifest-v1";
+export const BACKEND_TEST_CASE_ID_PATTERN = /^BE-[A-Z0-9]+(?:-[A-Z0-9]+)*-\d{3}$/;
+export const BACKEND_TEST_AC_ID_PATTERN = /^AC-[A-Z0-9]+(?:-[A-Z0-9]+)*$/;
+const SECRET_KEY = /(?:password|passwd|secret|token|api[_-]?key|private[_-]?key|credential|authorization)/i;
+const SECRET_VALUE = /(?:-----BEGIN [A-Z ]*PRIVATE KEY-----|\b(?:sk|ghp|github_pat|xox[baprs]|AKIA)[-_A-Za-z0-9]{12,}\b)/;
+export const backendTestCaseAutomationStatusSchema = z.enum([
+    "planned",
+    "generated",
+    "skipped",
+    "unsupported",
+]);
+export const backendTestCaseCategorySchema = z.enum([
+    "positive",
+    "negative",
+    "boundary",
+    "state-transition",
+    "auth",
+    "timeout",
+    "concurrency",
+    "other",
+]);
+const relativePosixPathSchema = z
+    .string()
+    .min(1)
+    .refine((value) => !path.posix.isAbsolute(value) &&
+    !value.includes("\\") &&
+    !value.split("/").some((segment) => segment === ".." || segment === ""), "file must be a relative posix path without .. segments");
+const gapSchema = z
+    .object({
+    acId: z.string().regex(BACKEND_TEST_AC_ID_PATTERN).optional(),
+    caseId: z.string().regex(BACKEND_TEST_CASE_ID_PATTERN).optional(),
+    description: z.string().min(1),
+    evidenceRef: z.string().min(1).optional(),
+})
+    .strict()
+    .refine((value) => Boolean(value.acId || value.caseId), {
+    message: "evidence gap requires acId and/or caseId",
+});
+const caseSchema = z
+    .object({
+    caseId: z.string().regex(BACKEND_TEST_CASE_ID_PATTERN),
+    acIds: z.array(z.string().regex(BACKEND_TEST_AC_ID_PATTERN)).min(1),
+    title: z.string().min(1),
+    category: backendTestCaseCategorySchema,
+    automationStatus: backendTestCaseAutomationStatusSchema,
+    endpointRef: z.string().min(1).optional(),
+    ruleRefs: z.array(z.string().min(1)).optional(),
+    file: relativePosixPathSchema.optional(),
+    symbol: z.string().min(1).optional(),
+    gapReason: z.string().min(1).optional(),
+    evidenceRef: z.string().min(1).optional(),
+})
+    .strict()
+    .superRefine((value, ctx) => {
+    if (value.automationStatus === "generated") {
+        if (!value.file) {
+            ctx.addIssue({
+                code: z.ZodIssueCode.custom,
+                message: "generated case requires file",
+                path: ["file"],
+            });
+        }
+        if (!value.symbol) {
+            ctx.addIssue({
+                code: z.ZodIssueCode.custom,
+                message: "generated case requires symbol",
+                path: ["symbol"],
+            });
+        }
+    }
+    if (value.automationStatus === "skipped" ||
+        value.automationStatus === "unsupported") {
+        if (!value.gapReason) {
+            ctx.addIssue({
+                code: z.ZodIssueCode.custom,
+                message: `${value.automationStatus} case requires gapReason`,
+                path: ["gapReason"],
+            });
+        }
+    }
+});
+export const backendTestCaseManifestSchema = z
+    .object({
+    schemaVersion: z.literal(1),
+    sourceBinding: z
+        .object({
+        taskId: z.string().min(1),
+        requirementPath: z.string().min(1),
+        requirementSha256: z.string().regex(/^[a-f0-9]{64}$/),
+        referencePaths: z.array(z.string().min(1)),
+        requirementIds: z.array(z.string().regex(/^(?:REQ|BR|AC)-[A-Z0-9]+(?:-[A-Z0-9]+)*$/)),
+    })
+        .strict(),
+    cases: z.array(caseSchema),
+    evidenceGaps: z.array(gapSchema).default([]),
+    coverageSummary: z
+        .object({
+        explicitAcCount: z.number().int().min(0),
+        coveredAcCount: z.number().int().min(0),
+        gappedAcCount: z.number().int().min(0),
+        caseCount: z.number().int().min(0),
+        generatedCount: z.number().int().min(0),
+        plannedCount: z.number().int().min(0),
+        skippedCount: z.number().int().min(0),
+        unsupportedCount: z.number().int().min(0),
+        /** Deterministic fraction in [0,1]; model must not invent this. */
+        acCoverageRatio: z.number().min(0).max(1),
+    })
+        .strict()
+        .optional(),
+})
+    .strict();
+function secretIssues(value, at = "$", issues = []) {
+    if (typeof value === "string" && SECRET_VALUE.test(value)) {
+        issues.push(`${at}: secret-like value is forbidden`);
+    }
+    if (Array.isArray(value)) {
+        value.forEach((item, index) => secretIssues(item, `${at}[${index}]`, issues));
+    }
+    else if (value && typeof value === "object") {
+        for (const [key, child] of Object.entries(value)) {
+            if (SECRET_KEY.test(key)) {
+                issues.push(`${at}.${key}: secret-shaped field name is forbidden`);
+            }
+            secretIssues(child, `${at}.${key}`, issues);
+        }
+    }
+    return issues;
+}
+export function extractStrictJsonObject(text) {
+    const trimmed = text.trim();
+    if (trimmed.startsWith("{") && trimmed.endsWith("}")) {
+        return JSON.parse(trimmed);
+    }
+    const blocks = [...trimmed.matchAll(/```json\s*\n([\s\S]*?)\n```/gi)];
+    if (blocks.length !== 1 || trimmed.replace(blocks[0][0], "").trim()) {
+        throw new Error("manifest output must be one pure JSON object or one fenced json block with no trailing text");
+    }
+    return JSON.parse(blocks[0][1]);
+}
+function assertSourceBinding(manifest, binding) {
+    const requirement = binding.sources.find((source) => source.kind === "requirement");
+    const references = binding.sources
+        .filter((source) => source.kind === "reference")
+        .map((source) => source.path)
+        .sort();
+    const actualReferences = [...manifest.sourceBinding.referencePaths].sort();
+    if (!requirement ||
+        manifest.sourceBinding.taskId !== binding.taskId ||
+        manifest.sourceBinding.requirementPath !== requirement.path ||
+        manifest.sourceBinding.requirementSha256 !== requirement.sha256) {
+        throw new Error("case manifest source binding does not match DAG requirement source");
+    }
+    if (JSON.stringify(actualReferences) !== JSON.stringify(references)) {
+        throw new Error("case manifest referencePaths do not match DAG source binding");
+    }
+    if (JSON.stringify(manifest.sourceBinding.requirementIds) !==
+        JSON.stringify(binding.requirementIds)) {
+        throw new Error("case manifest requirementIds do not match DAG source binding");
+    }
+}
+export function computeCaseManifestCoverageSummary(manifest) {
+    const explicitAc = manifest.sourceBinding.requirementIds.filter((id) => id.startsWith("AC-"));
+    const covered = new Set();
+    for (const item of manifest.cases) {
+        for (const acId of item.acIds)
+            covered.add(acId);
+    }
+    const gapped = new Set(manifest.evidenceGaps
+        .map((gap) => gap.acId)
+        .filter((id) => Boolean(id)));
+    const coveredExplicit = explicitAc.filter((id) => covered.has(id));
+    const gappedExplicit = explicitAc.filter((id) => !covered.has(id) && gapped.has(id));
+    const statusCounts = {
+        generatedCount: 0,
+        plannedCount: 0,
+        skippedCount: 0,
+        unsupportedCount: 0,
+    };
+    for (const item of manifest.cases) {
+        if (item.automationStatus === "generated")
+            statusCounts.generatedCount += 1;
+        else if (item.automationStatus === "planned")
+            statusCounts.plannedCount += 1;
+        else if (item.automationStatus === "skipped")
+            statusCounts.skippedCount += 1;
+        else
+            statusCounts.unsupportedCount += 1;
+    }
+    const explicitAcCount = explicitAc.length;
+    const coveredAcCount = coveredExplicit.length;
+    const acCoverageRatio = explicitAcCount === 0 ? 1 : coveredAcCount / explicitAcCount;
+    return {
+        explicitAcCount,
+        coveredAcCount,
+        gappedAcCount: gappedExplicit.length,
+        caseCount: manifest.cases.length,
+        ...statusCounts,
+        acCoverageRatio,
+    };
+}
+export function assertBackendTestCaseManifestInvariants(manifest, binding) {
+    if (binding)
+        assertSourceBinding(manifest, binding);
+    const seen = new Set();
+    for (const item of manifest.cases) {
+        if (seen.has(item.caseId)) {
+            throw new Error(`duplicate caseId: ${item.caseId}`);
+        }
+        seen.add(item.caseId);
+    }
+    const explicitAc = new Set(manifest.sourceBinding.requirementIds.filter((id) => id.startsWith("AC-")));
+    const knownAc = new Set(explicitAc);
+    // Allow ACs listed only via cases when sourceBinding.requirementIds is broader,
+    // but unknown relative to binding.requirementIds AC-* still fail when binding present.
+    const referencedAc = new Set();
+    for (const item of manifest.cases) {
+        for (const acId of item.acIds)
+            referencedAc.add(acId);
+    }
+    for (const gap of manifest.evidenceGaps) {
+        if (gap.acId)
+            referencedAc.add(gap.acId);
+    }
+    if (binding) {
+        const bindingAc = binding.requirementIds.filter((id) => id.startsWith("AC-"));
+        for (const acId of referencedAc) {
+            if (!bindingAc.includes(acId) && !explicitAc.has(acId)) {
+                throw new Error(`unknown AC reference: ${acId}`);
+            }
+        }
+        // Unknown = referenced AC not in sourceBinding.requirementIds at all
+        for (const acId of referencedAc) {
+            if (!manifest.sourceBinding.requirementIds.includes(acId)) {
+                // requirementIds may include non-AC; unknown AC means not present
+                if (!binding.requirementIds.includes(acId)) {
+                    throw new Error(`unknown AC reference: ${acId}`);
+                }
+            }
+        }
+        // Stricter: AC refs must appear in sourceBinding.requirementIds when they are AC-*
+        for (const acId of referencedAc) {
+            if (!manifest.sourceBinding.requirementIds.includes(acId)) {
+                throw new Error(`unknown AC reference: ${acId}`);
+            }
+        }
+    }
+    else {
+        for (const acId of referencedAc) {
+            if (!manifest.sourceBinding.requirementIds.includes(acId)) {
+                throw new Error(`unknown AC reference: ${acId}`);
+            }
+        }
+    }
+    const coveredByCase = new Set();
+    for (const item of manifest.cases) {
+        for (const acId of item.acIds)
+            coveredByCase.add(acId);
+    }
+    const gappedAc = new Set(manifest.evidenceGaps
+        .map((gap) => gap.acId)
+        .filter((id) => Boolean(id)));
+    for (const acId of explicitAc) {
+        if (!coveredByCase.has(acId) && !gappedAc.has(acId)) {
+            throw new Error(`explicit acceptance criterion ${acId} has no case and no evidence gap`);
+        }
+    }
+    if (manifest.coverageSummary) {
+        const expected = computeCaseManifestCoverageSummary(manifest);
+        const actual = manifest.coverageSummary;
+        for (const key of Object.keys(expected)) {
+            if (actual[key] !== expected[key]) {
+                throw new Error(`coverageSummary.${key} mismatch: expected ${String(expected[key])}, got ${String(actual[key])}`);
+            }
+        }
+    }
+}
+export async function materializeBackendTestCaseManifest(input) {
+    if (!input.sourceBinding) {
+        throw new Error("backend-test case manifest gate requires DAG sourceBinding");
+    }
+    if (!/^[a-z0-9][a-z0-9._-]*\.json$/.test(input.artifactName) ||
+        !/^[a-z0-9][a-z0-9._-]*$/.test(input.outputDir)) {
+        throw new Error("unsafe structured artifact path");
+    }
+    const nodePath = path.join(input.runDir, `${input.fromNodeId}.json`);
+    const record = JSON.parse(await readFile(nodePath, "utf8"));
+    const raw = record.assistantText?.trim() || record.stdout?.trim() || "";
+    let parsed;
+    try {
+        parsed = extractStrictJsonObject(raw);
+    }
+    catch (error) {
+        throw new Error(`invalid-output: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    const secrets = secretIssues(parsed);
+    if (secrets.length) {
+        throw new Error(`invalid-output: ${secrets.join("; ")}`);
+    }
+    const result = backendTestCaseManifestSchema.safeParse(parsed);
+    if (!result.success) {
+        throw new Error(`invalid-output: ${result.error.issues
+            .map((issue) => `${issue.path.join(".")}: ${issue.message}`)
+            .join("; ")}`);
+    }
+    const withSummary = {
+        ...result.data,
+        coverageSummary: result.data.coverageSummary ??
+            computeCaseManifestCoverageSummary(result.data),
+    };
+    assertBackendTestCaseManifestInvariants(withSummary, input.sourceBinding);
+    const relativePath = path.posix.join(input.outputDir, input.artifactName);
+    const artifactPath = await writeDagRunJsonArtifact(input.runDir, relativePath, withSummary);
+    const normalized = `${JSON.stringify(withSummary, null, 2)}\n`;
+    return {
+        path: artifactPath,
+        sha256: createHash("sha256").update(normalized).digest("hex"),
+        schemaId: BACKEND_TEST_CASE_MANIFEST_SCHEMA_ID,
+    };
+}
+/** Convert BE-ORDER-001 → test_BE_ORDER_001 for convention matching. */
+export function caseIdToPytestSymbolPrefix(caseId) {
+    return `test_${caseId.replace(/-/g, "_")}`;
+}
+async function walkPyTestFiles(rootDir, relativeDir = "") {
+    const abs = relativeDir ? path.join(rootDir, relativeDir) : rootDir;
+    let entries;
+    try {
+        entries = await readdir(abs, { withFileTypes: true });
+    }
+    catch {
+        return [];
+    }
+    const files = [];
+    for (const entry of entries) {
+        const rel = relativeDir
+            ? path.posix.join(relativeDir.replace(/\\/g, "/"), entry.name)
+            : entry.name;
+        if (entry.isDirectory()) {
+            files.push(...(await walkPyTestFiles(rootDir, rel)));
+        }
+        else if (entry.isFile() && /^test_.*\.py$/.test(entry.name)) {
+            files.push(rel.replace(/\\/g, "/"));
+        }
+    }
+    return files;
+}
+/**
+ * Locate pytest functions `def test_...` and optional docstring case IDs.
+ */
+export function scanPytestSymbolsInSource(source, file) {
+    const hits = [];
+    const fnRe = /^[ \t]*(?:async\s+)?def\s+(test_[A-Za-z0-9_]+)\s*\(/gm;
+    let match;
+    while ((match = fnRe.exec(source))) {
+        const symbol = match[1];
+        const after = source.slice(match.index + match[0].length, match.index + match[0].length + 400);
+        const doc = after.match(/:\s*(?:\r?\n)?[ \t]*(?:"""|''')([\s\S]*?)(?:"""|''')/);
+        const firstLine = doc?.[1]?.trim().split(/\r?\n/)[0] ?? "";
+        const idMatch = firstLine.match(/\b(BE-[A-Z0-9]+(?:-[A-Z0-9]+)*-\d{3})\b/);
+        const fromName = symbol.match(/^test_(BE_[A-Z0-9]+(?:_[A-Z0-9]+)*_\d{3})(?:_|$)/i);
+        let caseId = idMatch?.[1];
+        if (!caseId && fromName) {
+            const raw = fromName[1];
+            // BE_ORDER_001 → BE-ORDER-001 (module may contain underscores → treat last _NNN as seq)
+            const m = raw.match(/^(BE(?:_[A-Z0-9]+)+)_(\d{3})$/i);
+            if (m) {
+                caseId = `${m[1].replace(/_/g, "-").toUpperCase()}-${m[2]}`;
+                // Fix BE-ORDER-001 already has dashes from replace; above yields BE-ORDER-001 if raw BE_ORDER_001
+            }
+        }
+        hits.push({ file, symbol, caseId });
+    }
+    return hits;
+}
+export async function discoverPytestSymbols(workspaceRoot, testRoot = "testcase") {
+    const root = path.join(workspaceRoot, testRoot);
+    const files = await walkPyTestFiles(root);
+    const hits = [];
+    for (const file of files) {
+        const abs = path.join(root, file);
+        const source = await readFile(abs, "utf8");
+        const rel = path.posix.join(testRoot, file);
+        hits.push(...scanPytestSymbolsInSource(source, rel));
+    }
+    return hits;
+}
+function symbolMatchesCase(hit, caseId, declaredSymbol) {
+    if (declaredSymbol && hit.symbol === declaredSymbol)
+        return true;
+    if (hit.caseId === caseId)
+        return true;
+    const prefix = caseIdToPytestSymbolPrefix(caseId);
+    return hit.symbol === prefix || hit.symbol.startsWith(`${prefix}_`);
+}
+/**
+ * Traceability gate (W3.4):
+ * - skipped/unsupported require gapReason (schema-enforced; re-checked)
+ * - generated: declared file+symbol must exist on disk
+ * - planned: convention scan — if any pytest symbol maps to caseId it is OK;
+ *   if none, fail-closed only when *some* generated/automation evidence is expected?
+ *   MVP: planned cases must either have a convention symbol on disk OR remain planned
+ *   without generated claim. Fail only when automationStatus=generated is false mapping,
+ *   OR when a planned case has no symbol AND we require automation — per 需求 MVP:
+ *   "traceability gate 按约定命名扫描 ... 并与 manifest caseId 对齐"
+ *   Fail when: generated missing file/symbol; OR planned case with no matching symbol
+ *   when at least one test_*.py exists and case is not skipped/unsupported.
+ *   Safer AC-003: generated without real file/symbol fails; planned without symbol is OK
+ *   (still planned). Convention: if symbol exists for caseId, optional enrichment only.
+ */
+export async function assertBackendTestTraceability(input) {
+    const testRoot = input.testRoot ?? "testcase";
+    const hits = await discoverPytestSymbols(input.workspaceRoot, testRoot);
+    const matched = [];
+    for (const item of input.manifest.cases) {
+        if (item.automationStatus === "skipped" ||
+            item.automationStatus === "unsupported") {
+            if (!item.gapReason?.trim()) {
+                throw new Error(`traceability: ${item.caseId} is ${item.automationStatus} without gapReason`);
+            }
+            continue;
+        }
+        if (item.automationStatus === "generated") {
+            if (!item.file || !item.symbol) {
+                throw new Error(`traceability: generated case ${item.caseId} missing file/symbol`);
+            }
+            const abs = path.join(input.workspaceRoot, item.file);
+            let st;
+            try {
+                st = await stat(abs);
+            }
+            catch {
+                throw new Error(`traceability: generated case ${item.caseId} file not found: ${item.file}`);
+            }
+            if (!st.isFile()) {
+                throw new Error(`traceability: generated case ${item.caseId} path is not a file: ${item.file}`);
+            }
+            const source = await readFile(abs, "utf8");
+            const localHits = scanPytestSymbolsInSource(source, item.file);
+            const found = localHits.find((hit) => hit.symbol === item.symbol);
+            if (!found) {
+                throw new Error(`traceability: generated case ${item.caseId} symbol not found: ${item.symbol} in ${item.file}`);
+            }
+            // Exact symbol must also bind to the caseId (convention name or docstring/ID).
+            // Reject forged mappings like caseId=BE-ORDER-001 + symbol=test_unrelated.
+            const bound = found.caseId === item.caseId ||
+                (() => {
+                    const prefix = caseIdToPytestSymbolPrefix(item.caseId);
+                    return (found.symbol === prefix || found.symbol.startsWith(`${prefix}_`));
+                })();
+            if (!bound) {
+                throw new Error(`traceability: generated case ${item.caseId} symbol ${item.symbol} is not bound to caseId (name convention or docstring ID)`);
+            }
+            matched.push({
+                caseId: item.caseId,
+                file: found.file,
+                symbol: found.symbol,
+            });
+            continue;
+        }
+        // planned: if a convention symbol exists, record match; missing is allowed for planned
+        const hit = hits.find((candidate) => symbolMatchesCase(candidate, item.caseId, item.symbol));
+        if (hit) {
+            matched.push({
+                caseId: item.caseId,
+                file: hit.file,
+                symbol: hit.symbol,
+            });
+        }
+    }
+    return { ok: true, matched };
+}
+export async function runBackendTestTraceabilityGate(input) {
+    const relative = input.manifestRelativePath ?? "contracts/backend-test-case-manifest.json";
+    const manifestPath = path.join(input.runDir, relative);
+    const raw = JSON.parse(await readFile(manifestPath, "utf8"));
+    const parsed = backendTestCaseManifestSchema.safeParse(raw);
+    if (!parsed.success) {
+        throw new Error(`traceability: invalid manifest: ${parsed.error.issues
+            .map((issue) => issue.message)
+            .join("; ")}`);
+    }
+    assertBackendTestCaseManifestInvariants(parsed.data);
+    const result = await assertBackendTestTraceability({
+        workspaceRoot: input.workspaceRoot,
+        manifest: parsed.data,
+        testRoot: input.testRoot,
+    });
+    return {
+        ok: true,
+        matchedCount: result.matched.length,
+        manifestPath,
+    };
+}
+/** Shell-friendly: build a one-liner that invokes node -e would be heavy; export for shell-executor special command. */
+export function formatTraceabilityGateStdout(result) {
+    return `Traceability gate: pass\nManifest: ${result.manifestPath}\nMatched symbols: ${result.matchedCount}`;
+}