@tea-agent/loop-agent 0.13.0-beta.0 → 0.13.0

package/dist/workflows/dag/backend-test-result-contract.js

@@ -0,0 +1,568 @@
+import { createHash } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { z } from "zod";
+import { writeDagRunJsonArtifact } from "../../infrastructure/harness/artifact-store.js";
+export const BACKEND_TEST_RESULT_SCHEMA_ID = "backend-test-result-v1";
+const SECRET_KEY = /(?:password|passwd|secret|token|api[_-]?key|private[_-]?key|credential|authorization)/i;
+const SECRET_VALUE = /(?:-----BEGIN [A-Z ]*PRIVATE KEY-----|\b(?:sk|ghp|github_pat|xox[baprs]|AKIA)[-_A-Za-z0-9]{12,}\b)/;
+export const BACKEND_TEST_CLASSIFICATION_CATEGORIES = [
+    "ProductBug",
+    "TestBug",
+    "EnvFailure",
+    "ContractMismatch",
+    "FlakyTest",
+    "Unknown",
+];
+export const backendTestExecutionStatusSchema = z.enum([
+    "completed",
+    "collection-error",
+    "command-error",
+    "report-error",
+]);
+export const backendTestCollectionStatusSchema = z.enum([
+    "ok",
+    "error",
+    "unknown",
+    "skipped",
+]);
+/**
+ * Task Pool consumable outcome tokens (frozen for M2; no auto follow-up).
+ * - passed: all executed tests green
+ * - completed-with-failures: process completed with assertion/test failures
+ * - collection-error: import/collection failures
+ * - command-error: runner/command could not complete normally
+ * - report-error: junit missing/corrupt after execution attempt
+ */
+export const backendTestOutcomeSchema = z.enum([
+    "passed",
+    "completed-with-failures",
+    "collection-error",
+    "command-error",
+    "report-error",
+]);
+const junitRelativePathSchema = z
+    .string()
+    .min(1)
+    .refine((value) => !path.posix.isAbsolute(value) &&
+    !value.includes("\\") &&
+    value
+        .split("/")
+        .every((segment) => segment.length > 0 && segment !== "." && segment !== ".."), { message: "junit.relativePath must be a safe run-relative POSIX path" });
+const failureSummarySchema = z
+    .object({
+    classname: z.string().min(1),
+    name: z.string().min(1),
+    message: z.string().min(1),
+    kind: z.enum(["failure", "error"]).default("failure"),
+})
+    .strict();
+export const backendTestResultContractSchema = z
+    .object({
+    schemaVersion: z.literal(1),
+    /** Process-level status of the pytest invocation. Task Pool: use with outcome. */
+    executionStatus: backendTestExecutionStatusSchema,
+    pytestExitCode: z.number().int().min(0).max(255),
+    collectionStatus: backendTestCollectionStatusSchema,
+    /** Total tests counted from JUnit (or 0 when report unavailable). */
+    tests: z.number().int().min(0),
+    passed: z.number().int().min(0),
+    failed: z.number().int().min(0),
+    error: z.number().int().min(0),
+    skipped: z.number().int().min(0),
+    durationMs: z.number().nonnegative().optional(),
+    junit: z
+        .object({
+        relativePath: junitRelativePathSchema,
+        sha256: z
+            .string()
+            .regex(/^[a-f0-9]{64}$/, "junit.sha256 must be lowercase hex sha256"),
+    })
+        .strict(),
+    /** Non-secret command summary only (no tokens/passwords). */
+    commandSummary: z.string().min(1),
+    /** Truncated failure/error summaries for Task Pool triage. */
+    failures: z.array(failureSummarySchema),
+    /**
+     * Authoritative shell-facing outcome for backend-test-outcome-gate-shell.
+     * Do not override from retrospective Markdown.
+     */
+    outcome: backendTestOutcomeSchema,
+})
+    .strict()
+    .superRefine((value, ctx) => {
+    if (value.tests !==
+        value.passed + value.failed + value.error + value.skipped) {
+        ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: "tests must equal passed+failed+error+skipped",
+            path: ["tests"],
+        });
+    }
+    if (SECRET_KEY.test(value.commandSummary) ||
+        SECRET_VALUE.test(value.commandSummary)) {
+        ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: "commandSummary must not contain secret-like content",
+            path: ["commandSummary"],
+        });
+    }
+    for (const [index, failure] of value.failures.entries()) {
+        if (SECRET_VALUE.test(failure.message)) {
+            ctx.addIssue({
+                code: z.ZodIssueCode.custom,
+                message: "failure message must not contain secret-like content",
+                path: ["failures", index, "message"],
+            });
+        }
+    }
+});
+const MAX_FAILURE_MESSAGE = 500;
+const MAX_FAILURES = 50;
+function decodeXmlEntities(value) {
+    return value
+        .replace(/&lt;/g, "<")
+        .replace(/&gt;/g, ">")
+        .replace(/&quot;/g, '"')
+        .replace(/&apos;/g, "'")
+        .replace(/&amp;/g, "&");
+}
+const XML_ATTR_PATTERNS = {
+    tests: /\btests\s*=\s*"([^"]*)"/i,
+    failures: /\bfailures\s*=\s*"([^"]*)"/i,
+    errors: /\berrors\s*=\s*"([^"]*)"/i,
+    skipped: /\bskipped\s*=\s*"([^"]*)"/i,
+    time: /\btime\s*=\s*"([^"]*)"/i,
+    classname: /\bclassname\s*=\s*"([^"]*)"/i,
+    name: /\bname\s*=\s*"([^"]*)"/i,
+    message: /\bmessage\s*=\s*"([^"]*)"/i,
+};
+function attr(tag, name) {
+    const pattern = XML_ATTR_PATTERNS[name];
+    if (!pattern)
+        return undefined;
+    const match = tag.match(pattern);
+    return match ? decodeXmlEntities(match[1] ?? "") : undefined;
+}
+function truncate(value, max = MAX_FAILURE_MESSAGE) {
+    const normalized = value.replace(/\s+/g, " ").trim();
+    if (normalized.length <= max)
+        return normalized;
+    return `${normalized.slice(0, max - 1)}…`;
+}
+function assertNoSecrets(label, value) {
+    if (SECRET_KEY.test(value) || SECRET_VALUE.test(value)) {
+        throw new Error(`secret-like content rejected in ${label}`);
+    }
+}
+/**
+ * Minimal JUnit XML parser (no new npm deps). Fail-closed on non-JUnit markup.
+ */
+export function parseJunitXml(xml) {
+    const trimmed = xml.trim();
+    if (!trimmed) {
+        throw new Error("invalid junit xml: empty");
+    }
+    if (!/<testsuites\b/i.test(trimmed) && !/<testsuite\b/i.test(trimmed)) {
+        throw new Error("invalid junit xml: missing testsuites/testsuite root");
+    }
+    const hasSuitesRoot = /<testsuites\b/i.test(trimmed);
+    if ((hasSuitesRoot && !/<\/testsuites>\s*$/i.test(trimmed)) ||
+        (!hasSuitesRoot && !/<\/testsuite>\s*$/i.test(trimmed))) {
+        throw new Error("invalid junit xml: unclosed root element");
+    }
+    const openedCases = (trimmed.match(/<testcase\b/gi) ?? []).length;
+    const closedCases = (trimmed.match(/<\/testcase>/gi) ?? []).length;
+    const selfClosingCases = (trimmed.match(/<testcase\b[^>]*\/>/gi) ?? [])
+        .length;
+    if (openedCases !== closedCases + selfClosingCases) {
+        throw new Error("invalid junit xml: unclosed testcase element");
+    }
+    // Prefer root testsuites aggregates when present.
+    const suitesOpen = trimmed.match(/<testsuites\b[^>]*>/i)?.[0];
+    let tests = 0;
+    let failed = 0;
+    let errors = 0;
+    let skipped = 0;
+    let timeSec;
+    if (suitesOpen) {
+        const t = attr(suitesOpen, "tests");
+        const f = attr(suitesOpen, "failures");
+        const e = attr(suitesOpen, "errors");
+        const s = attr(suitesOpen, "skipped");
+        const time = attr(suitesOpen, "time");
+        if (t !== undefined)
+            tests = Number(t);
+        if (f !== undefined)
+            failed = Number(f);
+        if (e !== undefined)
+            errors = Number(e);
+        if (s !== undefined)
+            skipped = Number(s);
+        if (time !== undefined && time !== "")
+            timeSec = Number(time);
+    }
+    // Self-closing first so empty cases ending with /> are not greedily paired with a later </testcase>.
+    const caseRe = /<testcase\b([^>]*?)\/>|<testcase\b([^>]*)>([\s\S]*?)<\/testcase>/gi;
+    const failures = [];
+    let caseCount = 0;
+    let caseFailed = 0;
+    let caseErrors = 0;
+    let caseSkipped = 0;
+    let match = caseRe.exec(trimmed);
+    while (match !== null) {
+        caseCount += 1;
+        const openAttrs = match[1] ?? match[2] ?? "";
+        const body = match[3] ?? "";
+        const classname = attr(openAttrs, "classname") || "unknown";
+        const name = attr(openAttrs, "name") || "unknown";
+        const failureTag = body.match(/<failure\b([^>]*)>([\s\S]*?)<\/failure>|<failure\b([^>]*)\/>/i);
+        const errorTag = body.match(/<error\b([^>]*)>([\s\S]*?)<\/error>|<error\b([^>]*)\/>/i);
+        const skippedTag = /<skipped\b/i.test(body);
+        if (failureTag) {
+            caseFailed += 1;
+            const fAttrs = failureTag[1] ?? failureTag[3] ?? "";
+            const fBody = failureTag[2] ?? "";
+            const message = attr(fAttrs, "message") || fBody || "failure";
+            failures.push({
+                classname,
+                name,
+                message: truncate(message),
+                kind: "failure",
+            });
+        }
+        else if (errorTag) {
+            caseErrors += 1;
+            const eAttrs = errorTag[1] ?? errorTag[3] ?? "";
+            const eBody = errorTag[2] ?? "";
+            const message = attr(eAttrs, "message") || eBody || "error";
+            failures.push({
+                classname,
+                name,
+                message: truncate(message),
+                kind: "error",
+            });
+        }
+        else if (skippedTag) {
+            caseSkipped += 1;
+        }
+        match = caseRe.exec(trimmed);
+    }
+    // Prefer case-level facts when testcase elements exist; never trust root counters
+    // that hide <failure>/<error> children (fail-closed on contradictory aggregates).
+    if (caseCount > 0) {
+        if (suitesOpen) {
+            const declaredTests = attr(suitesOpen, "tests") !== undefined
+                ? Number(attr(suitesOpen, "tests"))
+                : undefined;
+            const declaredFailed = attr(suitesOpen, "failures") !== undefined
+                ? Number(attr(suitesOpen, "failures"))
+                : undefined;
+            const declaredErrors = attr(suitesOpen, "errors") !== undefined
+                ? Number(attr(suitesOpen, "errors"))
+                : undefined;
+            const declaredSkipped = attr(suitesOpen, "skipped") !== undefined
+                ? Number(attr(suitesOpen, "skipped"))
+                : undefined;
+            // pytest collection reports often use tests="0" with a synthetic error testcase.
+            // Always fail-closed when root counters under-report failure/error children.
+            if (declaredTests !== undefined &&
+                !Number.isNaN(declaredTests) &&
+                declaredTests !== caseCount) {
+                const collectionStyle = declaredTests === 0 && caseErrors > 0 && caseFailed === 0;
+                if (!collectionStyle) {
+                    throw new Error(`invalid junit xml: testsuites tests=${declaredTests} disagrees with testcase count=${caseCount}`);
+                }
+            }
+            if (declaredFailed !== undefined &&
+                !Number.isNaN(declaredFailed) &&
+                declaredFailed !== caseFailed) {
+                throw new Error(`invalid junit xml: testsuites failures=${declaredFailed} disagrees with case failures=${caseFailed}`);
+            }
+            if (declaredErrors !== undefined &&
+                !Number.isNaN(declaredErrors) &&
+                declaredErrors !== caseErrors) {
+                throw new Error(`invalid junit xml: testsuites errors=${declaredErrors} disagrees with case errors=${caseErrors}`);
+            }
+            if (declaredSkipped !== undefined &&
+                !Number.isNaN(declaredSkipped) &&
+                declaredSkipped !== caseSkipped) {
+                throw new Error(`invalid junit xml: testsuites skipped=${declaredSkipped} disagrees with case skipped=${caseSkipped}`);
+            }
+        }
+        tests = caseCount;
+        failed = caseFailed;
+        errors = caseErrors;
+        skipped = caseSkipped;
+    }
+    else if (!suitesOpen || Number.isNaN(tests) || tests === 0) {
+        // No testcase bodies: fall back to first testsuite attributes.
+        const suiteOpen = trimmed.match(/<testsuite\b[^>]*>/i)?.[0];
+        if (!suiteOpen) {
+            throw new Error("invalid junit xml: no testsuite data");
+        }
+        tests = Number(attr(suiteOpen, "tests") ?? "0");
+        failed = Number(attr(suiteOpen, "failures") ?? "0");
+        errors = Number(attr(suiteOpen, "errors") ?? "0");
+        skipped = Number(attr(suiteOpen, "skipped") ?? "0");
+        const time = attr(suiteOpen, "time");
+        if (time)
+            timeSec = Number(time);
+    }
+    if ([tests, failed, errors, skipped].some((n) => Number.isNaN(n) || n < 0)) {
+        throw new Error("invalid junit xml: non-numeric suite counters");
+    }
+    const passed = Math.max(0, tests - failed - errors - skipped);
+    return {
+        tests,
+        passed,
+        failed,
+        errors,
+        skipped,
+        durationMs: timeSec !== undefined && !Number.isNaN(timeSec)
+            ? Math.round(timeSec * 1000)
+            : undefined,
+        failures: failures.slice(0, MAX_FAILURES),
+    };
+}
+export function deriveBackendTestResult(input) {
+    assertNoSecrets("commandSummary", input.commandSummary);
+    const emptyJunitMeta = {
+        relativePath: input.junitRelativePath,
+        sha256: "0".repeat(64),
+    };
+    if (input.junitXml === null || input.junitXml === undefined) {
+        if (!input.allowMissingJunit) {
+            throw new Error("missing junit report");
+        }
+        const outcome = input.pytestExitCode === 0 ? "report-error" : "command-error";
+        const executionStatus = input.pytestExitCode === 0 ? "report-error" : "command-error";
+        const result = {
+            schemaVersion: 1,
+            executionStatus,
+            pytestExitCode: input.pytestExitCode,
+            collectionStatus: "unknown",
+            tests: 0,
+            passed: 0,
+            failed: 0,
+            error: 0,
+            skipped: 0,
+            junit: emptyJunitMeta,
+            commandSummary: input.commandSummary,
+            failures: [],
+            outcome,
+        };
+        return backendTestResultContractSchema.parse(result);
+    }
+    let parsed;
+    try {
+        parsed = parseJunitXml(input.junitXml);
+    }
+    catch (error) {
+        throw new Error(`invalid junit xml: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    const sha256 = createHash("sha256").update(input.junitXml).digest("hex");
+    const exit = input.pytestExitCode;
+    let executionStatus = "completed";
+    let collectionStatus = "ok";
+    let outcome = "passed";
+    // Collection-heavy signals: pytest exit 2 is common for collection errors;
+    // also when error cases exist with zero/low completed tests.
+    const looksLikeCollection = exit === 2 ||
+        (parsed.errors > 0 && parsed.passed + parsed.failed === 0) ||
+        parsed.failures.some((f) => f.kind === "error" &&
+            /collect|import|syntax/i.test(`${f.name} ${f.message}`));
+    if (looksLikeCollection && (parsed.errors > 0 || exit >= 2)) {
+        executionStatus = "collection-error";
+        collectionStatus = "error";
+        outcome = "collection-error";
+    }
+    else if (exit >= 2 && parsed.failed === 0 && parsed.errors === 0) {
+        executionStatus = "command-error";
+        collectionStatus = "unknown";
+        outcome = "command-error";
+    }
+    else if (parsed.failed > 0 || parsed.errors > 0 || exit === 1) {
+        executionStatus = "completed";
+        collectionStatus = "ok";
+        outcome = "completed-with-failures";
+    }
+    else if (exit === 0 && parsed.failed === 0 && parsed.errors === 0) {
+        executionStatus = "completed";
+        collectionStatus = "ok";
+        outcome = "passed";
+    }
+    else {
+        executionStatus = "command-error";
+        collectionStatus = "unknown";
+        outcome = "command-error";
+    }
+    const result = {
+        schemaVersion: 1,
+        executionStatus,
+        pytestExitCode: exit,
+        collectionStatus,
+        tests: parsed.tests,
+        passed: parsed.passed,
+        failed: parsed.failed,
+        error: parsed.errors,
+        skipped: parsed.skipped,
+        durationMs: parsed.durationMs,
+        junit: {
+            relativePath: input.junitRelativePath,
+            sha256,
+        },
+        commandSummary: input.commandSummary,
+        failures: parsed.failures.map((f) => ({
+            classname: f.classname || "unknown",
+            name: f.name || "unknown",
+            message: truncate(f.message || "failure"),
+            kind: f.kind,
+        })),
+        outcome,
+    };
+    return backendTestResultContractSchema.parse(result);
+}
+export function validateBackendTestResultJunitIntegrity(result, junitXml) {
+    backendTestResultContractSchema.parse(result);
+    const actualSha = createHash("sha256").update(junitXml).digest("hex");
+    if (result.junit.sha256 !== actualSha) {
+        throw new Error("junit sha256 mismatch against report content");
+    }
+}
+/**
+ * Deterministic classification constraints for Pi classifier (not the final label).
+ * Single-run failures must never suggest FlakyTest; collection/command never ProductBug.
+ */
+export function classifyCategoryHints(result) {
+    const forbidden = new Set();
+    const suggested = new Set();
+    // Single observation cannot prove flakiness.
+    forbidden.add("FlakyTest");
+    if (result.executionStatus === "collection-error" ||
+        result.executionStatus === "command-error" ||
+        result.executionStatus === "report-error" ||
+        result.outcome === "collection-error" ||
+        result.outcome === "command-error" ||
+        result.outcome === "report-error") {
+        forbidden.add("ProductBug");
+        suggested.add("EnvFailure");
+        suggested.add("Unknown");
+        if (result.executionStatus === "collection-error") {
+            suggested.add("TestBug");
+        }
+        return {
+            suggestedCategories: [...suggested],
+            forbiddenCategories: [...forbidden],
+            confidenceCap: 0.6,
+        };
+    }
+    if (result.outcome === "passed") {
+        return {
+            suggestedCategories: [],
+            forbiddenCategories: [
+                ...forbidden,
+                "ProductBug",
+                "TestBug",
+                "EnvFailure",
+            ],
+            confidenceCap: 1,
+        };
+    }
+    // Assertion failures: ProductBug / TestBug / Unknown allowed; not Flaky.
+    suggested.add("ProductBug");
+    suggested.add("TestBug");
+    suggested.add("Unknown");
+    return {
+        suggestedCategories: [...suggested],
+        forbiddenCategories: [...forbidden],
+        confidenceCap: 0.75,
+    };
+}
+async function readPytestExitCode(runDir, fromNodeId) {
+    const exitPath = path.join(runDir, "reports", "backend-test-pytest-exit.txt");
+    try {
+        const raw = (await readFile(exitPath, "utf8")).trim();
+        const code = Number(raw);
+        if (raw !== "" && Number.isInteger(code) && code >= 0 && code <= 255)
+            return code;
+        throw new Error(`invalid pytest exit evidence at ${exitPath}`);
+    }
+    catch (error) {
+        if (error.code !== "ENOENT")
+            throw error;
+        // Missing dedicated evidence may fall through to an explicit node marker.
+    }
+    const nodePath = path.join(runDir, `${fromNodeId}.json`);
+    try {
+        const record = JSON.parse(await readFile(nodePath, "utf8"));
+        const blob = `${record.stdout ?? ""}\n${record.stderr ?? ""}`;
+        const marker = blob.match(/pytestExitCode\s*=\s*(\d{1,3})/i);
+        if (marker) {
+            const code = Number(marker[1]);
+            if (Number.isInteger(code) && code >= 0 && code <= 255)
+                return code;
+        }
+    }
+    catch (error) {
+        if (error.code !== "ENOENT") {
+            throw new Error("invalid pytest exit evidence in execute node record");
+        }
+    }
+    throw new Error("missing valid pytest exit evidence");
+}
+export async function materializeBackendTestResultFromRunDir(input) {
+    if (!/^[a-z0-9][a-z0-9._-]*\.json$/.test(input.artifactName) ||
+        !/^[a-z0-9][a-z0-9._-]*$/.test(input.outputDir)) {
+        throw new Error("unsafe structured artifact path");
+    }
+    const junitRelativePath = input.junitRelativePath ?? "reports/backend-test-junit.xml";
+    const junitAbs = path.join(input.runDir, ...junitRelativePath.split("/"));
+    let junitXml = null;
+    try {
+        junitXml = await readFile(junitAbs, "utf8");
+    }
+    catch {
+        junitXml = null;
+    }
+    if (junitXml === null) {
+        throw new Error(`missing junit report at ${junitRelativePath} (fail-closed for Result v1)`);
+    }
+    if (!junitXml.trim()) {
+        throw new Error("invalid junit xml: empty report");
+    }
+    const pytestExitCode = await readPytestExitCode(input.runDir, input.fromNodeId);
+    const commandSummary = "PYTHONDONTWRITEBYTECODE=1 python -m pytest testcase/ -v -p no:cacheprovider --junitxml=reports/backend-test-junit.xml";
+    let result;
+    try {
+        result = deriveBackendTestResult({
+            pytestExitCode,
+            junitXml,
+            junitRelativePath,
+            commandSummary,
+        });
+    }
+    catch (error) {
+        throw new Error(`invalid junit/report: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    // Integrity: stored sha must match bytes we read.
+    validateBackendTestResultJunitIntegrity(result, junitXml);
+    const relativePath = path.posix.join(input.outputDir, input.artifactName);
+    const artifactPath = await writeDagRunJsonArtifact(input.runDir, relativePath, result);
+    const serialized = `${JSON.stringify(result, null, 2)}\n`;
+    return {
+        path: artifactPath,
+        sha256: createHash("sha256").update(serialized).digest("hex"),
+        schemaId: BACKEND_TEST_RESULT_SCHEMA_ID,
+    };
+}
+/** Shell snippet for backend-test-outcome-gate-shell (result.outcome authoritative). */
+export function buildBackendTestOutcomeGateShellSnippet(options) {
+    const resultRelativePath = options?.resultRelativePath ?? "contracts/backend-test-result.json";
+    return [
+        'test -n "${HARNESS_DAG_RUN_DIR:-}" || { echo "missing HARNESS_DAG_RUN_DIR for backend-test outcome gate" >&2; exit 2; }',
+        `RESULT="\${HARNESS_DAG_RUN_DIR}/${resultRelativePath}"`,
+        'test -f "${RESULT}" || { echo "missing backend-test result: ${RESULT}" >&2; exit 2; }',
+        `node -e 'const fs=require("fs");const r=JSON.parse(fs.readFileSync(process.argv[1],"utf8"));const outcome=String(r.outcome||"");const ok=outcome==="passed"&&Number(r.failed||0)===0&&Number(r.error||0)===0;console.log("backend-test outcome="+outcome+" passed="+r.passed+" failed="+r.failed+" error="+r.error+" executionStatus="+r.executionStatus);if(!ok){process.exit(1);}' "\${RESULT}"`,
+    ].join("; ");
+}