@kylecheng3146/agent-ops 0.5.3 → 0.6.0

package/dist/runtime/src/verify/acceptance-replay.js

@@ -0,0 +1,239 @@
+import { lstat, mkdir, mkdtemp, readFile, realpath, rm, writeFile, chmod } from "node:fs/promises";
+import { join, delimiter, isAbsolute } from "node:path";
+import { tmpdir } from "node:os";
+import { writePrivateFile } from "../security/permissions.js";
+import { redactSecrets } from "../security/redact.js";
+import { randomUUID } from "node:crypto";
+import { canonicalJson } from "../config/hash.js";
+import { sha256 } from "../fs/hash.js";
+import { AgentOpsError, resolveContainedPath } from "../fs/paths.js";
+import { criterionContractHash, taskContractHash } from "../task/contract.js";
+import { aggregateAcceptanceRun, makeAcceptanceCheck, makeAcceptanceRun } from "./acceptance-protocol.js";
+import { collectNodeAcceptance, collectJestAcceptance, collectVitestAcceptance, collectPytestAcceptance, collectRustAcceptance } from "./adapters/index.js";
+import { buildVerificationEvidence } from "./evidence.js";
+import { collectChangeSurface, resolveGitCommit } from "./change-surface.js";
+import { runVerificationCommand } from "./spawn.js";
+async function executableIdentity(command, cwd, env) {
+    const paths = isAbsolute(command) ? [command] : command.includes("/") ? [join(cwd, command)] :
+        (env.PATH ?? "").split(delimiter).filter(Boolean).map(path => join(path, command));
+    for (const path of paths) {
+        try {
+            const resolved = await realpath(path);
+            const entry = await lstat(resolved);
+            if (!entry.isFile() || (process.platform !== "win32" && (entry.mode & 0o111) === 0))
+                continue;
+            return sha256(canonicalJson({ path: resolved, bytes: sha256(await readFile(resolved)) }));
+        }
+        catch { /* Try the next fixed PATH entry. */ }
+    }
+    return "unavailable";
+}
+function replayEnvironment(checkout, phase, executionId) {
+    const env = Object.fromEntries(Object.entries(process.env).filter((pair) => pair[1] !== undefined && !/^(?:NODE_OPTIONS|NODE_PATH|PYTHONPATH|PYTHONHOME|CARGO_TARGET_DIR|RUSTC_WRAPPER|RUSTC_WORKSPACE_WRAPPER|AGENT_OPS_|CLAUDE_|CODEX_)/u.test(pair[0])));
+    env.PATH = [join(checkout, "node_modules/.bin"), join(checkout, ".venv/bin"),
+        ...(process.env.PATH ?? "").split(delimiter).filter(path => !path.includes("node_modules/.bin"))].join(delimiter);
+    env.AGENT_OPS_ACCEPTANCE_PHASE = phase;
+    env.AGENT_OPS_ACCEPTANCE_EXECUTION_ID = executionId;
+    return env;
+}
+function collected(runner, output, checkout, options) {
+    if (output.timedOut || output.signal !== null || output.stdoutTruncated || output.stderrTruncated || output.status === "UNKNOWN")
+        return makeAcceptanceRun({ ...options, framework: runner.adapter, results: [], completed: false, diagnostics: [output.failureClass] });
+    // Framework file identities must be repository-relative across disposable checkouts.
+    const normalize = (value) => typeof value === "string" && value.startsWith(checkout + "/") ? value.slice(checkout.length + 1) :
+        Array.isArray(value) ? value.map(normalize) : typeof value === "object" && value !== null ?
+            Object.fromEntries(Object.entries(value).map(([key, item]) => [key, normalize(item)])) : value;
+    let structured = output.stdout;
+    try {
+        structured = JSON.stringify(normalize(JSON.parse(structured)));
+    }
+    catch {
+        structured = structured.split("\n").map(line => { try {
+            return JSON.stringify(normalize(JSON.parse(line)));
+        }
+        catch {
+            return line;
+        } }).join("\n");
+    }
+    switch (runner.adapter) {
+        case "node": return collectNodeAcceptance(structured, options);
+        case "jest": return collectJestAcceptance(structured, options);
+        case "vitest": return collectVitestAcceptance(structured, options);
+        case "pytest": return collectPytestAcceptance(structured, options);
+        case "rust": return collectRustAcceptance(structured, options);
+        case "generic": {
+            const value = JSON.parse(structured);
+            if (value.protocolVersion !== 1 || value.executionId !== options.executionId || value.phase !== options.phase)
+                throw new TypeError("Generic output does not match this execution.");
+            return makeAcceptanceRun({ ...value, results: value.results.map(makeAcceptanceCheck) });
+        }
+    }
+}
+/** Clean disposable clones are verification checkouts, never managed writer worktrees. */
+export async function replayAcceptance(options) {
+    if (!options.trusted)
+        throw new AgentOpsError("VERIFICATION_UNTRUSTED", "Acceptance runners require the repository's current trust binding.");
+    if ((await collectChangeSurface(options.gitRunner)).paths.length > 0)
+        throw new AgentOpsError("ACCEPTANCE_COMMITTED_SOURCE_REQUIRED", "Formal acceptance replay requires a clean committed candidate.");
+    const candidate = await resolveGitCommit(options.gitRunner, "HEAD");
+    const groups = new Map();
+    for (const criterion of options.task.criteria) {
+        if (criterion.acceptance === undefined || criterion.acceptance.mode === "review-only")
+            continue;
+        for (const binding of criterion.acceptance.bindings) {
+            const runner = options.config.verification.acceptanceRunners?.find(r => r.id === binding.runnerId);
+            if (runner === undefined)
+                throw new AgentOpsError("ACCEPTANCE_RUNNER_MISSING", "The acceptance runner is not configured.");
+            const key = canonicalJson({ baseline: criterion.acceptance.baselineCommit, runner, materials: binding.materials });
+            const group = groups.get(key);
+            if (group === undefined)
+                groups.set(key, { baseline: criterion.acceptance.baselineCommit, runner, binding, criteria: [criterion] });
+            else if (!group.criteria.some(c => c.id === criterion.id))
+                group.criteria.push(criterion);
+        }
+    }
+    const reports = [];
+    for (const group of groups.values()) {
+        const materials = [];
+        for (const material of group.binding.materials) {
+            const metadata = await options.gitRunner.run(["ls-tree", "-z", candidate, "--", material.path]);
+            const entry = Buffer.from(metadata.stdout).toString("utf8").match(/^(100644|100755) blob ([a-f0-9]+)\t([^\0]+)\0$/u);
+            if (metadata.exitCode !== 0 || entry === null || entry[3] !== material.path)
+                throw new AgentOpsError("ACCEPTANCE_UNSAFE_MATERIAL", "Acceptance materials must be committed regular files.");
+            const blob = await options.gitRunner.run(["cat-file", "blob", entry[2]]);
+            if (blob.exitCode !== 0 || blob.stdout.byteLength > 1024 * 1024)
+                throw new AgentOpsError("ACCEPTANCE_MATERIAL_UNAVAILABLE", "Acceptance material exceeds its bound or is unavailable.");
+            materials.push({ ...material, bytes: blob.stdout, mode: entry[1] === "100755" ? 0o755 : 0o644 });
+        }
+        const materialDigest = sha256(canonicalJson(materials.map(m => ({ path: m.path, role: m.role, mode: m.mode, hash: sha256(m.bytes) }))));
+        for (const phase of ["baseline", "candidate"]) {
+            const commit = phase === "baseline" ? group.baseline : candidate;
+            const executionId = randomUUID();
+            const startedAt = new Date().toISOString();
+            const temp = await mkdtemp(join(tmpdir(), "agent-ops-acceptance-"));
+            const checkout = join(temp, "repository");
+            const env = replayEnvironment(checkout, phase, executionId);
+            const command = async (executable, args, cwd, timeoutMs = 60_000) => await runVerificationCommand({ id: group.runner.id, command: executable, args: [...args], cwd: ".", required: true,
+                evidence: { kind: "exit-code" }, timeoutMs }, { cwd, runner: options.processRunner, env, replaceEnv: true,
+                outputLimitBytes: 1024 * 1024, ...(options.signal === undefined ? {} : { signal: options.signal }) });
+            let aggregate;
+            let failure = "replay-unavailable";
+            let output;
+            let run;
+            const steps = [];
+            let executionDigest = sha256(executionId);
+            let argv = [];
+            const requested = [...new Set(group.criteria.flatMap(c => c.acceptance.bindings
+                    .filter(b => b.runnerId === group.runner.id && canonicalJson(b.materials) === canonicalJson(group.binding.materials)).flatMap(b => b.checkIds)))];
+            try {
+                for (const [cwd, args] of [[temp, ["clone", "--no-hardlinks", "--quiet", "--", options.root, checkout]],
+                    [checkout, ["fetch", "--quiet", "--", options.root, commit]], [checkout, ["checkout", "--detach", "--quiet", commit]]]) {
+                    const result = await command("git", args, cwd);
+                    if (result.status !== "PASS")
+                        throw new Error("checkout-unavailable");
+                }
+                const cwd = group.runner.cwd === "." ? checkout : await resolveContainedPath(checkout, group.runner.cwd);
+                for (const step of [...(group.runner.setup ?? []), ...(group.runner.build ?? [])]) {
+                    const result = await command(step.command, step.args, cwd, step.timeoutMs);
+                    steps.push({ command: step.command, args: step.args, status: result.status, failureClass: result.failureClass, stderr: result.stderr });
+                    if (result.status !== "PASS")
+                        throw new Error("setup-or-build-unavailable");
+                }
+                // Both sides receive exactly the candidate's explicit test material, after their own build.
+                for (const material of materials) {
+                    let parent = checkout;
+                    for (const segment of material.path.split("/").slice(0, -1)) {
+                        parent = join(parent, segment);
+                        try {
+                            if (!(await lstat(parent)).isDirectory())
+                                throw new Error("unsafe-material-parent");
+                        }
+                        catch (error) {
+                            if (error.code !== "ENOENT")
+                                throw error;
+                            await mkdir(parent);
+                        }
+                    }
+                    const path = join(checkout, ...material.path.split("/"));
+                    try {
+                        if (!(await lstat(path)).isFile())
+                            throw new Error("unsafe-material-target");
+                    }
+                    catch (error) {
+                        if (error.code !== "ENOENT")
+                            throw error;
+                    }
+                    await writeFile(path, material.bytes);
+                    await chmod(path, material.mode);
+                }
+                for (const step of group.runner.testBuild ?? []) {
+                    const result = await command(step.command, step.args, cwd, step.timeoutMs);
+                    steps.push({ command: step.command, args: step.args, status: result.status, failureClass: result.failureClass, stderr: result.stderr });
+                    if (result.status !== "PASS")
+                        throw new Error("test-build-unavailable");
+                }
+                argv = group.runner.args.flatMap(arg => arg === "{materials}" ?
+                    materials.filter(m => m.role === "test").map(m => m.path) : [arg]);
+                const dependencies = [];
+                for (const path of ["package.json", "package-lock.json", "pnpm-lock.yaml", "yarn.lock", "Cargo.toml", "Cargo.lock", "pyproject.toml", "requirements.txt"]) {
+                    try {
+                        dependencies.push({ path, hash: sha256(await readFile(join(checkout, path))) });
+                    }
+                    catch (error) {
+                        if (error.code !== "ENOENT")
+                            throw error;
+                    }
+                }
+                const identity = await executableIdentity(group.runner.command, cwd, env);
+                if (identity === "unavailable")
+                    throw new Error("missing-executable");
+                executionDigest = sha256(canonicalJson({ commit, materialDigest, runner: group.runner, argv, dependencies,
+                    executable: identity, platform: process.platform, arch: process.arch, environment: sha256(canonicalJson(env)) }));
+                output = await command(group.runner.command, argv, cwd, group.runner.timeoutMs);
+                run = collected(group.runner, output, checkout, { executionId, phase,
+                    frameworkVersion: group.runner.adapter === "node" ? process.version : "dependency-lock:" + sha256(canonicalJson(dependencies)) });
+                aggregate = aggregateAcceptanceRun(run, requested);
+                // Process errors never become assertion red, even if stdout claims a result.
+                if (output.exitCode === null || output.status === "UNKNOWN" ||
+                    (aggregate.status === "PASS" && output.exitCode !== 0) || (aggregate.status === "FAIL" && output.exitCode === 0))
+                    aggregate = undefined;
+                failure = aggregate?.failureClass ?? output.failureClass;
+            }
+            catch (error) {
+                failure = error instanceof Error && /^[a-z-]+$/u.test(error.message) ? error.message : "replay-unavailable";
+            }
+            finally {
+                await rm(temp, { recursive: true, force: true });
+            }
+            const finishedAt = new Date().toISOString();
+            const artifactContent = redactSecrets(JSON.stringify({ schemaVersion: 1, executionId, phase, commit,
+                pairedCommit: phase === "baseline" ? candidate : group.baseline, materialDigest, executionDigest,
+                runner: group.runner, argv, run: run ?? null, aggregate: aggregate ?? null, output: output ?? null, steps, failure, startedAt, finishedAt }));
+            const executionArtifact = ".agent-ops/tasks/acceptance/" + sha256(artifactContent) + ".json";
+            await writePrivateFile(join(options.root, ...executionArtifact.split("/")), artifactContent, options.root);
+            for (const criterion of group.criteria) {
+                const bindings = criterion.acceptance.bindings.filter(b => b.runnerId === group.runner.id && canonicalJson(b.materials) === canonicalJson(group.binding.materials));
+                const binding = { checkIds: [...new Set(bindings.flatMap(b => b.checkIds))], redCheckIds: [...new Set(bindings.flatMap(b => b.redCheckIds ?? []))] };
+                const checks = binding.checkIds.map(checkId => {
+                    const check = aggregate?.results.find(result => result.checkId === checkId);
+                    return { checkId, status: check?.status ?? "UNKNOWN",
+                        failureClass: check?.failureClass === "assertion-failed" ? "assertion" : check?.failureClass ?? failure };
+                });
+                const red = binding.redCheckIds ?? [];
+                const status = aggregate === undefined || aggregate.status === "UNKNOWN" ? "UNKNOWN" :
+                    phase === "candidate" ? checks.every(c => c.status === "PASS") ? "PASS" : "FAIL" :
+                        red.every(id => checks.some(c => c.checkId === id && c.status === "FAIL" && c.failureClass === "assertion")) ? "PASS" : "FAIL";
+                const evidence = buildVerificationEvidence({ taskId: options.task.id, criterionId: criterion.id,
+                    command: { id: group.runner.id, command: group.runner.command, args: argv, cwd: group.runner.cwd, required: true, evidence: { kind: "exit-code" } },
+                    scope: options.scope, startedAt, finishedAt, exitCode: output?.exitCode ?? null, testCount: checks.length,
+                    status, failureClass: status === "PASS" ? "none" : phase === "baseline" && aggregate?.status === "PASS" ? "non-discriminating" : failure,
+                    sourceFingerprint: options.sourceFingerprint, toolVersions: { node: process.version }, config: options.config,
+                    taskContractHash: taskContractHash(options.task), acceptance: { criterionContractHash: criterionContractHash(criterion),
+                        bindingHash: sha256(canonicalJson({ runnerId: group.runner.id, materials: group.binding.materials })), phase, commit, pairedCommit: phase === "baseline" ? candidate : group.baseline, materialDigest, executionDigest, executionArtifact, checks } });
+                const reference = await options.evidenceStore.save(evidence);
+                reports.push({ criterionId: criterion.id, runnerId: group.runner.id, phase, status, failureClass: evidence.failureClass, reference });
+            }
+        }
+    }
+    return reports;
+}

package/dist/runtime/src/verify/adapters/index.js

@@ -0,0 +1,5 @@
+export { collectJestAcceptance } from "./jest.js";
+export { collectNodeAcceptance } from "./node.js";
+export { collectPytestAcceptance } from "./pytest.js";
+export { collectRustAcceptance } from "./rust.js";
+export { collectVitestAcceptance } from "./vitest.js";

package/dist/runtime/src/verify/adapters/jest.js

@@ -0,0 +1,142 @@
+import { parseJsonDocument, parseJsonLines, stableCheckId } from "../acceptance-protocol.js";
+import { array, hasRetry, lower, number, object, result, run, string } from "./shared.js";
+function reportFromInput(input) {
+    if (typeof input === "string") {
+        const document = parseJsonDocument(input);
+        if (object(document) !== undefined)
+            return { report: object(document), diagnostics: [] };
+        const lines = parseJsonLines(input);
+        if (lines.values.length === 1 && object(lines.values[0]) !== undefined) {
+            return { report: object(lines.values[0]), diagnostics: lines.invalidLines.map((line) => `invalid-json-line:${line}`) };
+        }
+        return { report: undefined, diagnostics: ["malformed-output", ...lines.invalidLines.map((line) => `invalid-json-line:${line}`)] };
+    }
+    return object(input) === undefined
+        ? { report: undefined, diagnostics: ["malformed-report"] }
+        : { report: object(input), diagnostics: [] };
+}
+function suiteResults(suite) {
+    return array(suite.assertionResults) ?? array(suite.assertions) ?? array(suite.tests) ?? [];
+}
+function suiteFile(suite) {
+    return string(suite.testFilePath ?? suite.filePath ?? suite.file ?? suite.name) ?? "<jest-suite>";
+}
+function assertionId(suite, assertion) {
+    const ancestors = array(assertion.ancestorTitles)
+        ?.filter((item) => typeof item === "string") ?? [];
+    // Jest's fullName already contains ancestorTitles; prefer the leaf title so
+    // real JSON reports and reporter events share one identity format.
+    const name = string(assertion.title ?? assertion.name) ??
+        (ancestors.length === 0 ? string(assertion.fullName) : undefined);
+    if (name === undefined)
+        return undefined;
+    const location = object(assertion.location);
+    const line = number(location?.line);
+    const column = number(location?.column);
+    try {
+        return stableCheckId([
+            suiteFile(suite),
+            ...ancestors,
+            name,
+            ...(line === undefined || column === undefined ? [] : [`@${line}:${column}`])
+        ]);
+    }
+    catch {
+        return undefined;
+    }
+}
+function infrastructure(suite, report) {
+    const present = (value) => value !== undefined && value !== null &&
+        (!Array.isArray(value) || value.length > 0);
+    if (present(suite.testExecError) || present(suite.runtimeError) ||
+        present(report.runExecError) || present(report.unhandledErrors) || present(report.errors) ||
+        report.wasInterrupted === true)
+        return true;
+    const runtimeSuites = number(report.numRuntimeErrorTestSuites);
+    return runtimeSuites !== undefined && runtimeSuites > 0;
+}
+function evidence(suite, id) {
+    const file = suiteFile(suite);
+    return [`jest:assertion:${file}:${id}`];
+}
+/** Collect a final Jest JSON report or an equivalent structured reporter document. */
+export function collectJestAcceptance(input, options) {
+    const parsed = reportFromInput(input);
+    const report = parsed.report;
+    if (report === undefined) {
+        return run(options, "jest", [], false, [], parsed.diagnostics);
+    }
+    const suites = array(report.testResults) ?? array(report.testFiles) ?? array(report.files);
+    if (suites === undefined) {
+        return run(options, "jest", [], false, [], [
+            ...parsed.diagnostics,
+            "missing-test-results"
+        ]);
+    }
+    const results = [];
+    const diagnostics = [...parsed.diagnostics];
+    for (const rawSuite of suites) {
+        const suite = object(rawSuite);
+        if (suite === undefined) {
+            diagnostics.push("malformed-suite");
+            continue;
+        }
+        const suiteHasInfrastructure = infrastructure(suite, report);
+        const assertions = suiteResults(suite);
+        if (assertions.length === 0 && suiteHasInfrastructure) {
+            diagnostics.push(`suite-infrastructure:${suiteFile(suite)}`);
+        }
+        for (const rawAssertion of assertions) {
+            const assertion = object(rawAssertion);
+            if (assertion === undefined) {
+                diagnostics.push(`malformed-assertion:${suiteFile(suite)}`);
+                continue;
+            }
+            const id = assertionId(suite, assertion);
+            if (id === undefined) {
+                diagnostics.push(`missing-check-id:${suiteFile(suite)}`);
+                continue;
+            }
+            const status = lower(assertion.status ?? assertion.state);
+            const observedAttempts = number(assertion.attempts ?? assertion.invocations);
+            const attemptCount = observedAttempts !== undefined && observedAttempts > 0 ? observedAttempts : null;
+            const evidenceRefs = evidence(suite, id);
+            const hooks = object(assertion.hooks);
+            const hookFailure = (Array.isArray(assertion.hookErrors) && assertion.hookErrors.length > 0) ||
+                (assertion.hookErrors !== undefined && assertion.hookErrors !== null && !Array.isArray(assertion.hookErrors)) ||
+                (hooks !== undefined && Object.values(hooks).some(state => state !== "pass" && state !== "passed"));
+            if (hasRetry(assertion)) {
+                results.push(result(options, "jest", id, "UNKNOWN", evidenceRefs, "retry", attemptCount));
+            }
+            else if (suiteHasInfrastructure) {
+                results.push(result(options, "jest", id, "UNKNOWN", evidenceRefs, "infrastructure-error", attemptCount));
+            }
+            else if (hookFailure) {
+                results.push(result(options, "jest", id, "UNKNOWN", evidenceRefs, "hook-error", attemptCount));
+            }
+            else if (status === "passed" || status === "pass") {
+                results.push(result(options, "jest", id, "PASS", evidenceRefs, "none", attemptCount));
+            }
+            else if (status === "failed" || status === "fail") {
+                // Native Jest JSON does not distinguish a test-call assertion from an
+                // assertion thrown by before/after hooks. Only classified collector
+                // metadata can establish behavioral red; failureMessages cannot.
+                const assertionFailure = assertion.assertion === true || assertion.failureClass === "assertion-failed";
+                results.push(assertionFailure
+                    ? result(options, "jest", id, "FAIL", evidenceRefs, "assertion-failed", attemptCount)
+                    : result(options, "jest", id, "UNKNOWN", evidenceRefs, "infrastructure-error", attemptCount));
+            }
+            else if (status === "pending" || status === "todo" || status === "skipped" || status === "disabled") {
+                results.push(result(options, "jest", id, "UNKNOWN", evidenceRefs, status === "todo" ? "todo" : "skipped", attemptCount));
+            }
+            else {
+                results.push(result(options, "jest", id, "UNKNOWN", evidenceRefs, "malformed-output", attemptCount));
+            }
+        }
+    }
+    if ((report.success === false || (number(report.numFailedTestSuites) ?? 0) > 0) &&
+        !results.some(item => item.status !== "PASS"))
+        diagnostics.push("unexplained-failed-report");
+    const completed = report.completed !== false && report.interrupted !== true;
+    return run(options, "jest", results, completed, completed ? ["jest:report:complete"] : [], diagnostics);
+}

package/dist/runtime/src/verify/adapters/node-reporter.js

@@ -0,0 +1,20 @@
+/** JSONL reporter for node --test; preserves assertion Error metadata. */
+function errorData(value, seen = new Set()) {
+    if (value === null || typeof value !== "object")
+        return value;
+    if (seen.has(value))
+        return "[circular]";
+    seen.add(value);
+    if (value instanceof Error)
+        return { name: value.name,
+            code: value.code,
+            failureType: value.failureType,
+            cause: errorData(value.cause, seen) };
+    if (Array.isArray(value))
+        return value.map(item => errorData(item, seen));
+    return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, errorData(item, seen)]));
+}
+export default async function* nodeAcceptanceReporter(events) {
+    for await (const event of events)
+        yield JSON.stringify(errorData(event)) + "\n";
+}