@h9-foundry/agentforge-cli 0.7.1 → 0.8.0

package/dist/index.js

@@ -2,11 +2,11 @@ import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSy
 import { join } from "node:path";
 import { execFileSync } from "node:child_process";
 import yaml from "js-yaml";
-import { renderAuditBundleMarkdown } from "@h9-foundry/agentforge-audit";
+import { buildAuditBundle, createAuditEntry, renderAuditBundleMarkdown } from "@h9-foundry/agentforge-audit";
 import { createWorkflowState, findWorkspaceRoot } from "@h9-foundry/agentforge-context-engine";
 import { createPolicyEngine, loadPolicyDocument, resolvePolicy } from "@h9-foundry/agentforge-policy-engine";
 import { runWorkflow } from "@h9-foundry/agentforge-runtime";
-import { agentforgeConfigSchema, auditBundleSchema, designArtifactSchema, designRequestSchema, implementationRequestSchema, incidentRequestSchema, maintenanceRequestSchema, planningArtifactSchema, planningRequestSchema, qaRequestSchema, releaseRequestSchema, securityRequestSchema, workflowDefinitionSchema } from "@h9-foundry/agentforge-schemas";
+import { agentforgeConfigSchema, auditBundleSchema, benchmarkArtifactSchema, designArtifactSchema, designRequestSchema, evalArtifactSchema, evalFixtureCorpusSchema, implementationRequestSchema, incidentRequestSchema, maintenanceRequestSchema, planningArtifactSchema, planningRequestSchema, qaRequestSchema, releaseRequestSchema, schemaFixtures, securityRequestSchema, workflowDefinitionSchema } from "@h9-foundry/agentforge-schemas";
 import { createBuiltinAdapters } from "./internal/builtin-adapters.js";
 import { createBuiltinAgentRegistry } from "./internal/builtin-agents.js";
 import { LocalPluginRegistry } from "./internal/local-plugin-registry.js";
@@ -856,6 +856,15 @@ function readLatestCompleteRunBundle(runsRoot) {
         if (typeof value !== "string" || value.length === 0) {
             return undefined;
         }
+        const compactDateTimeMatch = value.match(/^(\d{4})-(\d{2})-(\d{2})-(\d{2})(\d{2})(\d{2})$/);
+        if (compactDateTimeMatch) {
+            const [, year, month, day, hour, minute, second] = compactDateTimeMatch;
+            const isoCandidate = `${year}-${month}-${day}T${hour}:${minute}:${second}Z`;
+            const parsedCompactDateTime = Date.parse(isoCandidate);
+            if (!Number.isNaN(parsedCompactDateTime)) {
+                return parsedCompactDateTime;
+            }
+        }
         const parsedDate = Date.parse(value);
         if (!Number.isNaN(parsedDate)) {
             return parsedDate;
@@ -872,20 +881,24 @@ function readLatestCompleteRunBundle(runsRoot) {
         const stats = statSync(bundlePath);
         const bundle = JSON.parse(readFileSync(bundlePath, "utf8"));
         const bundleRunId = typeof bundle.runId === "string" ? bundle.runId : entry;
-        const completedAtMs = parseRunTimestampMs(bundle.finishedAt) ??
+        const parsedCompletedAtMs = parseRunTimestampMs(bundle.finishedAt) ??
             parseRunTimestampMs(bundle.startedAt) ??
             parseRunTimestampMs(bundleRunId) ??
-            parseRunTimestampMs(entry) ??
-            stats.mtimeMs;
+            parseRunTimestampMs(entry);
+        const completedAtMs = parsedCompletedAtMs ?? stats.mtimeMs;
         return {
             runDir: entry,
             bundle,
             bundleRunId,
-            completedAtMs
+            completedAtMs,
+            hasExplicitTimestamp: typeof parsedCompletedAtMs === "number"
         };
     })
         .filter((candidate) => Boolean(candidate))
         .sort((left, right) => {
+        if (left.hasExplicitTimestamp !== right.hasExplicitTimestamp) {
+            return left.hasExplicitTimestamp ? -1 : 1;
+        }
         if (left.completedAtMs !== right.completedAtMs) {
             return right.completedAtMs - left.completedAtMs;
         }
@@ -893,6 +906,29 @@ function readLatestCompleteRunBundle(runsRoot) {
     });
     return candidates[0] ? { runDir: candidates[0].runDir, bundle: candidates[0].bundle } : undefined;
 }
+function readRunBundleByRef(root, runRef) {
+    const config = loadAgentForgeConfig(root);
+    const runsRoot = join(root, config.runtime.runsPath);
+    const bundlePath = runRef.endsWith(".json") || runRef.includes("/")
+        ? (runRef.startsWith("/") ? runRef : join(root, runRef))
+        : join(runsRoot, runRef, "bundle.json");
+    if (!existsSync(bundlePath)) {
+        throw new Error(`Run bundle not found: ${runRef}`);
+    }
+    const bundle = auditBundleSchema.parse(JSON.parse(readFileSync(bundlePath, "utf8")));
+    return {
+        runId: typeof bundle.runId === "string" ? bundle.runId : runRef,
+        bundlePath,
+        bundle
+    };
+}
+function extractEvalArtifact(bundle, runRef) {
+    const artifact = bundle.lifecycleArtifacts.find((candidate) => candidate.artifactKind === "eval-result");
+    if (!artifact) {
+        throw new Error(`Run ${runRef} does not contain an eval-result artifact.`);
+    }
+    return evalArtifactSchema.parse(artifact);
+}
 function loadAgentForgeConfig(root) {
     const configPath = join(root, ".agentops", "agentops.yaml");
     if (!existsSync(configPath)) {
@@ -921,6 +957,425 @@ function loadAgentForgeConfig(root) {
 function ensureDirectory(pathValue) {
     mkdirSync(pathValue, { recursive: true });
 }
+function writeYamlFile(filePath, value) {
+    writeFileSync(filePath, yaml.dump(value), "utf8");
+}
+function loadEvalFixtureCorpus() {
+    return evalFixtureCorpusSchema.parse(schemaFixtures.evalFixtureCorpus);
+}
+function getEvalSpec(specId) {
+    const corpus = loadEvalFixtureCorpus();
+    const spec = corpus.specs.find((candidate) => candidate.id === specId);
+    if (!spec) {
+        throw new Error(`Unknown eval spec: ${specId}`);
+    }
+    return spec;
+}
+function toBundleRef(run) {
+    return `.agentops/runs/${run.runId}/bundle.json`;
+}
+function toSummaryRef(run) {
+    return `.agentops/runs/${run.runId}/summary.md`;
+}
+function toSetupRun(workflow, run) {
+    return {
+        workflow,
+        runId: run.runId,
+        bundlePath: toBundleRef(run)
+    };
+}
+function createBlankEvalWorkspace(root, evalRunId, specId) {
+    const workspaceRoot = join(root, ".agentops", "evals", specId, evalRunId, "workspace");
+    ensureDirectory(workspaceRoot);
+    const evidenceRoot = join(workspaceRoot, ".agentops", "evidence");
+    ensureDirectory(evidenceRoot);
+    execFileSync("git", ["init"], { cwd: workspaceRoot, stdio: "ignore" });
+    execFileSync("git", ["config", "user.email", "eval@example.com"], { cwd: workspaceRoot, stdio: "ignore" });
+    execFileSync("git", ["config", "user.name", "AgentForge Eval"], { cwd: workspaceRoot, stdio: "ignore" });
+    writeFileSync(join(workspaceRoot, "package.json"), JSON.stringify({
+        name: "fixture",
+        repository: {
+            type: "git",
+            url: "https://github.com/H9-Foundry/fixture.git"
+        },
+        scripts: {
+            test: "echo test",
+            lint: "echo lint",
+            typecheck: "echo typecheck",
+            build: "echo build"
+        }
+    }, null, 2), "utf8");
+    writeFileSync(join(workspaceRoot, "pnpm-lock.yaml"), "lockfileVersion: '9.0'\n", "utf8");
+    writeFileSync(join(workspaceRoot, "src.ts"), "export const value = 1;\n", "utf8");
+    writeFileSync(join(evidenceRoot, "dependency-alerts.json"), JSON.stringify({
+        alerts: [
+            {
+                package: "example-dependency",
+                severity: "moderate",
+                summary: "Upgrade pending review for deterministic eval coverage."
+            }
+        ]
+    }, null, 2), "utf8");
+    writeFileSync(join(evidenceRoot, "docs-task.md"), "# Docs follow-up\n\n- Align workflow documentation after maintenance triage.\n", "utf8");
+    execFileSync("git", ["add", "."], { cwd: workspaceRoot, stdio: "ignore" });
+    execFileSync("git", ["-c", "commit.gpgsign=false", "commit", "-m", "init"], { cwd: workspaceRoot, stdio: "ignore" });
+    writeFileSync(join(workspaceRoot, "src.ts"), "export const value = 2;\n", "utf8");
+    initProject(workspaceRoot);
+    return workspaceRoot;
+}
+function evalRedactionCategories() {
+    return ["github-token", "api-key", "aws-key", "bearer-token", "password", "private-key"];
+}
+function createEvalBundle(root, spec, evaluatedRun, workspacePath, setupRuns, deterministicChecks, modelDependentChecks) {
+    const config = loadAgentForgeConfig(root);
+    const policy = resolvePolicy(loadPolicyDocument(join(root, ".agentops", "policy.yaml")), process.env.CI ? "ci" : "local");
+    const state = createWorkflowState({
+        cwd: root,
+        workflow: `eval:${spec.id}`,
+        mode: "inspect",
+        policy
+    });
+    const runsRoot = join(root, config.runtime.runsPath);
+    const outputDir = join(runsRoot, state.runId);
+    ensureDirectory(outputDir);
+    const jsonPath = join(outputDir, "bundle.json");
+    const markdownPath = join(outputDir, "summary.md");
+    const failureCount = deterministicChecks.filter((check) => check.status === "failed").length;
+    const passed = failureCount === 0;
+    const startedAt = new Date().toISOString();
+    const evalArtifact = evalArtifactSchema.parse({
+        schemaVersion: state.version,
+        artifactKind: "eval-result",
+        lifecycleDomain: "evaluate",
+        workflow: {
+            name: state.workflow,
+            displayName: "Eval Runner"
+        },
+        source: {
+            sourceType: "workflow-run",
+            runId: state.runId,
+            inputRefs: [
+                ...(evaluatedRun?.jsonPath ? [evaluatedRun.jsonPath] : []),
+                ...setupRuns.map((setup) => setup.bundlePath)
+            ],
+            issueRefs: ["#165"],
+            githubRefs: []
+        },
+        status: passed ? "complete" : "draft",
+        generatedAt: startedAt,
+        repo: {
+            root: state.repo.root,
+            name: state.repo.name,
+            branch: state.repo.branch
+        },
+        provenance: {
+            generatedBy: "agentforge-runtime",
+            schemaVersion: state.version,
+            executionEnvironment: state.context.ciExecution ? "ci" : "local",
+            repoRoot: state.repo.root
+        },
+        redaction: {
+            applied: true,
+            strategyVersion: "1.0.0",
+            categories: evalRedactionCategories()
+        },
+        auditLink: {
+            bundlePath: jsonPath,
+            entryIds: [`${state.runId}-eval-runner`],
+            findingIds: [],
+            proposedActionIds: []
+        },
+        summary: passed
+            ? `Eval result for ${spec.id} passed ${deterministicChecks.length} deterministic check(s).`
+            : `Eval result for ${spec.id} failed ${failureCount} deterministic check(s).`,
+        payload: {
+            specId: spec.id,
+            specName: spec.name,
+            workflow: spec.workflow,
+            repoFixture: spec.repoFixture,
+            workspacePath,
+            evaluatedRunId: evaluatedRun?.runId,
+            evaluatedBundlePath: evaluatedRun ? toBundleRef(evaluatedRun) : undefined,
+            setupRuns,
+            deterministicChecks,
+            modelDependentChecks,
+            passed,
+            failureCount,
+            warningCount: 0
+        }
+    });
+    state.lifecycleArtifacts = [evalArtifact];
+    state.auditTrail = [
+        createAuditEntry({
+            id: `${state.runId}-eval-runner`,
+            nodeId: "eval-runner",
+            nodeName: "eval-runner",
+            kind: "deterministic",
+            startedAt,
+            completedAt: new Date().toISOString(),
+            status: passed ? "success" : "failed",
+            summary: evalArtifact.summary,
+            toolsRequested: [],
+            toolsExecuted: [],
+            blockedActions: [],
+            validationPassed: passed
+        }),
+        createAuditEntry({
+            id: `${state.runId}-report`,
+            nodeId: "report",
+            nodeName: "final-report",
+            kind: "report",
+            startedAt,
+            completedAt: new Date().toISOString(),
+            status: "success",
+            summary: "Generated eval result artifacts.",
+            toolsRequested: [],
+            toolsExecuted: [],
+            blockedActions: [],
+            validationPassed: true
+        })
+    ];
+    const bundle = buildAuditBundle(state, {
+        startedAt,
+        finishedAt: new Date().toISOString(),
+        status: passed ? "success" : "partial",
+        jsonPath,
+        markdownPath,
+        provenance: {
+            generatedBy: "agentforge-runtime",
+            schemaVersion: state.version,
+            executionEnvironment: state.context.ciExecution ? "ci" : "local",
+            repoRoot: state.repo.root
+        },
+        redaction: {
+            applied: true,
+            strategyVersion: "1.0.0",
+            categories: evalRedactionCategories()
+        },
+        components: []
+    });
+    writeFileSync(jsonPath, JSON.stringify(bundle, null, 2), "utf8");
+    writeFileSync(markdownPath, renderAuditBundleMarkdown(bundle), "utf8");
+    return { bundle, jsonPath, markdownPath, outputDir };
+}
+function compareDeterministicChecks(baselineChecks, candidateChecks) {
+    const regressions = [];
+    const improvements = [];
+    const nonComparableFindings = [];
+    let unchangedCount = 0;
+    const baselineByName = new Map(baselineChecks.map((check) => [check.name, check]));
+    const candidateByName = new Map(candidateChecks.map((check) => [check.name, check]));
+    const checkNames = [...new Set([...baselineByName.keys(), ...candidateByName.keys()])].sort();
+    for (const name of checkNames) {
+        const baselineCheck = baselineByName.get(name);
+        const candidateCheck = candidateByName.get(name);
+        if (!baselineCheck || !candidateCheck) {
+            nonComparableFindings.push(`Deterministic check \`${name}\` is missing from one of the eval results.`);
+            continue;
+        }
+        if (baselineCheck.status === candidateCheck.status) {
+            unchangedCount += 1;
+            continue;
+        }
+        if (baselineCheck.status === "not_applicable" || candidateCheck.status === "not_applicable") {
+            nonComparableFindings.push(`Deterministic check \`${name}\` changed between comparable and not_applicable states (${baselineCheck.status} -> ${candidateCheck.status}).`);
+            continue;
+        }
+        if (baselineCheck.status === "passed" && candidateCheck.status === "failed") {
+            regressions.push({
+                name,
+                classification: "regression",
+                baselineStatus: baselineCheck.status,
+                candidateStatus: candidateCheck.status,
+                details: candidateCheck.details ?? baselineCheck.details
+            });
+            continue;
+        }
+        if (baselineCheck.status === "failed" && candidateCheck.status === "passed") {
+            improvements.push({
+                name,
+                classification: "improvement",
+                baselineStatus: baselineCheck.status,
+                candidateStatus: candidateCheck.status,
+                details: candidateCheck.details ?? baselineCheck.details
+            });
+            continue;
+        }
+        nonComparableFindings.push(`Deterministic check \`${name}\` changed in an unsupported way (${baselineCheck.status} -> ${candidateCheck.status}).`);
+    }
+    return { regressions, improvements, unchangedCount, nonComparableFindings };
+}
+function compareEvalArtifacts(baselineRunId, baselineBundlePath, baselineArtifact, candidateRunId, candidateBundlePath, candidateArtifact) {
+    if (baselineArtifact.payload.specId !== candidateArtifact.payload.specId) {
+        return {
+            runId: candidateRunId,
+            bundlePath: candidateBundlePath,
+            specId: candidateArtifact.payload.specId,
+            workflow: candidateArtifact.payload.workflow,
+            comparable: false,
+            passed: candidateArtifact.payload.passed,
+            failureCount: candidateArtifact.payload.failureCount,
+            deterministicCheckCount: candidateArtifact.payload.deterministicChecks.length,
+            regressions: [],
+            improvements: [],
+            unchangedCount: 0,
+            nonComparableFindings: [
+                `Spec mismatch: baseline ${baselineArtifact.payload.specId} vs candidate ${candidateArtifact.payload.specId}.`
+            ]
+        };
+    }
+    if (baselineArtifact.payload.workflow !== candidateArtifact.payload.workflow) {
+        return {
+            runId: candidateRunId,
+            bundlePath: candidateBundlePath,
+            specId: candidateArtifact.payload.specId,
+            workflow: candidateArtifact.payload.workflow,
+            comparable: false,
+            passed: candidateArtifact.payload.passed,
+            failureCount: candidateArtifact.payload.failureCount,
+            deterministicCheckCount: candidateArtifact.payload.deterministicChecks.length,
+            regressions: [],
+            improvements: [],
+            unchangedCount: 0,
+            nonComparableFindings: [
+                `Workflow mismatch: baseline ${baselineArtifact.payload.workflow} vs candidate ${candidateArtifact.payload.workflow}.`
+            ]
+        };
+    }
+    const comparison = compareDeterministicChecks(baselineArtifact.payload.deterministicChecks, candidateArtifact.payload.deterministicChecks);
+    return {
+        runId: candidateRunId,
+        bundlePath: candidateBundlePath,
+        specId: candidateArtifact.payload.specId,
+        workflow: candidateArtifact.payload.workflow,
+        comparable: comparison.nonComparableFindings.length === 0,
+        passed: candidateArtifact.payload.passed,
+        failureCount: candidateArtifact.payload.failureCount,
+        deterministicCheckCount: candidateArtifact.payload.deterministicChecks.length,
+        regressions: comparison.regressions,
+        improvements: comparison.improvements,
+        unchangedCount: comparison.unchangedCount,
+        nonComparableFindings: comparison.nonComparableFindings
+    };
+}
+function createBenchmarkBundle(root, baselineRunId, baselineBundlePath, baselineArtifact, comparedRuns) {
+    const config = loadAgentForgeConfig(root);
+    const policy = resolvePolicy(loadPolicyDocument(join(root, ".agentops", "policy.yaml")), process.env.CI ? "ci" : "local");
+    const state = createWorkflowState({
+        cwd: root,
+        workflow: "eval:compare",
+        mode: "inspect",
+        policy
+    });
+    const runsRoot = join(root, config.runtime.runsPath);
+    const outputDir = join(runsRoot, state.runId);
+    ensureDirectory(outputDir);
+    const jsonPath = join(outputDir, "bundle.json");
+    const markdownPath = join(outputDir, "summary.md");
+    const regressionCount = comparedRuns.reduce((total, candidate) => total + candidate.regressions.length, 0);
+    const improvementCount = comparedRuns.reduce((total, candidate) => total + candidate.improvements.length, 0);
+    const unchangedCount = comparedRuns.reduce((total, candidate) => total + candidate.unchangedCount, 0);
+    const nonComparableCount = comparedRuns.reduce((total, candidate) => total + candidate.nonComparableFindings.length, 0);
+    const summaryConclusion = regressionCount > 0
+        ? `Detected ${regressionCount} deterministic regression(s) across compared eval results.`
+        : improvementCount > 0
+            ? `Detected ${improvementCount} deterministic improvement(s) with no regressions.`
+            : nonComparableCount > 0
+                ? `Compared eval results contain ${nonComparableCount} non-comparable difference(s) and no deterministic regressions.`
+                : "No deterministic regressions detected across compared eval results.";
+    const benchmarkArtifact = benchmarkArtifactSchema.parse({
+        schemaVersion: state.version,
+        artifactKind: "benchmark-summary",
+        lifecycleDomain: "evaluate",
+        workflow: {
+            name: state.workflow,
+            displayName: "Eval Benchmark Compare"
+        },
+        source: {
+            sourceType: "workflow-run",
+            runId: state.runId,
+            inputRefs: [baselineBundlePath, ...comparedRuns.map((candidate) => candidate.bundlePath)],
+            issueRefs: ["#166"],
+            githubRefs: []
+        },
+        status: "complete",
+        generatedAt: new Date().toISOString(),
+        repo: {
+            root: state.repo.root,
+            name: state.repo.name,
+            branch: state.repo.branch
+        },
+        provenance: {
+            generatedBy: "agentforge-runtime",
+            schemaVersion: state.version,
+            executionEnvironment: state.context.ciExecution ? "ci" : "local",
+            repoRoot: state.repo.root
+        },
+        redaction: {
+            applied: true,
+            strategyVersion: "1.0.0",
+            categories: evalRedactionCategories()
+        },
+        auditLink: {
+            bundlePath: jsonPath,
+            entryIds: [`${state.runId}-benchmark-compare`],
+            findingIds: [],
+            proposedActionIds: []
+        },
+        summary: summaryConclusion,
+        payload: {
+            baselineRunId,
+            baselineBundlePath,
+            baselineSpecId: baselineArtifact.payload.specId,
+            baselineWorkflow: baselineArtifact.payload.workflow,
+            comparedRuns,
+            regressionCount,
+            improvementCount,
+            unchangedCount,
+            nonComparableCount,
+            summaryConclusion
+        }
+    });
+    state.lifecycleArtifacts = [benchmarkArtifact];
+    state.auditTrail = [
+        createAuditEntry({
+            id: `${state.runId}-benchmark-compare`,
+            nodeId: "benchmark-compare",
+            nodeName: "benchmark-compare",
+            kind: "deterministic",
+            startedAt: new Date().toISOString(),
+            completedAt: new Date().toISOString(),
+            status: regressionCount > 0 ? "failed" : "success",
+            summary: benchmarkArtifact.summary,
+            toolsRequested: [],
+            toolsExecuted: [],
+            blockedActions: [],
+            validationPassed: regressionCount === 0
+        })
+    ];
+    const bundle = buildAuditBundle(state, {
+        startedAt: new Date().toISOString(),
+        finishedAt: new Date().toISOString(),
+        status: regressionCount > 0 || nonComparableCount > 0 ? "partial" : "success",
+        jsonPath,
+        markdownPath,
+        provenance: {
+            generatedBy: "agentforge-runtime",
+            schemaVersion: state.version,
+            executionEnvironment: state.context.ciExecution ? "ci" : "local",
+            repoRoot: state.repo.root
+        },
+        redaction: {
+            applied: true,
+            strategyVersion: "1.0.0",
+            categories: evalRedactionCategories()
+        },
+        components: []
+    });
+    writeFileSync(jsonPath, JSON.stringify(bundle, null, 2), "utf8");
+    writeFileSync(markdownPath, renderAuditBundleMarkdown(bundle), "utf8");
+    return { bundle, jsonPath, markdownPath, outputDir };
+}
 function ensureInitFiles(root) {
     const created = [];
     const configDir = join(root, ".agentops");
@@ -1121,6 +1576,261 @@ export async function runLocalWorkflow(workflowName, cwd = process.cwd()) {
         artifactKinds: bundle.lifecycleArtifacts.map((artifact) => artifact.artifactKind)
     };
 }
+function checkResult(status, name, expected, actual, details) {
+    return {
+        name,
+        status,
+        expected,
+        actual,
+        ...(details ? { details } : {})
+    };
+}
+function compareEvalSpec(spec, bundle, executionError) {
+    const checks = [];
+    if (!bundle) {
+        checks.push(checkResult("failed", "workflow-execution", "successful workflow execution", executionError ?? "unknown failure", "The eval runner could not produce an evaluated workflow bundle."));
+        return {
+            deterministicChecks: checks,
+            modelDependentChecks: [
+                {
+                    name: "rubric-scoring",
+                    status: "not_executed",
+                    details: "Provider-dependent scoring is out of scope for the first local eval runner slice."
+                }
+            ]
+        };
+    }
+    checks.push(checkResult(bundle.status === spec.expectedStatus ? "passed" : "failed", "run-status", spec.expectedStatus, bundle.status, "The evaluated workflow status should match the deterministic eval spec."));
+    checks.push(checkResult(bundle.redaction.applied === spec.redactionExpectations.applied ? "passed" : "failed", "redaction-applied", String(spec.redactionExpectations.applied), String(bundle.redaction.applied)));
+    for (const category of spec.redactionExpectations.expectedCategories) {
+        checks.push(checkResult(bundle.redaction.categories.includes(category) ? "passed" : "failed", `redaction-category:${category}`, category, bundle.redaction.categories.join(", ")));
+    }
+    checks.push(checkResult(bundle.policy.defaults.executionMode === spec.policyExpectations.executionMode ? "passed" : "failed", "policy-execution-mode", spec.policyExpectations.executionMode, bundle.policy.defaults.executionMode));
+    if (spec.policyExpectations.readOnly) {
+        checks.push(checkResult(bundle.policy.defaults.writes !== "allow" ? "passed" : "failed", "policy-read-only", "writes not equal allow", bundle.policy.defaults.writes));
+    }
+    for (const sideEffectClass of spec.policyExpectations.sideEffectClasses) {
+        checks.push(checkResult("not_applicable", `side-effect-class:${sideEffectClass}`, sideEffectClass, undefined, "The first eval runner records policy posture and workflow outputs but does not inspect adapter-level side-effect execution traces."));
+    }
+    for (const expectedArtifact of spec.artifactExpectations) {
+        const actualArtifact = bundle.lifecycleArtifacts.find((artifact) => artifact.artifactKind === expectedArtifact.artifactKind);
+        checks.push(checkResult(actualArtifact ? "passed" : "failed", `artifact-kind:${expectedArtifact.artifactKind}`, expectedArtifact.artifactKind, actualArtifact?.artifactKind));
+        if (!actualArtifact || typeof actualArtifact.payload !== "object" || actualArtifact.payload === null) {
+            continue;
+        }
+        const payload = actualArtifact.payload;
+        for (const field of expectedArtifact.requiredPayloadFields) {
+            checks.push(checkResult(field in payload ? "passed" : "failed", `payload-field:${expectedArtifact.artifactKind}:${field}`, field, Object.keys(payload).join(", ")));
+        }
+        for (const term of expectedArtifact.requiredSummaryTerms) {
+            const summary = actualArtifact.summary.toLowerCase();
+            checks.push(checkResult(summary.includes(term.toLowerCase()) ? "passed" : "failed", `summary-term:${expectedArtifact.artifactKind}:${term}`, term, actualArtifact.summary));
+        }
+    }
+    if (spec.artifactExpectations.length === 0) {
+        checks.push(checkResult(bundle.lifecycleArtifacts.length === 0 ? "passed" : "failed", "artifact-count", "0", String(bundle.lifecycleArtifacts.length)));
+    }
+    return {
+        deterministicChecks: checks,
+        modelDependentChecks: [
+            {
+                name: "rubric-scoring",
+                status: "not_executed",
+                details: "Provider-dependent scoring is out of scope for the first local eval runner slice."
+            }
+        ]
+    };
+}
+async function executeEvalWorkflow(spec, workspaceRoot) {
+    const setupRuns = [];
+    const requestsRoot = join(workspaceRoot, ".agentops", "requests");
+    ensureDirectory(requestsRoot);
+    const runPlanning = async () => {
+        writeYamlFile(join(requestsRoot, "planning.yaml"), schemaFixtures.planningRequest);
+        return runLocalWorkflow("planning-discovery", workspaceRoot);
+    };
+    const runDesign = async () => {
+        const planningRun = await runPlanning();
+        setupRuns.push(toSetupRun("planning-discovery", planningRun));
+        writeYamlFile(join(requestsRoot, "design.yaml"), {
+            ...schemaFixtures.designRequest,
+            planningBriefRef: toBundleRef(planningRun)
+        });
+        return runLocalWorkflow("architecture-design-review", workspaceRoot);
+    };
+    const runImplementation = async () => {
+        const designRun = await runDesign();
+        setupRuns.push(toSetupRun("architecture-design-review", designRun));
+        writeYamlFile(join(requestsRoot, "implementation.yaml"), {
+            ...schemaFixtures.implementationRequest,
+            designRecordRef: toBundleRef(designRun)
+        });
+        return runLocalWorkflow("implementation-proposal", workspaceRoot);
+    };
+    const runQa = async () => {
+        const implementationRun = await runImplementation();
+        setupRuns.push(toSetupRun("implementation-proposal", implementationRun));
+        writeYamlFile(join(requestsRoot, "qa.yaml"), {
+            ...schemaFixtures.qaRequest,
+            targetRef: toBundleRef(implementationRun),
+            evidenceSources: [toSummaryRef(implementationRun)]
+        });
+        return runLocalWorkflow("qa-review", workspaceRoot);
+    };
+    const runSecurity = async () => {
+        const qaRun = await runQa();
+        setupRuns.push(toSetupRun("qa-review", qaRun));
+        writeYamlFile(join(requestsRoot, "security.yaml"), {
+            ...schemaFixtures.securityRequest,
+            targetRef: toBundleRef(qaRun),
+            evidenceSources: [toSummaryRef(qaRun)]
+        });
+        return runLocalWorkflow("security-review", workspaceRoot);
+    };
+    const runRelease = async () => {
+        const securityRun = await runSecurity();
+        setupRuns.push(toSetupRun("security-review", securityRun));
+        const qaRun = setupRuns.find((run) => run.workflow === "qa-review");
+        if (!qaRun) {
+            throw new Error("QA setup run was not recorded before release eval execution.");
+        }
+        writeYamlFile(join(requestsRoot, "release.yaml"), {
+            ...schemaFixtures.releaseRequest,
+            qaReportRefs: [qaRun.bundlePath],
+            securityReportRefs: [toBundleRef(securityRun)],
+            evidenceSources: [toSummaryRef(securityRun)]
+        });
+        return runLocalWorkflow("release-readiness", workspaceRoot);
+    };
+    switch (spec.workflow) {
+        case "pr-review":
+            return { evaluatedRun: await runLocalWorkflow("pr-review", workspaceRoot), setupRuns };
+        case "planning-discovery":
+            writeYamlFile(join(requestsRoot, "planning.yaml"), spec.request);
+            return { evaluatedRun: await runLocalWorkflow("planning-discovery", workspaceRoot), setupRuns };
+        case "architecture-design-review": {
+            const planningRun = await runPlanning();
+            setupRuns.push(toSetupRun("planning-discovery", planningRun));
+            writeYamlFile(join(requestsRoot, "design.yaml"), {
+                ...spec.request,
+                planningBriefRef: toBundleRef(planningRun)
+            });
+            return { evaluatedRun: await runLocalWorkflow("architecture-design-review", workspaceRoot), setupRuns };
+        }
+        case "implementation-proposal": {
+            const designRun = await runDesign();
+            setupRuns.push(toSetupRun("architecture-design-review", designRun));
+            writeYamlFile(join(requestsRoot, "implementation.yaml"), {
+                ...spec.request,
+                designRecordRef: toBundleRef(designRun)
+            });
+            return { evaluatedRun: await runLocalWorkflow("implementation-proposal", workspaceRoot), setupRuns };
+        }
+        case "qa-review": {
+            const implementationRun = await runImplementation();
+            setupRuns.push(toSetupRun("implementation-proposal", implementationRun));
+            writeYamlFile(join(requestsRoot, "qa.yaml"), {
+                ...spec.request,
+                targetRef: toBundleRef(implementationRun),
+                evidenceSources: [toSummaryRef(implementationRun)]
+            });
+            return { evaluatedRun: await runLocalWorkflow("qa-review", workspaceRoot), setupRuns };
+        }
+        case "security-review": {
+            const qaRun = await runQa();
+            setupRuns.push(toSetupRun("qa-review", qaRun));
+            writeYamlFile(join(requestsRoot, "security.yaml"), {
+                ...spec.request,
+                targetRef: toBundleRef(qaRun),
+                evidenceSources: [toSummaryRef(qaRun)]
+            });
+            return { evaluatedRun: await runLocalWorkflow("security-review", workspaceRoot), setupRuns };
+        }
+        case "maintenance-triage": {
+            const releaseRun = await runRelease();
+            setupRuns.push(toSetupRun("release-readiness", releaseRun));
+            writeYamlFile(join(requestsRoot, "maintenance.yaml"), {
+                ...spec.request,
+                releaseReportRefs: [toBundleRef(releaseRun)]
+            });
+            return { evaluatedRun: await runLocalWorkflow("maintenance-triage", workspaceRoot), setupRuns };
+        }
+    }
+}
+export async function runLocalEval(specId, cwd = process.cwd()) {
+    const root = findWorkspaceRoot(cwd);
+    ensureInitFiles(root);
+    const spec = getEvalSpec(specId);
+    const controlPolicy = resolvePolicy(loadPolicyDocument(join(root, ".agentops", "policy.yaml")), process.env.CI ? "ci" : "local");
+    const controlState = createWorkflowState({
+        cwd: root,
+        workflow: `eval:${spec.id}`,
+        mode: controlPolicy.defaults.executionMode,
+        policy: controlPolicy
+    });
+    const workspaceRoot = spec.repoFixture === "agentforge-monorepo" ? root : createBlankEvalWorkspace(root, controlState.runId, spec.id);
+    let evaluatedRun;
+    let setupRuns = [];
+    let executionError;
+    try {
+        const result = await executeEvalWorkflow(spec, workspaceRoot);
+        evaluatedRun = result.evaluatedRun;
+        setupRuns = result.setupRuns;
+    }
+    catch (error) {
+        executionError = error instanceof Error ? error.message : String(error);
+    }
+    const evaluatedBundle = evaluatedRun && existsSync(evaluatedRun.jsonPath)
+        ? auditBundleSchema.parse(JSON.parse(readFileSync(evaluatedRun.jsonPath, "utf8")))
+        : undefined;
+    const { deterministicChecks, modelDependentChecks } = compareEvalSpec(spec, evaluatedBundle, executionError);
+    const { bundle, jsonPath, markdownPath, outputDir } = createEvalBundle(root, spec, evaluatedRun, workspaceRoot, setupRuns, deterministicChecks, modelDependentChecks);
+    return {
+        runId: bundle.runId,
+        specId: spec.id,
+        workflow: spec.workflow,
+        outputDir,
+        jsonPath,
+        markdownPath,
+        status: bundle.status,
+        evaluatedRunId: evaluatedRun?.runId,
+        evaluatedBundlePath: evaluatedRun ? toBundleRef(evaluatedRun) : undefined,
+        setupRunCount: setupRuns.length,
+        deterministicCheckCount: deterministicChecks.length,
+        deterministicFailures: deterministicChecks.filter((check) => check.status === "failed").length,
+        artifactKinds: bundle.lifecycleArtifacts.map((artifact) => artifact.artifactKind)
+    };
+}
+export function compareLocalEvalRuns(baselineRunRef, candidateRunRefs, cwd = process.cwd()) {
+    if (candidateRunRefs.length === 0) {
+        throw new Error("Provide at least one candidate eval run to compare against the baseline.");
+    }
+    const root = findWorkspaceRoot(cwd);
+    ensureInitFiles(root);
+    const baseline = readRunBundleByRef(root, baselineRunRef);
+    const baselineArtifact = extractEvalArtifact(baseline.bundle, baselineRunRef);
+    const comparedRuns = candidateRunRefs.map((candidateRunRef) => {
+        const candidate = readRunBundleByRef(root, candidateRunRef);
+        const candidateArtifact = extractEvalArtifact(candidate.bundle, candidateRunRef);
+        return compareEvalArtifacts(baseline.runId, baseline.bundlePath, baselineArtifact, candidate.runId, candidate.bundlePath, candidateArtifact);
+    });
+    const { bundle, jsonPath, markdownPath, outputDir } = createBenchmarkBundle(root, baseline.runId, baseline.bundlePath, baselineArtifact, comparedRuns);
+    return {
+        runId: bundle.runId,
+        outputDir,
+        jsonPath,
+        markdownPath,
+        status: bundle.status,
+        baselineRunId: baseline.runId,
+        comparedRunIds: comparedRuns.map((candidate) => candidate.runId),
+        comparableRunCount: comparedRuns.filter((candidate) => candidate.comparable).length,
+        regressionCount: comparedRuns.reduce((total, candidate) => total + candidate.regressions.length, 0),
+        improvementCount: comparedRuns.reduce((total, candidate) => total + candidate.improvements.length, 0),
+        unchangedCount: comparedRuns.reduce((total, candidate) => total + candidate.unchangedCount, 0),
+        nonComparableCount: comparedRuns.reduce((total, candidate) => total + candidate.nonComparableFindings.length, 0),
+        artifactKinds: bundle.lifecycleArtifacts.map((artifact) => artifact.artifactKind)
+    };
+}
 export function explainLastRun(cwd = process.cwd()) {
     const root = findWorkspaceRoot(cwd);
     const config = loadAgentForgeConfig(root);