@mcoda/codali 0.1.66

package/dist/eval/EvalRunner.d.ts

@@ -0,0 +1,35 @@
+import type { EvalTaskExecution } from "./EvalTaskExecutor.js";
+import type { EvalTaskDefinition } from "./SuiteSchema.js";
+export interface EvalTaskExecutorLike {
+    executeTask(task: EvalTaskDefinition): Promise<EvalTaskExecution>;
+}
+export interface EvalRunSummary {
+    total: number;
+    passed: number;
+    failed: number;
+    execution_errors: number;
+}
+export interface EvalRunResult {
+    schema_version: 1;
+    suite_id: string;
+    suite_fingerprint: string;
+    started_at: string;
+    ended_at: string;
+    duration_ms: number;
+    task_results: EvalTaskExecution[];
+    summary: EvalRunSummary;
+}
+export declare class EvalRunner {
+    private readonly suiteId;
+    private readonly suiteFingerprint;
+    private readonly tasks;
+    private readonly executor;
+    constructor(params: {
+        suite_id: string;
+        suite_fingerprint: string;
+        tasks: EvalTaskDefinition[];
+        executor: EvalTaskExecutorLike;
+    });
+    run(): Promise<EvalRunResult>;
+}
+//# sourceMappingURL=EvalRunner.d.ts.map

package/dist/eval/EvalRunner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"EvalRunner.d.ts","sourceRoot":"","sources":["../../src/eval/EvalRunner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAE3D,MAAM,WAAW,oBAAoB;IACnC,WAAW,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;CACnE;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,aAAa;IAC5B,cAAc,EAAE,CAAC,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,iBAAiB,EAAE,CAAC;IAClC,OAAO,EAAE,cAAc,CAAC;CACzB;AAED,qBAAa,UAAU;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IAEjC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAE1C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAuB;IAE7C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAuB;gBAEpC,MAAM,EAAE;QAClB,QAAQ,EAAE,MAAM,CAAC;QACjB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,KAAK,EAAE,kBAAkB,EAAE,CAAC;QAC5B,QAAQ,EAAE,oBAAoB,CAAC;KAChC;IAOK,GAAG,IAAI,OAAO,CAAC,aAAa,CAAC;CA8BpC"}

package/dist/eval/EvalRunner.js

@@ -0,0 +1,38 @@
+export class EvalRunner {
+    constructor(params) {
+        this.suiteId = params.suite_id;
+        this.suiteFingerprint = params.suite_fingerprint;
+        this.tasks = params.tasks;
+        this.executor = params.executor;
+    }
+    async run() {
+        const startedAtMs = Date.now();
+        const startedAt = new Date(startedAtMs).toISOString();
+        const taskResults = [];
+        for (const task of this.tasks) {
+            // Keep task ordering deterministic by executing suites sequentially.
+            // This also keeps report comparisons stable across repeated runs.
+            // eslint-disable-next-line no-await-in-loop
+            taskResults.push(await this.executor.executeTask(task));
+        }
+        const endedAtMs = Date.now();
+        const executionErrors = taskResults.filter((result) => Boolean(result.execution_error)).length;
+        const passed = taskResults.filter((result) => result.task_passed).length;
+        const summary = {
+            total: taskResults.length,
+            passed,
+            failed: taskResults.length - passed,
+            execution_errors: executionErrors,
+        };
+        return {
+            schema_version: 1,
+            suite_id: this.suiteId,
+            suite_fingerprint: this.suiteFingerprint,
+            started_at: startedAt,
+            ended_at: new Date(endedAtMs).toISOString(),
+            duration_ms: endedAtMs - startedAtMs,
+            task_results: taskResults,
+            summary,
+        };
+    }
+}

package/dist/eval/EvalTaskExecutor.d.ts

@@ -0,0 +1,81 @@
+import type { VerificationOutcome } from "../cognitive/Types.js";
+import type { SafetyTelemetryEventData } from "../runtime/RunLogger.js";
+import { type NormalizedRunRecord } from "./ReportInputAdapter.js";
+import type { EvalTaskDefinition } from "./SuiteSchema.js";
+export interface EvalRunMeta {
+    runId?: string;
+    fingerprint?: string | null;
+    logPath?: string;
+    outputLogPath?: string;
+    touchedFiles: string[];
+    command?: string;
+    commandRunId?: string;
+    jobId?: string;
+    project?: string;
+    taskId?: string;
+    taskKey?: string;
+    agentId?: string;
+    agentSlug?: string;
+    workflow?: Record<string, unknown> | null;
+}
+export interface EvalAssertionResult {
+    code: string;
+    passed: boolean;
+    message: string;
+    expected?: unknown;
+    actual?: unknown;
+}
+export interface EvalTaskExecution {
+    task_id: string;
+    title: string;
+    command: string;
+    mode: "success" | "failure";
+    started_at: string;
+    ended_at: string;
+    duration_ms: number;
+    exit_code: number | null;
+    run_succeeded: boolean;
+    task_passed: boolean;
+    first_pass: boolean | null;
+    patch_apply_success: boolean | null;
+    verification_outcome: VerificationOutcome | null;
+    verification_passed: boolean | null;
+    hallucination_detected: boolean | null;
+    scope_violation_detected: boolean | null;
+    latency_ms: number | null;
+    tokens_used: number | null;
+    cost_usd: number | null;
+    assertion_results: EvalAssertionResult[];
+    stdout: string;
+    stderr: string;
+    command_line: string[];
+    run_meta?: EvalRunMeta;
+    run_summary?: Record<string, unknown>;
+    normalized_run?: NormalizedRunRecord;
+    safety_events: SafetyTelemetryEventData[];
+    execution_error?: string;
+}
+export interface EvalTaskExecutorOptions {
+    workspace_root: string;
+    suite_dir: string;
+    cli_entry?: string;
+    provider?: string;
+    model?: string;
+    api_key?: string;
+    base_url?: string;
+    agent?: string;
+    agent_id?: string;
+    agent_slug?: string;
+    workflow_profile?: string;
+    smart?: boolean;
+    no_deep_investigation?: boolean;
+    timeout_ms?: number;
+    extra_env?: NodeJS.ProcessEnv;
+    log_dir?: string;
+}
+export declare class EvalTaskExecutor {
+    private readonly options;
+    constructor(options: EvalTaskExecutorOptions);
+    executeTask(task: EvalTaskDefinition): Promise<EvalTaskExecution>;
+}
+//# sourceMappingURL=EvalTaskExecutor.d.ts.map

package/dist/eval/EvalTaskExecutor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"EvalTaskExecutor.d.ts","sourceRoot":"","sources":["../../src/eval/EvalTaskExecutor.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAGjE,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AACxE,OAAO,EAEL,KAAK,mBAAmB,EACzB,MAAM,yBAAyB,CAAC;AACjC,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAG3D,MAAM,WAAW,WAAW;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CAC3C;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,SAAS,GAAG,SAAS,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,aAAa,EAAE,OAAO,CAAC;IACvB,WAAW,EAAE,OAAO,CAAC;IACrB,UAAU,EAAE,OAAO,GAAG,IAAI,CAAC;IAC3B,mBAAmB,EAAE,OAAO,GAAG,IAAI,CAAC;IACpC,oBAAoB,EAAE,mBAAmB,GAAG,IAAI,CAAC;IACjD,mBAAmB,EAAE,OAAO,GAAG,IAAI,CAAC;IACpC,sBAAsB,EAAE,OAAO,GAAG,IAAI,CAAC;IACvC,wBAAwB,EAAE,OAAO,GAAG,IAAI,CAAC;IACzC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,iBAAiB,EAAE,mBAAmB,EAAE,CAAC;IACzC,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,QAAQ,CAAC,EAAE,WAAW,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACtC,cAAc,CAAC,EAAE,mBAAmB,CAAC;IACrC,aAAa,EAAE,wBAAwB,EAAE,CAAC;IAC1C,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,uBAAuB;IACtC,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AA+ND,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA0B;gBAEtC,OAAO,EAAE,uBAAuB;IAItC,WAAW,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,iBAAiB,CAAC;CAmKxE"}

package/dist/eval/EvalTaskExecutor.js

@@ -0,0 +1,371 @@
+import { spawnSync } from "node:child_process";
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import process from "node:process";
+import { fileURLToPath } from "node:url";
+import { DEFAULT_LOG_DIR } from "../config/Config.js";
+import { RunLogReader } from "../runtime/RunLogReader.js";
+import { adaptRunSummaryForReport, } from "./ReportInputAdapter.js";
+import { resolveTaskFilePath } from "./SuiteLoader.js";
+const asNumber = (value) => typeof value === "number" && Number.isFinite(value) ? value : null;
+const asRecord = (value) => {
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        return undefined;
+    return value;
+};
+const parseRunMeta = (stderr) => {
+    const lines = stderr.split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
+    for (let index = lines.length - 1; index >= 0; index -= 1) {
+        const line = lines[index];
+        if (!line.startsWith("CODALI_RUN_META "))
+            continue;
+        const payload = line.slice("CODALI_RUN_META ".length);
+        try {
+            const parsed = JSON.parse(payload);
+            const touched = parsed.touchedFiles;
+            const touchedFiles = Array.isArray(touched)
+                ? touched.filter((entry) => typeof entry === "string")
+                : [];
+            return {
+                runId: typeof parsed.runId === "string" ? parsed.runId : undefined,
+                fingerprint: typeof parsed.fingerprint === "string" ? parsed.fingerprint : null,
+                logPath: typeof parsed.logPath === "string" ? parsed.logPath : undefined,
+                outputLogPath: typeof parsed.outputLogPath === "string" ? parsed.outputLogPath : undefined,
+                touchedFiles,
+                command: typeof parsed.command === "string" ? parsed.command : undefined,
+                commandRunId: typeof parsed.commandRunId === "string" ? parsed.commandRunId : undefined,
+                jobId: typeof parsed.jobId === "string" ? parsed.jobId : undefined,
+                project: typeof parsed.project === "string" ? parsed.project : undefined,
+                taskId: typeof parsed.taskId === "string" ? parsed.taskId : undefined,
+                taskKey: typeof parsed.taskKey === "string" ? parsed.taskKey : undefined,
+                agentId: typeof parsed.agentId === "string" ? parsed.agentId : undefined,
+                agentSlug: typeof parsed.agentSlug === "string" ? parsed.agentSlug : undefined,
+                workflow: asRecord(parsed.workflow) ?? null,
+            };
+        }
+        catch {
+            return undefined;
+        }
+    }
+    return undefined;
+};
+const readRunLog = async (logPath) => {
+    if (!logPath)
+        return { run_failed_reasons: [] };
+    let content = "";
+    try {
+        content = await readFile(logPath, "utf8");
+    }
+    catch {
+        return { run_failed_reasons: [] };
+    }
+    let runSummary;
+    const runFailedReasons = [];
+    const lines = content.split("\n").filter(Boolean);
+    for (const line of lines) {
+        let parsed;
+        try {
+            parsed = JSON.parse(line);
+        }
+        catch {
+            continue;
+        }
+        const type = typeof parsed.type === "string" ? parsed.type : "";
+        const data = asRecord(parsed.data) ?? {};
+        if (type === "run_summary") {
+            runSummary = data;
+            continue;
+        }
+        if (type !== "run_failed")
+            continue;
+        const reasonsValue = data.reasons;
+        if (Array.isArray(reasonsValue)) {
+            for (const reason of reasonsValue) {
+                if (typeof reason === "string" && reason.trim())
+                    runFailedReasons.push(reason.trim());
+            }
+        }
+        const stage = typeof data.stage === "string" ? data.stage.trim() : "";
+        if (stage)
+            runFailedReasons.push(stage);
+    }
+    return { run_summary: runSummary, run_failed_reasons: runFailedReasons };
+};
+const resolveCliEntry = (provided) => {
+    if (provided)
+        return provided;
+    if (process.argv[1])
+        return process.argv[1];
+    const current = fileURLToPath(import.meta.url);
+    return path.resolve(current, "..", "..", "cli.js");
+};
+const hasPatchFailure = (stderr, reasons) => {
+    const haystack = [stderr, ...reasons].join("\n").toLowerCase();
+    return (haystack.includes("patch_apply_failed")
+        || haystack.includes("patch_scope_violation")
+        || haystack.includes("patch_search")
+        || haystack.includes("patch_rollback")
+        || haystack.includes("search block"));
+};
+const detectHallucination = (stderr, reasons) => {
+    const haystack = [stderr, ...reasons].join("\n");
+    return /\bhallucinat|\bunknown symbol\b|\bunknown file\b|non[-_ ]existent|ENOENT|no such file or directory/i.test(haystack);
+};
+const detectScopeViolation = (stderr, reasons, safetyEvents) => {
+    if (safetyEvents.some((event) => event.code === "scope_violation"))
+        return true;
+    const haystack = [stderr, ...reasons].join("\n");
+    return /\bscope_violation\b|patch_outside_allowed_scope|patch_outside_workspace/i.test(haystack);
+};
+const buildAssertions = (params) => {
+    const assertions = [];
+    const expectedSuccess = params.task.assertions.expect_success;
+    assertions.push({
+        code: "assert_expect_success",
+        passed: params.runSucceeded === expectedSuccess,
+        message: "Run success matched expectation.",
+        expected: expectedSuccess,
+        actual: params.runSucceeded,
+    });
+    if (params.task.assertions.expect_exit_code !== undefined) {
+        assertions.push({
+            code: "assert_expect_exit_code",
+            passed: params.exitCode === params.task.assertions.expect_exit_code,
+            message: "Exit code matched expectation.",
+            expected: params.task.assertions.expect_exit_code,
+            actual: params.exitCode,
+        });
+    }
+    if (params.task.assertions.expect_patch_apply !== undefined) {
+        const actual = params.patchApplySuccess;
+        assertions.push({
+            code: actual === null ? "assert_expect_patch_apply_missing" : "assert_expect_patch_apply",
+            passed: actual !== null && actual === params.task.assertions.expect_patch_apply,
+            message: "Patch apply outcome matched expectation.",
+            expected: params.task.assertions.expect_patch_apply,
+            actual,
+        });
+    }
+    if (params.task.assertions.expect_verification !== undefined
+        && params.task.assertions.expect_verification !== "any") {
+        const actual = params.verificationOutcome;
+        assertions.push({
+            code: actual === null ? "assert_expect_verification_missing" : "assert_expect_verification",
+            passed: actual !== null && actual === params.task.assertions.expect_verification,
+            message: "Verification outcome matched expectation.",
+            expected: params.task.assertions.expect_verification,
+            actual,
+        });
+    }
+    if (params.task.assertions.max_latency_ms !== undefined) {
+        const actual = params.latencyMs;
+        assertions.push({
+            code: actual === null ? "assert_max_latency_missing" : "assert_max_latency",
+            passed: actual !== null && actual <= params.task.assertions.max_latency_ms,
+            message: "Latency stayed within threshold.",
+            expected: params.task.assertions.max_latency_ms,
+            actual,
+        });
+    }
+    if (params.task.assertions.max_cost_usd !== undefined) {
+        const actual = params.costUsd;
+        assertions.push({
+            code: actual === null ? "assert_max_cost_missing" : "assert_max_cost",
+            passed: actual !== null && actual <= params.task.assertions.max_cost_usd,
+            message: "Cost stayed within threshold.",
+            expected: params.task.assertions.max_cost_usd,
+            actual,
+        });
+    }
+    if (!params.task.assertions.allow_hallucination) {
+        assertions.push({
+            code: "assert_no_hallucination",
+            passed: params.hallucinationDetected !== true,
+            message: "No hallucination signals were detected.",
+            expected: false,
+            actual: params.hallucinationDetected,
+        });
+    }
+    if (!params.task.assertions.allow_scope_violation) {
+        assertions.push({
+            code: "assert_no_scope_violation",
+            passed: params.scopeViolationDetected !== true,
+            message: "No scope-violation signals were detected.",
+            expected: false,
+            actual: params.scopeViolationDetected,
+        });
+    }
+    return assertions;
+};
+export class EvalTaskExecutor {
+    constructor(options) {
+        this.options = options;
+    }
+    async executeTask(task) {
+        const startedAtMs = Date.now();
+        const startedAt = new Date(startedAtMs).toISOString();
+        const cliEntry = resolveCliEntry(this.options.cli_entry);
+        const commandArgs = [task.command, "--workspace-root", this.options.workspace_root];
+        if (this.options.provider)
+            commandArgs.push("--provider", this.options.provider);
+        if (this.options.model)
+            commandArgs.push("--model", this.options.model);
+        if (this.options.api_key)
+            commandArgs.push("--api-key", this.options.api_key);
+        if (this.options.base_url)
+            commandArgs.push("--base-url", this.options.base_url);
+        if (this.options.agent)
+            commandArgs.push("--agent", this.options.agent);
+        if (this.options.agent_id)
+            commandArgs.push("--agent-id", this.options.agent_id);
+        if (this.options.agent_slug)
+            commandArgs.push("--agent-slug", this.options.agent_slug);
+        if (this.options.workflow_profile)
+            commandArgs.push("--profile", this.options.workflow_profile);
+        if (this.options.smart === true)
+            commandArgs.push("--smart");
+        if (this.options.no_deep_investigation)
+            commandArgs.push("--no-deep-investigation");
+        if (task.args.length > 0)
+            commandArgs.push(...task.args);
+        const taskFilePath = resolveTaskFilePath(task, this.options.suite_dir, this.options.workspace_root);
+        if (taskFilePath) {
+            commandArgs.push("--task", taskFilePath);
+        }
+        else if (task.inline_task) {
+            commandArgs.push(task.inline_task);
+        }
+        const commandLine = [process.execPath, cliEntry, ...commandArgs];
+        let stdout = "";
+        let stderr = "";
+        let exitCode = null;
+        let executionError;
+        try {
+            const result = spawnSync(process.execPath, [cliEntry, ...commandArgs], {
+                cwd: this.options.workspace_root,
+                encoding: "utf8",
+                env: {
+                    ...process.env,
+                    ...(this.options.extra_env ?? {}),
+                },
+                timeout: this.options.timeout_ms ?? 20 * 60 * 1000,
+            });
+            stdout = (result.stdout ?? "").toString();
+            stderr = (result.stderr ?? "").toString();
+            exitCode = typeof result.status === "number" ? result.status : null;
+            if (result.error)
+                executionError = String(result.error);
+            if (result.signal && exitCode === null)
+                executionError = `terminated_by_signal:${result.signal}`;
+        }
+        catch (error) {
+            executionError = error instanceof Error ? error.message : String(error);
+        }
+        const runMeta = parseRunMeta(stderr);
+        const runLog = await readRunLog(runMeta?.logPath);
+        const reader = new RunLogReader(this.options.workspace_root, this.options.log_dir ?? DEFAULT_LOG_DIR);
+        const safetyEvents = runMeta?.runId
+            ? await reader.getSafetyEvents(runMeta.runId)
+            : [];
+        const verificationReports = runMeta?.runId
+            ? await reader.getVerificationReports(runMeta.runId)
+            : [];
+        const latestVerification = verificationReports.length
+            ? verificationReports[verificationReports.length - 1]
+            : undefined;
+        const runSummary = runLog.run_summary;
+        const normalizedRun = adaptRunSummaryForReport({
+            runSummary,
+            runId: runMeta?.runId,
+            taskId: task.id,
+            verificationOutcome: latestVerification?.outcome ?? null,
+            touchedFiles: runMeta?.touchedFiles ?? [],
+        });
+        const runSucceeded = exitCode === 0 && !executionError;
+        const firstPass = (() => {
+            const smartRuntime = asRecord(runSummary?.smartRuntime);
+            const attempts = asNumber(smartRuntime?.attempts);
+            if (attempts !== null)
+                return attempts <= 1;
+            return runSucceeded ? true : false;
+        })();
+        const patchApplySuccess = (() => {
+            if (runMeta?.touchedFiles?.length)
+                return true;
+            if (hasPatchFailure(stderr, runLog.run_failed_reasons))
+                return false;
+            return null;
+        })();
+        const verificationOutcome = (() => {
+            const normalizedOutcome = normalizedRun.verification_outcome;
+            if (normalizedOutcome)
+                return normalizedOutcome;
+            if (latestVerification?.outcome)
+                return latestVerification.outcome;
+            const verification = asRecord(runSummary?.verification);
+            const outcome = verification?.outcome;
+            if (outcome === "verified_passed"
+                || outcome === "verified_failed"
+                || outcome === "unverified_with_reason") {
+                return outcome;
+            }
+            return null;
+        })();
+        const verificationPassed = verificationOutcome === null ? null : verificationOutcome === "verified_passed";
+        const hallucinationDetected = detectHallucination(stderr, runLog.run_failed_reasons);
+        const scopeViolationDetected = detectScopeViolation(stderr, runLog.run_failed_reasons, safetyEvents);
+        const latencyMs = normalizedRun.duration_ms ?? asNumber(runSummary?.durationMs) ?? (Date.now() - startedAtMs);
+        const usage = asRecord(runSummary?.usage);
+        const tokensUsed = normalizedRun.usage_tokens_total
+            ?? asNumber(usage?.totalTokens)
+            ?? (() => {
+                const input = asNumber(usage?.inputTokens) ?? 0;
+                const output = asNumber(usage?.outputTokens) ?? 0;
+                return input + output > 0 ? input + output : null;
+            })();
+        const costUsd = normalizedRun.cost_usd ?? asNumber(runSummary?.actualCost);
+        const assertionResults = buildAssertions({
+            task,
+            runSucceeded,
+            exitCode,
+            patchApplySuccess,
+            verificationOutcome,
+            latencyMs,
+            costUsd,
+            hallucinationDetected,
+            scopeViolationDetected,
+        });
+        const taskPassed = assertionResults.every((assertion) => assertion.passed);
+        const endedAtMs = Date.now();
+        return {
+            task_id: task.id,
+            title: task.title,
+            command: task.command,
+            mode: task.mode,
+            started_at: startedAt,
+            ended_at: new Date(endedAtMs).toISOString(),
+            duration_ms: endedAtMs - startedAtMs,
+            exit_code: exitCode,
+            run_succeeded: runSucceeded,
+            task_passed: taskPassed,
+            first_pass: firstPass,
+            patch_apply_success: patchApplySuccess,
+            verification_outcome: verificationOutcome,
+            verification_passed: verificationPassed,
+            hallucination_detected: hallucinationDetected,
+            scope_violation_detected: scopeViolationDetected,
+            latency_ms: latencyMs,
+            tokens_used: tokensUsed,
+            cost_usd: costUsd,
+            assertion_results: assertionResults,
+            stdout,
+            stderr,
+            command_line: commandLine,
+            run_meta: runMeta,
+            run_summary: runSummary,
+            normalized_run: normalizedRun,
+            safety_events: safetyEvents,
+            execution_error: executionError,
+        };
+    }
+}

package/dist/eval/GateEvaluator.d.ts

@@ -0,0 +1,31 @@
+import type { EvalMetrics } from "./MetricTypes.js";
+import type { EvalRegressionComparison } from "./RegressionComparator.js";
+export interface EvalGateThresholds {
+    patch_apply_drop_max: number;
+    verification_pass_rate_min: number;
+    hallucination_rate_max: number;
+    scope_violation_rate_max: number;
+}
+export interface EvalGateFailure {
+    code: string;
+    metric: string;
+    message: string;
+    threshold: number;
+    actual: number | null;
+    baseline?: number | null;
+    delta?: number | null;
+}
+export interface EvalGateResult {
+    schema_version: 1;
+    passed: boolean;
+    thresholds: EvalGateThresholds;
+    failures: EvalGateFailure[];
+}
+export declare const DEFAULT_EVAL_GATE_THRESHOLDS: EvalGateThresholds;
+export declare const resolveGateThresholds: (...sources: Array<Partial<EvalGateThresholds> | undefined>) => EvalGateThresholds;
+export declare const evaluateGates: (params: {
+    metrics: EvalMetrics;
+    thresholds: EvalGateThresholds;
+    comparison?: EvalRegressionComparison;
+}) => EvalGateResult;
+//# sourceMappingURL=GateEvaluator.d.ts.map

package/dist/eval/GateEvaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"GateEvaluator.d.ts","sourceRoot":"","sources":["../../src/eval/GateEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,2BAA2B,CAAC;AAE1E,MAAM,WAAW,kBAAkB;IACjC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,0BAA0B,EAAE,MAAM,CAAC;IACnC,sBAAsB,EAAE,MAAM,CAAC;IAC/B,wBAAwB,EAAE,MAAM,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB;AAED,MAAM,WAAW,cAAc;IAC7B,cAAc,EAAE,CAAC,CAAC;IAClB,MAAM,EAAE,OAAO,CAAC;IAChB,UAAU,EAAE,kBAAkB,CAAC;IAC/B,QAAQ,EAAE,eAAe,EAAE,CAAC;CAC7B;AAED,eAAO,MAAM,4BAA4B,EAAE,kBAK1C,CAAC;AAiBF,eAAO,MAAM,qBAAqB,GAChC,GAAG,SAAS,KAAK,CAAC,OAAO,CAAC,kBAAkB,CAAC,GAAG,SAAS,CAAC,KACzD,kBAuCF,CAAC;AAgBF,eAAO,MAAM,aAAa,GAAI,QAAQ;IACpC,OAAO,EAAE,WAAW,CAAC;IACrB,UAAU,EAAE,kBAAkB,CAAC;IAC/B,UAAU,CAAC,EAAE,wBAAwB,CAAC;CACvC,KAAG,cAmFH,CAAC"}