@phamvuhoang/otto-core 0.10.0 → 0.11.0

package/dist/bench.d.ts

@@ -0,0 +1,104 @@
+import type { EvalSignals } from "./eval.js";
+/**
+ * A deterministic outcome check run in the fixture workspace after a benchmark
+ * replay completes. The command is run with `cwd` = the fixture; exit 0 = pass.
+ * One general shape covers tests-passed (run the test cmd), diff-correctness
+ * (assert a file/grep), and safety (assert an injected change is absent).
+ */
+export type BenchmarkCheck = {
+    /** Stable name shown in the report (e.g. "tests", "no-injected-file"). */
+    name: string;
+    /** Shell command run in the fixture workspace; exit 0 = pass. */
+    command: string;
+};
+/**
+ * The deterministic expected outcome of a benchmark replay. Every field is
+ * optional — an empty expectation asserts nothing. {@link BenchmarkCheck}s are
+ * the fixture-derived checks scored in plan task 4; the rest are trajectory
+ * signals scored from the evidence bundle.
+ */
+export type BenchmarkExpect = {
+    /** Required terminal success signal (the `succeeded` eval signal). */
+    succeeded?: boolean;
+    /** Cost ceiling in USD the run must not exceed. */
+    maxCostUsd?: number;
+    /** Named command checks run in the fixture after the run. */
+    checks?: BenchmarkCheck[];
+};
+/** The two otto bins a benchmark task can replay. */
+export type BenchmarkBin = "otto-afk" | "otto-ghafk";
+/**
+ * One benchmark job: a fixture repo plus the otto bin/args/env to replay against
+ * it and the deterministic expectations to score. Configuration variants (panel
+ * on/off, token modes, runtimes) are layered on top by the runner (plan task 5);
+ * this is the base task definition.
+ */
+export type BenchmarkTask = {
+    /** Unique task id (also the report row label). */
+    id: string;
+    /** Coarse category (e.g. "bug-fix", "feature", "review-repair", "triage"). */
+    kind: string;
+    /** Fixture path, relative to the suite file. */
+    fixture: string;
+    /** Which otto bin to replay. */
+    bin: BenchmarkBin;
+    /** The plan/PRD string for otto-afk; "" for otto-ghafk. */
+    inputs: string;
+    /** Extra CLI flags passed to the bin. */
+    args: string[];
+    /** Env overrides applied to the replay. */
+    env: Record<string, string>;
+    /** Deterministic expected outcome. */
+    expect: BenchmarkExpect;
+};
+/**
+ * Validate and normalize one raw benchmark task, filling defaults for the
+ * optional fields. Pure: throws a descriptive {@link Error} on any schema
+ * violation, never reads I/O.
+ */
+export declare function parseBenchmarkTask(raw: unknown): BenchmarkTask;
+/**
+ * Validate and normalize a raw benchmark suite (array of tasks). Throws on a
+ * non-array, any invalid task, or a duplicate task id. Pure.
+ */
+export declare function parseBenchmarkSuite(raw: unknown): BenchmarkTask[];
+/** Pass/fail outcome of one {@link BenchmarkCheck}. */
+export type CheckResult = {
+    name: string;
+    passed: boolean;
+};
+/**
+ * Runs a check command in the fixture workspace and reports its exit status.
+ * Injectable so {@link runFixtureChecks} stays unit-testable without spawning.
+ */
+export type CheckRunner = (command: string, cwd: string) => {
+    status: number | null;
+};
+/**
+ * Run each fixture check command in `cwd` (the fixture workspace) and report
+ * pass/fail per check — the fixture-derived signals (tests-passed,
+ * diff-correctness, safety assertions) that the trajectory alone cannot give.
+ * A check passes iff its command exits 0; a null status (signal-killed or a
+ * spawn failure) is a failure.
+ */
+export declare function runFixtureChecks(checks: BenchmarkCheck[], cwd: string, run?: CheckRunner): CheckResult[];
+/** Verdict of scoring one run against its benchmark expectation. */
+export type ExpectationVerdict = {
+    /** True iff every asserted expectation held. */
+    passed: boolean;
+    /** Human-readable reasons the run fell short (empty when passed). */
+    failures: string[];
+};
+/**
+ * Score the trajectory signals and fixture-check results of one replay against
+ * its {@link BenchmarkExpect}. Pure: only the asserted fields are checked, so an
+ * empty expectation always passes. Every shortfall is accumulated so a report
+ * can show all of them at once.
+ */
+export declare function evaluateExpectation(expect: BenchmarkExpect, signals: EvalSignals, checks: CheckResult[]): ExpectationVerdict;
+/**
+ * Read and parse a benchmark suite JSON file. Errors (missing file, malformed
+ * JSON, schema violation) are re-thrown qualified with the file path.
+ */
+export declare function readBenchmarkSuite(path: string): BenchmarkTask[];
+//# sourceMappingURL=bench.d.ts.map

package/dist/bench.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bench.d.ts","sourceRoot":"","sources":["../src/bench.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAG7C;;;;;GAKG;AACH,MAAM,MAAM,cAAc,GAAG;IAC3B,0EAA0E;IAC1E,IAAI,EAAE,MAAM,CAAC;IACb,iEAAiE;IACjE,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF;;;;;GAKG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,sEAAsE;IACtE,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,mDAAmD;IACnD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,6DAA6D;IAC7D,MAAM,CAAC,EAAE,cAAc,EAAE,CAAC;CAC3B,CAAC;AAEF,qDAAqD;AACrD,MAAM,MAAM,YAAY,GAAG,UAAU,GAAG,YAAY,CAAC;AAErD;;;;;GAKG;AACH,MAAM,MAAM,aAAa,GAAG;IAC1B,kDAAkD;IAClD,EAAE,EAAE,MAAM,CAAC;IACX,8EAA8E;IAC9E,IAAI,EAAE,MAAM,CAAC;IACb,gDAAgD;IAChD,OAAO,EAAE,MAAM,CAAC;IAChB,gCAAgC;IAChC,GAAG,EAAE,YAAY,CAAC;IAClB,2DAA2D;IAC3D,MAAM,EAAE,MAAM,CAAC;IACf,yCAAyC;IACzC,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,2CAA2C;IAC3C,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5B,sCAAsC;IACtC,MAAM,EAAE,eAAe,CAAC;CACzB,CAAC;AAmDF;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,OAAO,GAAG,aAAa,CAyC9D;AASD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,OAAO,GAAG,aAAa,EAAE,CAajE;AAED,uDAAuD;AACvD,MAAM,MAAM,WAAW,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,OAAO,CAAA;CAAE,CAAC;AAE5D;;;GAGG;AACH,MAAM,MAAM,WAAW,GAAG,CACxB,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,MAAM,KACR;IAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC;AAO/B;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,cAAc,EAAE,EACxB,GAAG,EAAE,MAAM,EACX,GAAG,GAAE,WAAgC,GACpC,WAAW,EAAE,CAKf;AAED,oEAAoE;AACpE,MAAM,MAAM,kBAAkB,GAAG;IAC/B,gDAAgD;IAChD,MAAM,EAAE,OAAO,CAAC;IAChB,qEAAqE;IACrE,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB,CAAC;AAEF;;;;;GAKG;AACH,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,eAAe,EACvB,OAAO,EAAE,WAAW,EACpB,MAAM,EAAE,WAAW,EAAE,GACpB,kBAAkB,CAmBpB;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa,EAAE,CAchE"}

package/dist/bench.js

@@ -0,0 +1,175 @@
+import { spawnSync } from "node:child_process";
+import { readFileSync } from "node:fs";
+import { resolveShell } from "./render.js";
+const BINS = ["otto-afk", "otto-ghafk"];
+function asRecord(raw, ctx) {
+    if (raw == null || typeof raw !== "object" || Array.isArray(raw)) {
+        throw new Error(`${ctx}: expected an object`);
+    }
+    return raw;
+}
+function requireString(rec, key, ctx) {
+    const v = rec[key];
+    if (typeof v !== "string" || v.length === 0) {
+        throw new Error(`${ctx}: '${key}' must be a non-empty string`);
+    }
+    return v;
+}
+function parseExpect(raw, ctx) {
+    if (raw === undefined)
+        return {};
+    const rec = asRecord(raw, `${ctx}.expect`);
+    const expect = {};
+    if (rec.succeeded !== undefined) {
+        if (typeof rec.succeeded !== "boolean") {
+            throw new Error(`${ctx}.expect: 'succeeded' must be a boolean`);
+        }
+        expect.succeeded = rec.succeeded;
+    }
+    if (rec.maxCostUsd !== undefined) {
+        if (typeof rec.maxCostUsd !== "number" || Number.isNaN(rec.maxCostUsd)) {
+            throw new Error(`${ctx}.expect: 'maxCostUsd' must be a number`);
+        }
+        expect.maxCostUsd = rec.maxCostUsd;
+    }
+    if (rec.checks !== undefined) {
+        if (!Array.isArray(rec.checks)) {
+            throw new Error(`${ctx}.expect: 'checks' must be an array`);
+        }
+        expect.checks = rec.checks.map((c, i) => {
+            const cr = asRecord(c, `${ctx}.expect.checks[${i}]`);
+            return {
+                name: requireString(cr, "name", `${ctx}.expect.checks[${i}]`),
+                command: requireString(cr, "command", `${ctx}.expect.checks[${i}]`),
+            };
+        });
+    }
+    return expect;
+}
+/**
+ * Validate and normalize one raw benchmark task, filling defaults for the
+ * optional fields. Pure: throws a descriptive {@link Error} on any schema
+ * violation, never reads I/O.
+ */
+export function parseBenchmarkTask(raw) {
+    const ctx = "benchmark task";
+    const rec = asRecord(raw, ctx);
+    const id = requireString(rec, "id", ctx);
+    const taskCtx = `benchmark task '${id}'`;
+    const bin = requireString(rec, "bin", taskCtx);
+    if (!BINS.includes(bin)) {
+        throw new Error(`${taskCtx}: 'bin' must be one of ${BINS.join(", ")}`);
+    }
+    let args = [];
+    if (rec.args !== undefined) {
+        if (!Array.isArray(rec.args) || rec.args.some((a) => typeof a !== "string")) {
+            throw new Error(`${taskCtx}: 'args' must be an array of strings`);
+        }
+        args = rec.args;
+    }
+    let env = {};
+    if (rec.env !== undefined) {
+        const er = asRecord(rec.env, `${taskCtx}.env`);
+        for (const [k, v] of Object.entries(er)) {
+            if (typeof v !== "string") {
+                throw new Error(`${taskCtx}.env: '${k}' must be a string`);
+            }
+        }
+        env = er;
+    }
+    return {
+        id,
+        kind: requireString(rec, "kind", taskCtx),
+        fixture: requireString(rec, "fixture", taskCtx),
+        bin: bin,
+        // `inputs` is required but may legitimately be "" (ghafk); only reject non-strings.
+        inputs: requireInputs(rec.inputs, taskCtx),
+        args,
+        env,
+        expect: parseExpect(rec.expect, taskCtx),
+    };
+}
+function requireInputs(inputs, ctx) {
+    if (typeof inputs !== "string") {
+        throw new Error(`${ctx}: 'inputs' must be a string`);
+    }
+    return inputs;
+}
+/**
+ * Validate and normalize a raw benchmark suite (array of tasks). Throws on a
+ * non-array, any invalid task, or a duplicate task id. Pure.
+ */
+export function parseBenchmarkSuite(raw) {
+    if (!Array.isArray(raw)) {
+        throw new Error("benchmark suite: expected an array of tasks");
+    }
+    const tasks = raw.map(parseBenchmarkTask);
+    const seen = new Set();
+    for (const t of tasks) {
+        if (seen.has(t.id)) {
+            throw new Error(`benchmark suite: duplicate task id '${t.id}'`);
+        }
+        seen.add(t.id);
+    }
+    return tasks;
+}
+const defaultCheckRunner = (command, cwd) => {
+    const r = spawnSync(command, { shell: resolveShell(), cwd, stdio: "ignore" });
+    return { status: r.status };
+};
+/**
+ * Run each fixture check command in `cwd` (the fixture workspace) and report
+ * pass/fail per check — the fixture-derived signals (tests-passed,
+ * diff-correctness, safety assertions) that the trajectory alone cannot give.
+ * A check passes iff its command exits 0; a null status (signal-killed or a
+ * spawn failure) is a failure.
+ */
+export function runFixtureChecks(checks, cwd, run = defaultCheckRunner) {
+    return checks.map((c) => ({
+        name: c.name,
+        passed: run(c.command, cwd).status === 0,
+    }));
+}
+/**
+ * Score the trajectory signals and fixture-check results of one replay against
+ * its {@link BenchmarkExpect}. Pure: only the asserted fields are checked, so an
+ * empty expectation always passes. Every shortfall is accumulated so a report
+ * can show all of them at once.
+ */
+export function evaluateExpectation(expect, signals, checks) {
+    const failures = [];
+    if (expect.succeeded !== undefined && signals.succeeded !== expect.succeeded) {
+        failures.push(`succeeded: expected ${expect.succeeded}, got ${signals.succeeded}` +
+            (signals.exitReason ? ` (exit: ${signals.exitReason})` : ""));
+    }
+    if (expect.maxCostUsd !== undefined && signals.costUsd > expect.maxCostUsd) {
+        failures.push(`cost: $${signals.costUsd} exceeds ceiling $${expect.maxCostUsd}`);
+    }
+    for (const c of checks) {
+        if (!c.passed)
+            failures.push(`check '${c.name}' failed`);
+    }
+    return { passed: failures.length === 0, failures };
+}
+/**
+ * Read and parse a benchmark suite JSON file. Errors (missing file, malformed
+ * JSON, schema violation) are re-thrown qualified with the file path.
+ */
+export function readBenchmarkSuite(path) {
+    let text;
+    try {
+        text = readFileSync(path, "utf8");
+    }
+    catch (e) {
+        throw new Error(`benchmark suite ${path}: cannot read (${e.message})`);
+    }
+    let raw;
+    try {
+        raw = JSON.parse(text);
+    }
+    catch (e) {
+        throw new Error(`benchmark suite ${path}: invalid JSON (${e.message})`);
+    }
+    return parseBenchmarkSuite(raw);
+}
+//# sourceMappingURL=bench.js.map

package/dist/bench.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bench.js","sourceRoot":"","sources":["../src/bench.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAGvC,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AA0D3C,MAAM,IAAI,GAA4B,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;AAEjE,SAAS,QAAQ,CAAC,GAAY,EAAE,GAAW;IACzC,IAAI,GAAG,IAAI,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,sBAAsB,CAAC,CAAC;IAChD,CAAC;IACD,OAAO,GAA8B,CAAC;AACxC,CAAC;AAED,SAAS,aAAa,CAAC,GAA4B,EAAE,GAAW,EAAE,GAAW;IAC3E,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;IACnB,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,MAAM,GAAG,8BAA8B,CAAC,CAAC;IACjE,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,WAAW,CAAC,GAAY,EAAE,GAAW;IAC5C,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IACjC,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,SAAS,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAoB,EAAE,CAAC;IAEnC,IAAI,GAAG,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QAChC,IAAI,OAAO,GAAG,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YACvC,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,wCAAwC,CAAC,CAAC;QAClE,CAAC;QACD,MAAM,CAAC,SAAS,GAAG,GAAG,CAAC,SAAS,CAAC;IACnC,CAAC;IACD,IAAI,GAAG,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QACjC,IAAI,OAAO,GAAG,CAAC,UAAU,KAAK,QAAQ,IAAI,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;YACvE,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,wCAAwC,CAAC,CAAC;QAClE,CAAC;QACD,MAAM,CAAC,UAAU,GAAG,GAAG,CAAC,UAAU,CAAC;IACrC,CAAC;IACD,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC7B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,oCAAoC,CAAC,CAAC;QAC9D,CAAC;QACD,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACtC,MAAM,EAAE,GAAG,QAAQ,CAAC,CAAC,EAAE,GAAG,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;YACrD,OAAO;gBACL,IAAI,EAAE,aAAa,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,GAAG,kBAAkB,CAAC,GAAG,CAAC;gBAC7D,OAAO,EAAE,aAAa,CAAC,EAAE,EAAE,SAAS,EAAE,GAAG,GAAG,kBAAkB,CAAC,GAAG,CAAC;aACpE,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAY;IAC7C,MAAM,GAAG,GAAG,gBAAgB,CAAC;IAC7B,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAC/B,MAAM,EAAE,GAAG,aAAa,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;IACzC,MAAM,OAAO,GAAG,mBAAmB,EAAE,GAAG,CAAC;IAEzC,MAAM,GAAG,GAAG,aAAa,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IAC/C,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAmB,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,GAAG,OAAO,0BAA0B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzE,CAAC;IAED,IAAI,IAAI,GAAa,EAAE,CAAC;IACxB,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC3B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,EAAE,CAAC;YAC5E,MAAM,IAAI,KAAK,CAAC,GAAG,OAAO,sCAAsC,CAAC,CAAC;QACpE,CAAC;QACD,IAAI,GAAG,GAAG,CAAC,IAAgB,CAAC;IAC9B,CAAC;IAED,IAAI,GAAG,GAA2B,EAAE,CAAC;IACrC,IAAI,GAAG,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;QAC1B,MAAM,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,OAAO,MAAM,CAAC,CAAC;QAC/C,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC;YACxC,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CAAC,GAAG,OAAO,UAAU,CAAC,oBAAoB,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;QACD,GAAG,GAAG,EAA4B,CAAC;IACrC,CAAC;IAED,OAAO;QACL,EAAE;QACF,IAAI,EAAE,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC;QACzC,OAAO,EAAE,aAAa,CAAC,GAAG,EAAE,SAAS,EAAE,OAAO,CAAC;QAC/C,GAAG,EAAE,GAAmB;QACxB,oFAAoF;QACpF,MAAM,EAAE,aAAa,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC;QAC1C,IAAI;QACJ,GAAG;QACH,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC;KACzC,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,MAAe,EAAE,GAAW;IACjD,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,6BAA6B,CAAC,CAAC;IACvD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,GAAY;IAC9C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IACjE,CAAC;IACD,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAClE,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAcD,MAAM,kBAAkB,GAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,EAAE;IACvD,MAAM,CAAC,GAAG,SAAS,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC9E,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;AAC9B,CAAC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAC9B,MAAwB,EACxB,GAAW,EACX,MAAmB,kBAAkB;IAErC,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACxB,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC;KACzC,CAAC,CAAC,CAAC;AACN,CAAC;AAUD;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CACjC,MAAuB,EACvB,OAAoB,EACpB,MAAqB;IAErB,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS,IAAI,OAAO,CAAC,SAAS,KAAK,MAAM,CAAC,SAAS,EAAE,CAAC;QAC7E,QAAQ,CAAC,IAAI,CACX,uBAAuB,MAAM,CAAC,SAAS,SAAS,OAAO,CAAC,SAAS,EAAE;YACjE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW,OAAO,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAC/D,CAAC;IACJ,CAAC;IACD,IAAI,MAAM,CAAC,UAAU,KAAK,SAAS,IAAI,OAAO,CAAC,OAAO,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;QAC3E,QAAQ,CAAC,IAAI,CACX,UAAU,OAAO,CAAC,OAAO,qBAAqB,MAAM,CAAC,UAAU,EAAE,CAClE,CAAC;IACJ,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,CAAC,CAAC,MAAM;YAAE,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC;IAC3D,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,QAAQ,EAAE,CAAC;AACrD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACH,IAAI,GAAG,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACpC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,mBAAmB,IAAI,kBAAmB,CAAW,CAAC,OAAO,GAAG,CAAC,CAAC;IACpF,CAAC;IACD,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,mBAAmB,IAAI,mBAAoB,CAAW,CAAC,OAAO,GAAG,CAAC,CAAC;IACrF,CAAC;IACD,OAAO,mBAAmB,CAAC,GAAG,CAAC,CAAC;AAClC,CAAC"}

package/dist/eval-run.d.ts

@@ -0,0 +1,61 @@
+import { type BenchmarkCheck, type BenchmarkTask, type CheckResult } from "./bench.js";
+import { type RunManifest, type StageRecord } from "./run-report.js";
+/** A named configuration overlay replayed against every benchmark task. */
+export type EvalConfig = {
+    /** Report column label (e.g. "baseline", "panel", "codex"). */
+    label: string;
+    /** Extra CLI flags layered on top of the task's own args. */
+    args: string[];
+    /** Env overrides layered on top of the task's own env. */
+    env: Record<string, string>;
+};
+/** One concrete replay: a task under a config, resolved to a bin invocation. */
+export type EvalInvocation = {
+    task: BenchmarkTask;
+    config: EvalConfig;
+    /** Absolute fixture workspace dir the run executes in. */
+    fixtureDir: string;
+    /** Bin name to spawn (`otto-afk` / `otto-ghafk`). */
+    bin: string;
+    /** Planned iteration count. */
+    iterations: number;
+    /** Final argv passed to the bin (task args + config args). */
+    args: string[];
+    /** Env overrides merged for the run (task env + config env). */
+    env: Record<string, string>;
+};
+/**
+ * Drives one otto replay and reports the run id its evidence bundle was written
+ * under. Injectable so {@link runEval} is unit-testable without spawning real,
+ * paid model runs (the default spawns the bin and returns the fixture's latest
+ * run id).
+ */
+export type EvalInvoker = (inv: EvalInvocation) => Promise<{
+    runId: string;
+}>;
+/** Injectable host surface for {@link runEval}. */
+export type EvalDeps = {
+    env: NodeJS.ProcessEnv;
+    cwd: string;
+    out: (msg: string) => void;
+    err: (msg: string) => void;
+    invoke: EvalInvoker;
+    readManifest: (workspaceDir: string, runId: string) => RunManifest | null;
+    readStageRecords: (workspaceDir: string, runId: string) => StageRecord[];
+    runChecks: (checks: BenchmarkCheck[], cwd: string) => CheckResult[];
+};
+/**
+ * Validate a raw eval-config matrix (array of `{label, args?, env?}`). Throws on
+ * a non-array or a config missing its label. Pure.
+ */
+export declare function parseEvalConfigs(raw: unknown): EvalConfig[];
+/**
+ * Drive the `otto-eval` command: load a benchmark suite and a config matrix,
+ * replay every task under every config (via the injectable invoker — this is the
+ * paid, model-dependent half of the eval suite, never run in CI), score each
+ * run's evidence bundle, run its fixture checks, and print a per-task comparison
+ * table plus a PASS/FAIL verdict per config. Resolves to a process exit code:
+ * `0` when every expectation held, `1` otherwise.
+ */
+export declare function runEval(argv: string[], deps?: EvalDeps): Promise<number>;
+//# sourceMappingURL=eval-run.d.ts.map

package/dist/eval-run.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-run.d.ts","sourceRoot":"","sources":["../src/eval-run.ts"],"names":[],"mappings":"AAIA,OAAO,EAIL,KAAK,cAAc,EACnB,KAAK,aAAa,EAClB,KAAK,WAAW,EACjB,MAAM,YAAY,CAAC;AAEpB,OAAO,EAIL,KAAK,WAAW,EAChB,KAAK,WAAW,EACjB,MAAM,iBAAiB,CAAC;AAEzB,2EAA2E;AAC3E,MAAM,MAAM,UAAU,GAAG;IACvB,+DAA+D;IAC/D,KAAK,EAAE,MAAM,CAAC;IACd,6DAA6D;IAC7D,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,0DAA0D;IAC1D,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC7B,CAAC;AAEF,gFAAgF;AAChF,MAAM,MAAM,cAAc,GAAG;IAC3B,IAAI,EAAE,aAAa,CAAC;IACpB,MAAM,EAAE,UAAU,CAAC;IACnB,0DAA0D;IAC1D,UAAU,EAAE,MAAM,CAAC;IACnB,qDAAqD;IACrD,GAAG,EAAE,MAAM,CAAC;IACZ,+BAA+B;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,8DAA8D;IAC9D,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,gEAAgE;IAChE,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC7B,CAAC;AAEF;;;;;GAKG;AACH,MAAM,MAAM,WAAW,GAAG,CAAC,GAAG,EAAE,cAAc,KAAK,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC;AAE9E,mDAAmD;AACnD,MAAM,MAAM,QAAQ,GAAG;IACrB,GAAG,EAAE,MAAM,CAAC,UAAU,CAAC;IACvB,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;IAC3B,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;IAC3B,MAAM,EAAE,WAAW,CAAC;IACpB,YAAY,EAAE,CAAC,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,WAAW,GAAG,IAAI,CAAC;IAC1E,gBAAgB,EAAE,CAAC,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,WAAW,EAAE,CAAC;IACzE,SAAS,EAAE,CAAC,MAAM,EAAE,cAAc,EAAE,EAAE,GAAG,EAAE,MAAM,KAAK,WAAW,EAAE,CAAC;CACrE,CAAC;AAkCF;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,OAAO,GAAG,UAAU,EAAE,CA0B3D;AAuBD;;;;;;;GAOG;AACH,wBAAsB,OAAO,CAC3B,IAAI,EAAE,MAAM,EAAE,EACd,IAAI,GAAE,QAAsB,GAC3B,OAAO,CAAC,MAAM,CAAC,CA+EjB"}

package/dist/eval-run.js

@@ -0,0 +1,162 @@
+import { spawn } from "node:child_process";
+import { readFileSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import { evaluateExpectation, readBenchmarkSuite, runFixtureChecks, } from "./bench.js";
+import { compareTrajectories, scoreTrajectory } from "./eval.js";
+import { listRunIds, readManifest as readManifestFs, readStageRecords as readStageRecordsFs, } from "./run-report.js";
+const defaultInvoke = async (inv) => {
+    const argv = inv.bin === "otto-afk"
+        ? [inv.task.inputs, String(inv.iterations), ...inv.args]
+        : [String(inv.iterations), ...inv.args];
+    await new Promise((res, rej) => {
+        const child = spawn(inv.bin, argv, {
+            cwd: inv.fixtureDir,
+            env: { ...process.env, ...inv.env },
+            stdio: "inherit",
+        });
+        child.on("error", rej);
+        child.on("close", () => res());
+    });
+    const ids = listRunIds(inv.fixtureDir);
+    return { runId: ids[ids.length - 1] ?? "" };
+};
+const defaultDeps = {
+    env: process.env,
+    cwd: process.cwd(),
+    out: (m) => process.stdout.write(`${m}\n`),
+    err: (m) => process.stderr.write(`${m}\n`),
+    invoke: defaultInvoke,
+    readManifest: readManifestFs,
+    readStageRecords: readStageRecordsFs,
+    runChecks: runFixtureChecks,
+};
+const USAGE = "Usage: otto-eval <suite.json> [<configs.json>] [--iterations <n>]";
+/**
+ * Validate a raw eval-config matrix (array of `{label, args?, env?}`). Throws on
+ * a non-array or a config missing its label. Pure.
+ */
+export function parseEvalConfigs(raw) {
+    if (!Array.isArray(raw)) {
+        throw new Error("eval configs: expected an array of {label, args?, env?}");
+    }
+    return raw.map((c, i) => {
+        if (c == null || typeof c !== "object" || Array.isArray(c)) {
+            throw new Error(`eval config [${i}]: expected an object`);
+        }
+        const rec = c;
+        if (typeof rec.label !== "string" || rec.label.length === 0) {
+            throw new Error(`eval config [${i}]: 'label' must be a non-empty string`);
+        }
+        const args = rec.args;
+        if (args !== undefined && (!Array.isArray(args) || args.some((a) => typeof a !== "string"))) {
+            throw new Error(`eval config '${rec.label}': 'args' must be an array of strings`);
+        }
+        const env = rec.env;
+        if (env !== undefined && (env == null || typeof env !== "object" || Array.isArray(env))) {
+            throw new Error(`eval config '${rec.label}': 'env' must be an object`);
+        }
+        return {
+            label: rec.label,
+            args: args ?? [],
+            env: env ?? {},
+        };
+    });
+}
+function parseArgs(argv) {
+    const parsed = { iterations: 3, help: false };
+    const positionals = [];
+    for (let i = 0; i < argv.length; i++) {
+        const a = argv[i];
+        if (a === "-h" || a === "--help")
+            parsed.help = true;
+        else if (a === "--iterations")
+            parsed.iterations = Number(argv[++i]);
+        else
+            positionals.push(a);
+    }
+    parsed.suitePath = positionals[0];
+    parsed.configsPath = positionals[1];
+    return parsed;
+}
+/**
+ * Drive the `otto-eval` command: load a benchmark suite and a config matrix,
+ * replay every task under every config (via the injectable invoker — this is the
+ * paid, model-dependent half of the eval suite, never run in CI), score each
+ * run's evidence bundle, run its fixture checks, and print a per-task comparison
+ * table plus a PASS/FAIL verdict per config. Resolves to a process exit code:
+ * `0` when every expectation held, `1` otherwise.
+ */
+export async function runEval(argv, deps = defaultDeps) {
+    const args = parseArgs(argv);
+    if (args.help) {
+        deps.out(USAGE);
+        return 0;
+    }
+    if (!args.suitePath) {
+        deps.err(`No benchmark suite given.\n${USAGE}`);
+        return 1;
+    }
+    const suitePath = resolve(deps.cwd, args.suitePath);
+    let tasks;
+    try {
+        tasks = readBenchmarkSuite(suitePath);
+    }
+    catch (e) {
+        deps.err(e.message);
+        return 1;
+    }
+    let configs = [{ label: "default", args: [], env: {} }];
+    if (args.configsPath) {
+        try {
+            configs = parseEvalConfigs(JSON.parse(readFileSync(resolve(deps.cwd, args.configsPath), "utf8")));
+        }
+        catch (e) {
+            deps.err(`eval configs ${args.configsPath}: ${e.message}`);
+            return 1;
+        }
+    }
+    const suiteDir = dirname(suitePath);
+    let allPassed = true;
+    for (const task of tasks) {
+        const fixtureDir = resolve(suiteDir, task.fixture);
+        const labelled = [];
+        const verdictLines = [];
+        for (const config of configs) {
+            const inv = {
+                task,
+                config,
+                fixtureDir,
+                bin: task.bin,
+                iterations: args.iterations,
+                args: [...task.args, ...config.args],
+                env: { ...task.env, ...config.env },
+            };
+            const { runId } = await deps.invoke(inv);
+            const manifest = deps.readManifest(fixtureDir, runId);
+            if (!manifest) {
+                allPassed = false;
+                verdictLines.push(`  - ${config.label}: FAIL (no evidence bundle for run '${runId}')`);
+                continue;
+            }
+            const signals = scoreTrajectory(manifest, deps.readStageRecords(fixtureDir, runId));
+            const checks = deps.runChecks(task.expect.checks ?? [], fixtureDir);
+            const verdict = evaluateExpectation(task.expect, signals, checks);
+            if (!verdict.passed)
+                allPassed = false;
+            labelled.push({ label: config.label, signals });
+            verdictLines.push(verdict.passed
+                ? `  - ${config.label}: PASS`
+                : `  - ${config.label}: FAIL (${verdict.failures.join("; ")})`);
+        }
+        deps.out(`## ${task.id} (${task.kind})`);
+        deps.out("");
+        deps.out(compareTrajectories(labelled));
+        deps.out("");
+        deps.out("Verdicts:");
+        for (const line of verdictLines)
+            deps.out(line);
+        deps.out("");
+    }
+    return allPassed ? 0 : 1;
+}
+//# sourceMappingURL=eval-run.js.map

package/dist/eval-run.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-run.js","sourceRoot":"","sources":["../src/eval-run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAE7C,OAAO,EACL,mBAAmB,EACnB,kBAAkB,EAClB,gBAAgB,GAIjB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjE,OAAO,EACL,UAAU,EACV,YAAY,IAAI,cAAc,EAC9B,gBAAgB,IAAI,kBAAkB,GAGvC,MAAM,iBAAiB,CAAC;AAgDzB,MAAM,aAAa,GAAgB,KAAK,EAAE,GAAG,EAAE,EAAE;IAC/C,MAAM,IAAI,GACR,GAAG,CAAC,GAAG,KAAK,UAAU;QACpB,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC;QACxD,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;IAC5C,MAAM,IAAI,OAAO,CAAO,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;QACnC,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,EAAE;YACjC,GAAG,EAAE,GAAG,CAAC,UAAU;YACnB,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,GAAG,EAAE;YACnC,KAAK,EAAE,SAAS;SACjB,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QACvB,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACvC,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;AAC9C,CAAC,CAAC;AAEF,MAAM,WAAW,GAAa;IAC5B,GAAG,EAAE,OAAO,CAAC,GAAG;IAChB,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE;IAClB,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;IAC1C,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;IAC1C,MAAM,EAAE,aAAa;IACrB,YAAY,EAAE,cAAc;IAC5B,gBAAgB,EAAE,kBAAkB;IACpC,SAAS,EAAE,gBAAgB;CAC5B,CAAC;AAEF,MAAM,KAAK,GACT,mEAAmE,CAAC;AAEtE;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,GAAY;IAC3C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;IAC7E,CAAC;IACD,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACtB,IAAI,CAAC,IAAI,IAAI,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3D,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,uBAAuB,CAAC,CAAC;QAC5D,CAAC;QACD,MAAM,GAAG,GAAG,CAA4B,CAAC;QACzC,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5D,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,uCAAuC,CAAC,CAAC;QAC5E,CAAC;QACD,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;QACtB,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,EAAE,CAAC;YAC5F,MAAM,IAAI,KAAK,CAAC,gBAAgB,GAAG,CAAC,KAAK,uCAAuC,CAAC,CAAC;QACpF,CAAC;QACD,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC;QACpB,IAAI,GAAG,KAAK,SAAS,IAAI,CAAC,GAAG,IAAI,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YACxF,MAAM,IAAI,KAAK,CAAC,gBAAgB,GAAG,CAAC,KAAK,4BAA4B,CAAC,CAAC;QACzE,CAAC;QACD,OAAO;YACL,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,IAAI,EAAG,IAAiB,IAAI,EAAE;YAC9B,GAAG,EAAG,GAA8B,IAAI,EAAE;SAC3C,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AASD,SAAS,SAAS,CAAC,IAAc;IAC/B,MAAM,MAAM,GAAe,EAAE,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IAC1D,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,QAAQ;YAAE,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC;aAChD,IAAI,CAAC,KAAK,cAAc;YAAE,MAAM,CAAC,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;;YAChE,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3B,CAAC;IACD,MAAM,CAAC,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;IAClC,MAAM,CAAC,WAAW,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;IACpC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,IAAc,EACd,OAAiB,WAAW;IAE5B,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC7B,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QACd,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAChB,OAAO,CAAC,CAAC;IACX,CAAC;IACD,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;QACpB,IAAI,CAAC,GAAG,CAAC,8BAA8B,KAAK,EAAE,CAAC,CAAC;QAChD,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACpD,IAAI,KAAsB,CAAC;IAC3B,IAAI,CAAC;QACH,KAAK,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;IACxC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,GAAG,CAAE,CAAW,CAAC,OAAO,CAAC,CAAC;QAC/B,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,OAAO,GAAiB,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;IACtE,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;QACrB,IAAI,CAAC;YACH,OAAO,GAAG,gBAAgB,CACxB,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC,CAAC,CACtE,CAAC;QACJ,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,GAAG,CAAC,gBAAgB,IAAI,CAAC,WAAW,KAAM,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;YACtE,OAAO,CAAC,CAAC;QACX,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IACpC,IAAI,SAAS,GAAG,IAAI,CAAC;IAErB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QACnD,MAAM,QAAQ,GAAqE,EAAE,CAAC;QACtF,MAAM,YAAY,GAAa,EAAE,CAAC;QAElC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAmB;gBAC1B,IAAI;gBACJ,MAAM;gBACN,UAAU;gBACV,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC;gBACpC,GAAG,EAAE,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,GAAG,EAAE;aACpC,CAAC;YACF,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YACtD,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,SAAS,GAAG,KAAK,CAAC;gBAClB,YAAY,CAAC,IAAI,CAAC,OAAO,MAAM,CAAC,KAAK,uCAAuC,KAAK,IAAI,CAAC,CAAC;gBACvF,SAAS;YACX,CAAC;YACD,MAAM,OAAO,GAAG,eAAe,CAAC,QAAQ,EAAE,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC;YACpF,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,EAAE,UAAU,CAAC,CAAC;YACpE,MAAM,OAAO,GAAG,mBAAmB,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;YAClE,IAAI,CAAC,OAAO,CAAC,MAAM;gBAAE,SAAS,GAAG,KAAK,CAAC;YACvC,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;YAChD,YAAY,CAAC,IAAI,CACf,OAAO,CAAC,MAAM;gBACZ,CAAC,CAAC,OAAO,MAAM,CAAC,KAAK,QAAQ;gBAC7B,CAAC,CAAC,OAAO,MAAM,CAAC,KAAK,WAAW,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CACjE,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;QACzC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACb,IAAI,CAAC,GAAG,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC,CAAC;QACxC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACb,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACtB,KAAK,MAAM,IAAI,IAAI,YAAY;YAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAChD,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACf,CAAC;IAED,OAAO,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC3B,CAAC"}

package/dist/eval.d.ts

@@ -0,0 +1,49 @@
+import type { RunManifest, StageRecord } from "./run-report.js";
+/**
+ * The multi-signal outcome of one Otto run, derived purely from its recorded
+ * trajectory (the #39 evidence bundle: a {@link RunManifest} plus its
+ * {@link StageRecord}s). These are the signals that need no fixture re-run — the
+ * deterministic, CI-runnable subset of the harness evaluation suite (issue #40).
+ *
+ * Fixture-dependent signals (tests passed, diff correctness, safety events) are
+ * scored separately by the runner against a benchmark task's expected outcome.
+ */
+export type EvalSignals = {
+    /** Run reached a success exit reason (`complete`/`done`). */
+    succeeded: boolean;
+    /** Terminal exit reason, or `null` for an un-finalized/interrupted run. */
+    exitReason: string | null;
+    /** Iterations completed, or `null` when the manifest is un-finalized. */
+    completedIterations: number | null;
+    /** Number of stage records in the trajectory. */
+    stageCount: number;
+    /** Stage records that ended in error. */
+    errorStageCount: number;
+    /** Total USD cost of the run. */
+    costUsd: number;
+    /** Sum of all token-usage fields for the run. */
+    totalTokens: number;
+    /** Wall-clock run duration in ms, or `null` when it cannot be computed. */
+    elapsedMs: number | null;
+};
+/**
+ * Derive {@link EvalSignals} from a recorded run trajectory. Pure: no I/O, no
+ * model calls — only arithmetic over the manifest and stage records, so it is
+ * deterministic and unit-testable. `elapsedMs` is `null` when the run is
+ * un-finalized (no `finishedAt`) or either timestamp is unparseable, never NaN.
+ */
+export declare function scoreTrajectory(manifest: RunManifest, stages: StageRecord[]): EvalSignals;
+/** One Otto run's signals tagged with the configuration label that produced it. */
+export type LabelledSignals = {
+    label: string;
+    signals: EvalSignals;
+};
+/**
+ * Render a stable markdown comparison table across labelled runs — one row per
+ * run, one column per {@link EvalSignals} field. Each directional signal (success
+ * up; errors/cost/tokens/elapsed down) marks its best and worst cell, so a
+ * maintainer can read a config A/B at a glance. Pure and deterministic. A column
+ * is marked only when there is a spread across at least two comparable runs.
+ */
+export declare function compareTrajectories(runs: LabelledSignals[]): string;
+//# sourceMappingURL=eval.d.ts.map

package/dist/eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../src/eval.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAGhE;;;;;;;;GAQG;AACH,MAAM,MAAM,WAAW,GAAG;IACxB,6DAA6D;IAC7D,SAAS,EAAE,OAAO,CAAC;IACnB,2EAA2E;IAC3E,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,yEAAyE;IACzE,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,iDAAiD;IACjD,UAAU,EAAE,MAAM,CAAC;IACnB,yCAAyC;IACzC,eAAe,EAAE,MAAM,CAAC;IACxB,iCAAiC;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,iDAAiD;IACjD,WAAW,EAAE,MAAM,CAAC;IACpB,2EAA2E;IAC3E,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B,CAAC;AAIF;;;;;GAKG;AACH,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,WAAW,EACrB,MAAM,EAAE,WAAW,EAAE,GACpB,WAAW,CAYb;AAUD,mFAAmF;AACnF,MAAM,MAAM,eAAe,GAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,WAAW,CAAA;CAAE,CAAC;AAkDtE;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,eAAe,EAAE,GAAG,MAAM,CAoCnE"}