@irisrun/evals 0.1.0

package/dist/evals.d.ts

@@ -0,0 +1,23 @@
+import type { EngineDeps, Json, TurnOutcome } from "@irisrun/core";
+import { type SessionInspection } from "@irisrun/inspect";
+export interface EvalCase<S extends Json> {
+    name: string;
+    build(): {
+        deps: EngineDeps<S>;
+        sessionId: string;
+    };
+    turns?: number;
+}
+export type Scorer<S extends Json> = (inspection: SessionInspection, outcome: TurnOutcome<S>) => Json;
+export interface EvalResult {
+    name: string;
+    score: Json;
+    status: TurnOutcome<Json>["status"] | "open";
+}
+export interface SuiteResult {
+    results: EvalResult[];
+}
+/** Run one eval case to score. Deterministic and reproducible across invocations. */
+export declare function runEval<S extends Json>(c: EvalCase<S>, scorer: Scorer<S>): Promise<EvalResult>;
+/** Run a suite of cases under one scorer; aggregate the reproducible results. */
+export declare function runSuite<S extends Json>(cases: EvalCase<S>[], scorer: Scorer<S>): Promise<SuiteResult>;

package/dist/evals.js

@@ -0,0 +1,33 @@
+// The reproducible-eval arbiter (spec 03 §7): "reproducible evals are the arbiter,
+// not editorial taste." An EvalCase is a DETERMINISTIC scenario; a Scorer reads the
+// recorded session (via @irisrun/inspect) and the last turn outcome. runEval calls
+// case.build() on EVERY invocation, so it gets a FRESH store AND fresh performers
+// (the scripted-model/-tool closure index resets to 0); within a single run the
+// built performers PERSIST across the `turns` (a park→resume advances the index).
+// It runs EXACTLY `turns` (default 1) sequential runTurn calls — NEVER
+// loop-until-finished (a perpetually-parking case must not hang) — and scores the
+// LAST outcome. Same case+scorer → byte-identical score; swapped tactic → different.
+// Runner is core runTurn (no host needed). READ-ONLY scoring.
+import { runTurn } from "@irisrun/core";
+import { inspectSession } from "@irisrun/inspect";
+/** Run one eval case to score. Deterministic and reproducible across invocations. */
+export async function runEval(c, scorer) {
+    const { deps, sessionId } = c.build(); // FRESH store + performers (index resets to 0)
+    const turns = c.turns ?? 1;
+    let outcome = null;
+    for (let i = 0; i < turns; i++) {
+        // reuse `deps` across turns → performers persist (park→resume advances state)
+        outcome = await runTurn(deps, sessionId);
+    }
+    const inspection = await inspectSession(deps.store, sessionId);
+    const status = outcome ? outcome.status : "open";
+    const score = outcome ? scorer(inspection, outcome) : null;
+    return { name: c.name, score, status };
+}
+/** Run a suite of cases under one scorer; aggregate the reproducible results. */
+export async function runSuite(cases, scorer) {
+    const results = [];
+    for (const c of cases)
+        results.push(await runEval(c, scorer));
+    return { results };
+}

package/dist/index.d.ts

@@ -0,0 +1,5 @@
+export declare const PACKAGE = "@irisrun/evals";
+export { runEval, runSuite } from "./evals.js";
+export type { EvalCase, Scorer, EvalResult, SuiteResult } from "./evals.js";
+export { reproduce } from "./reproduce.js";
+export type { ReproReport } from "./reproduce.js";

package/dist/index.js

@@ -0,0 +1,4 @@
+// @irisrun/evals — public surface (host; reproducible-eval arbiter, read-only scoring).
+export const PACKAGE = "@irisrun/evals";
+export { runEval, runSuite } from "./evals.js";
+export { reproduce } from "./reproduce.js";

package/dist/reproduce.d.ts

@@ -0,0 +1,18 @@
+import type { Json } from "@irisrun/core";
+import type { EvalCase, Scorer, EvalResult } from "./evals.js";
+export type ReproReport = {
+    name: string;
+    reproducible: boolean;
+    runs: number;
+    result: EvalResult;
+    journalDigest: string;
+    divergence?: {
+        run: number;
+        field: "score" | "status" | "journal";
+    };
+};
+/** Run an EvalCase N≥2 times and prove byte-identical results. Deterministic given a
+ *  deterministic case; locates the first divergence otherwise. */
+export declare function reproduce<S extends Json>(c: EvalCase<S>, scorer: Scorer<S>, opts?: {
+    runs?: number;
+}): Promise<ReproReport>;

package/dist/reproduce.js

@@ -0,0 +1,69 @@
+// reproduce() (roadmap P2-8): makes "reproducible evals" an EXPLICIT, provable
+// feature, not just an implicit property. It runs an EvalCase N independent times
+// (each `case.build()` is a fresh store + performers, index reset to 0 — the EvalCase
+// contract) and proves byte-identical {score, status, FULL-journal digest} across
+// runs. The journal digest is the strong claim: not only does the score match, the
+// entire recorded session is byte-identical run-to-run. First divergence is located.
+//
+// PRECONDITION: the full-journal digest reads from seq 0, so it covers the COMPLETE
+// journal only when the case does not truncate (the eval norm — short cases, default
+// no snapshot, or keepHistory). If a case truncates, the digest covers the retained
+// tail; the reproducibility verdict stays sound (both runs truncate identically).
+import { runTurn, canonicalize, decode } from "@irisrun/core";
+import { inspectSession } from "@irisrun/inspect";
+// A tiny pure FNV-1a (32-bit) hex hash — a short fingerprint, not a security hash.
+// Inlined so @irisrun/evals adds no dependency (mirrors @irisrun/audit's fnv.ts).
+function fnv1a32hex(s) {
+    let h = 0x811c9dc5;
+    for (let i = 0; i < s.length; i++) {
+        h ^= s.charCodeAt(i);
+        h = Math.imul(h, 0x01000193) >>> 0;
+    }
+    return (h >>> 0).toString(16).padStart(8, "0");
+}
+/** Run an EvalCase N≥2 times and prove byte-identical results. Deterministic given a
+ *  deterministic case; locates the first divergence otherwise. */
+export async function reproduce(c, scorer, opts) {
+    const runs = Math.max(2, opts?.runs ?? 2); // reproducibility needs ≥2 runs
+    const turns = c.turns ?? 1;
+    let firstResult = null;
+    let canonical = null;
+    let divergence;
+    for (let i = 0; i < runs; i++) {
+        const { deps, sessionId } = c.build(); // FRESH store + performers (index resets to 0)
+        let outcome = null;
+        for (let t = 0; t < turns; t++)
+            outcome = await runTurn(deps, sessionId);
+        const inspection = await inspectSession(deps.store, sessionId);
+        const status = outcome ? outcome.status : "open";
+        const score = outcome ? scorer(inspection, outcome) : null;
+        const rows = await deps.store.readJournal(sessionId, 0); // full retained journal
+        const journalDigest = fnv1a32hex(canonicalize(rows.map((r) => decode(r.bytes))));
+        const sig = { score: canonicalize(score), status: canonicalize(status), journalDigest };
+        if (i === 0) {
+            firstResult = { name: c.name, score, status };
+            canonical = sig;
+        }
+        else if (canonical) {
+            // First divergence wins, in field-precedence order score → status → journal
+            // (a run differing in several fields reports the first by this order).
+            if (sig.score !== canonical.score)
+                divergence = { run: i, field: "score" };
+            else if (sig.status !== canonical.status)
+                divergence = { run: i, field: "status" };
+            else if (sig.journalDigest !== canonical.journalDigest)
+                divergence = { run: i, field: "journal" };
+            if (divergence)
+                break;
+        }
+    }
+    return {
+        name: c.name,
+        reproducible: divergence === undefined,
+        runs,
+        // firstResult/canonical are always set (runs ≥ 2 ⇒ the i===0 branch ran)
+        result: firstResult,
+        journalDigest: canonical.journalDigest,
+        divergence,
+    };
+}

package/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "@irisrun/evals",
+  "version": "0.1.0",
+  "type": "module",
+  "description": "Iris reproducible-eval arbiter (spec 03 §7) — run a deterministic scenario (fresh store + scripted performers per run), then score the recorded session via @irisrun/inspect. The arbiter is reproducibility, not taste: the same case+scorer re-runs byte-identically; a swapped tactic scores differently but reproducibly. Runner is core runTurn; deps @irisrun/core + @irisrun/inspect.",
+  "exports": {
+    ".": {
+      "iris-src": "./src/index.ts",
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "dependencies": {
+    "@irisrun/core": "^0.1.0",
+    "@irisrun/inspect": "^0.1.0"
+  },
+  "license": "MIT",
+  "engines": {
+    "node": ">=24"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/xoai/iris.git",
+    "directory": "packages/evals"
+  },
+  "homepage": "https://github.com/xoai/iris#readme",
+  "files": [
+    "dist"
+  ]
+}