@crewhaus/eval-runner 0.1.8 → 0.2.1

package/dist/aggregate.d.ts

@@ -1,5 +1,14 @@
 import type { SampleResult } from "./types";
 export declare function quantile(sorted: ReadonlyArray<number>, q: number): number;
+/**
+ * Retry honesty (RunEvalOptions.retryErrors): a retried sample appears in
+ * `samples` exactly ONCE — the retry REPLACED the errored first attempt —
+ * so passRate's denominator still counts each dataset sample once. A retry
+ * that passes counts as a normal pass; a retry that errors again lands in
+ * `errorCount` (and, like every errored sample, drags passRate down while
+ * staying out of meanScore/latency/token aggregates). The discarded first
+ * attempt is counted nowhere, by design: it was infra noise, not a result.
+ */
 export declare function aggregate(samples: ReadonlyArray<SampleResult>): {
     passRate: number;
     meanScore: number;

package/dist/aggregate.js

@@ -11,6 +11,15 @@ export function quantile(sorted, q) {
     const fract = idx - lo;
     return (sorted[lo] ?? 0) * (1 - fract) + (sorted[hi] ?? 0) * fract;
 }
+/**
+ * Retry honesty (RunEvalOptions.retryErrors): a retried sample appears in
+ * `samples` exactly ONCE — the retry REPLACED the errored first attempt —
+ * so passRate's denominator still counts each dataset sample once. A retry
+ * that passes counts as a normal pass; a retry that errors again lands in
+ * `errorCount` (and, like every errored sample, drags passRate down while
+ * staying out of meanScore/latency/token aggregates). The discarded first
+ * attempt is counted nowhere, by design: it was infra noise, not a result.
+ */
 export function aggregate(samples) {
     const ok = samples.filter((s) => s.error === undefined);
     const total = samples.length;

package/dist/index.js

@@ -86,6 +86,8 @@ export async function runEval(args) {
         startedAt,
         specHash,
         datasetName: dataset.name,
+        ...(opts.datasetHash !== undefined ? { datasetHash: opts.datasetHash } : {}),
+        ...(opts.gradersHash !== undefined ? { gradersHash: opts.gradersHash } : {}),
         graderNames: graders.map((g) => g.name),
         model: ir.agent.model,
         ...(opts.judgeModel !== undefined ? { judgeModel: opts.judgeModel } : {}),
@@ -117,7 +119,7 @@ export async function runEval(args) {
             throw new RunnerError(`run interrupted before sample "${sample.id}"`);
         }
         try {
-            return await runSample({
+            const runOnce = () => runSample({
                 sample,
                 invoker,
                 graders,
@@ -125,6 +127,20 @@ export async function runEval(args) {
                 model: ir.agent.model,
                 ...(opts.seed !== undefined ? { seed: opts.seed } : {}),
             });
+            const first = await runOnce();
+            // Noise auto-retry (failure-arbiter item 7): an errored SampleResult
+            // means the INVOKER failed (provider timeout, 429, sandbox blip),
+            // and a graderError means a GRADER threw (judge infra blip) — both
+            // are infra noise, not graded failures. Retry exactly once within
+            // the run; the retried outcome replaces the errored one wholesale
+            // (the second runSample rewrites the same per-sample artifact dir)
+            // and is tagged `retried: true` so reports and triage can tell.
+            // Skipped on SIGINT or when the caller opted out (`--no-retry`).
+            const infraNoise = first.error !== undefined || first.graderError !== undefined;
+            if (!infraNoise || opts.retryErrors === false || interrupted) {
+                return first;
+            }
+            return { ...(await runOnce()), retried: true };
         }
         finally {
             release();
@@ -163,6 +179,8 @@ export async function runEval(args) {
         config: {
             specHash,
             datasetName: dataset.name,
+            ...(opts.datasetHash !== undefined ? { datasetHash: opts.datasetHash } : {}),
+            ...(opts.gradersHash !== undefined ? { gradersHash: opts.gradersHash } : {}),
             graderNames: graders.map((g) => g.name),
             model: ir.agent.model,
             ...(opts.judgeModel !== undefined ? { judgeModel: opts.judgeModel } : {}),

package/dist/run-sample.js

@@ -79,20 +79,28 @@ export async function runSample(args) {
         latencyMs,
     };
     const perGrader = [];
+    // A grader throwing is grader INFRA noise (judge 429/timeout), not a graded
+    // verdict on the agent's output. It still lands as a failed perGrader entry
+    // (so `overall` honestly fails), but the structured `graderError` below
+    // lets the run loop retry the sample and triage classify it as noise.
+    const graderErrors = [];
     for (const g of graders) {
         let result;
         try {
             result = await g.grader(sample, runResult);
         }
         catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            graderErrors.push(`${g.name}: ${msg}`);
             result = {
                 passed: false,
                 score: 0,
-                rationale: `grader threw: ${err instanceof Error ? err.message : String(err)}`,
+                rationale: `grader threw: ${msg}`,
             };
         }
         perGrader.push({ name: g.name, ...result });
     }
+    const graderError = graderErrors.length > 0 ? `grader threw: ${graderErrors.join("; ")}` : undefined;
     // Overall = AND of all graders, score = mean.
     const overall = {
         passed: error === undefined && perGrader.every((g) => g.passed),
@@ -113,6 +121,7 @@ export async function runSample(args) {
         agentOutput,
         grades: { overall, perGrader },
         ...(error !== undefined ? { error } : {}),
+        ...(graderError !== undefined ? { graderError } : {}),
     };
     await Bun.write(join(sampleDir, "grades.json"), JSON.stringify(sampleResult.grades, null, 2));
     await Bun.write(join(sampleDir, "meta.json"), JSON.stringify({
@@ -125,6 +134,7 @@ export async function runSample(args) {
         tokens,
         model,
         ...(error !== undefined ? { error } : {}),
+        ...(graderError !== undefined ? { graderError } : {}),
         ...(args.seed !== undefined ? { seed: args.seed } : {}),
     }, null, 2));
     return sampleResult;

package/dist/types.d.ts

@@ -52,6 +52,24 @@ export type SampleResult = {
         } & GradeResult>;
     };
     readonly error?: string;
+    /**
+     * Set when one or more GRADERS threw while grading this sample (judge
+     * provider 429/timeout, rubric fetch failure, …) — grader infrastructure
+     * noise, distinct from `error` (the INVOKER failed) and from an honest
+     * graded failure. The thrown grader still contributes a failed
+     * `perGrader` entry (rationale `grader threw: …`), so `grades.overall`
+     * fails; this field preserves the structured evidence so the retry loop
+     * can retry the sample and triage can classify the failure as noise.
+     */
+    readonly graderError?: string;
+    /**
+     * True when this result replaced an ERRORED first attempt via the runner's
+     * bounded noise retry (see {@link RunEvalOptions.retryErrors}) — invoker
+     * errors and grader throws (`graderError`) alike. Set on the retried
+     * outcome regardless of whether the retry passed or errored again; absent
+     * on samples that succeeded (or failed grading) on attempt one.
+     */
+    readonly retried?: boolean;
 };
 export type EvalRunSummary = {
     readonly runId: string;
@@ -74,6 +92,10 @@ export type EvalRunSummary = {
     readonly config: {
         readonly specHash: string;
         readonly datasetName: string;
+        /** sha256 of the dataset file bytes, when the caller supplied one. */
+        readonly datasetHash?: string;
+        /** sha256 of the parsed GradersConfig, when the caller supplied one. */
+        readonly gradersHash?: string;
         readonly graderNames: ReadonlyArray<string>;
         readonly model: string;
         readonly judgeModel?: string;
@@ -90,6 +112,32 @@ export type RunEvalOptions = {
     readonly judgeModel?: string;
     readonly invoker?: AgentInvoker;
     readonly cwd?: string;
+    /**
+     * Retry a sample ONCE, within the run, when its result is an ERROR
+     * (`SampleResult.error` — the INVOKER failed: provider timeout, 429,
+     * sandbox blip) or a GRADER threw (`SampleResult.graderError` — judge
+     * infra noise; the agent may have answered fine). Infra noise, not a
+     * graded failure. The retried outcome replaces the errored one wholesale
+     * (per-sample artifacts included) and is tagged `retried: true`.
+     * Default: true. `crewhaus eval --no-retry` opts out. Interrupted runs
+     * (SIGINT) never retry.
+     */
+    readonly retryErrors?: boolean;
+    /**
+     * Content hash (sha256 hex) of the dataset file the samples came from.
+     * Purely informational — persisted into `run.json` / `results.json` so
+     * downstream consumers (run-history index, dataset-drift detection) can
+     * tell whether two runs saw byte-identical data.
+     */
+    readonly datasetHash?: string;
+    /**
+     * Content hash (sha256 hex) of the parsed GradersConfig the graders came
+     * from. Mirrors {@link datasetHash}: purely informational, persisted into
+     * `run.json` / `results.json` so downstream consumers (notably the item-30
+     * sentinel) can tell whether two runs graded with byte-identical config —
+     * a changed rubric/threshold must not be misattributed to provider drift.
+     */
+    readonly gradersHash?: string;
 };
 export type GraderEntry = {
     readonly name: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crewhaus/eval-runner",
-  "version": "0.1.8",
+  "version": "0.2.1",
   "type": "module",
   "description": "Eval runner — per-sample isolation, concurrency, grading, persistence",
   "main": "dist/index.js",
@@ -15,35 +15,35 @@
     "test": "bun test src"
   },
   "dependencies": {
-    "@crewhaus/agent-context-isolation": "0.1.8",
-    "@crewhaus/compiler": "0.1.8",
-    "@crewhaus/errors": "0.1.8",
-    "@crewhaus/eval-dataset": "0.1.8",
-    "@crewhaus/eval-grader": "0.1.8",
-    "@crewhaus/eval-judge": "0.1.8",
-    "@crewhaus/event-log": "0.1.8",
-    "@crewhaus/hooks-engine": "0.1.8",
-    "@crewhaus/ir": "0.1.8",
-    "@crewhaus/logging": "0.1.8",
-    "@crewhaus/mcp-host": "0.1.8",
-    "@crewhaus/permission-engine": "0.1.8",
-    "@crewhaus/run-context": "0.1.8",
-    "@crewhaus/runtime-core": "0.1.8",
-    "@crewhaus/skills-registry": "0.1.8",
-    "@crewhaus/slash-commands": "0.1.8",
-    "@crewhaus/spec": "0.1.8",
-    "@crewhaus/sub-agent-spawner": "0.1.8",
-    "@crewhaus/tenancy": "0.1.8",
-    "@crewhaus/tool-bash": "0.1.8",
-    "@crewhaus/tool-catalog": "0.1.8",
-    "@crewhaus/tool-fetch": "0.1.8",
-    "@crewhaus/tool-fs": "0.1.8",
-    "@crewhaus/tool-image": "0.1.8",
-    "@crewhaus/tool-mcp": "0.1.8",
-    "@crewhaus/tool-task": "0.1.8",
-    "@crewhaus/tool-todo": "0.1.8",
-    "@crewhaus/tool-web": "0.1.8",
-    "@crewhaus/trace-event-bus": "0.1.8",
+    "@crewhaus/agent-context-isolation": "0.2.1",
+    "@crewhaus/compiler": "0.2.1",
+    "@crewhaus/errors": "0.2.1",
+    "@crewhaus/eval-dataset": "0.2.1",
+    "@crewhaus/eval-grader": "0.2.1",
+    "@crewhaus/eval-judge": "0.2.1",
+    "@crewhaus/event-log": "0.2.1",
+    "@crewhaus/hooks-engine": "0.2.1",
+    "@crewhaus/ir": "0.2.1",
+    "@crewhaus/logging": "0.2.1",
+    "@crewhaus/mcp-host": "0.2.1",
+    "@crewhaus/permission-engine": "0.2.1",
+    "@crewhaus/run-context": "0.2.1",
+    "@crewhaus/runtime-core": "0.2.1",
+    "@crewhaus/skills-registry": "0.2.1",
+    "@crewhaus/slash-commands": "0.2.1",
+    "@crewhaus/spec": "0.2.1",
+    "@crewhaus/sub-agent-spawner": "0.2.1",
+    "@crewhaus/tenancy": "0.2.1",
+    "@crewhaus/tool-bash": "0.2.1",
+    "@crewhaus/tool-catalog": "0.2.1",
+    "@crewhaus/tool-fetch": "0.2.1",
+    "@crewhaus/tool-fs": "0.2.1",
+    "@crewhaus/tool-image": "0.2.1",
+    "@crewhaus/tool-mcp": "0.2.1",
+    "@crewhaus/tool-task": "0.2.1",
+    "@crewhaus/tool-todo": "0.2.1",
+    "@crewhaus/tool-web": "0.2.1",
+    "@crewhaus/trace-event-bus": "0.2.1",
     "zod": "^3.23.8"
   },
   "license": "Apache-2.0",