@m4trix/evals 0.11.0 → 0.13.0

package/dist/cli-simple.cjs

@@ -56,7 +56,8 @@ var defaultRunnerConfig = {
     ],
     excludeDirectories: ["node_modules", "dist", ".next", ".git", ".pnpm-store"]
   },
-  artifactDirectory: ".eval-results"
+  artifactDirectory: ".eval-results",
+  maxConcurrency: 1
 };
 function toRunnerConfigOverrides(config) {
   if (!config) {
@@ -89,6 +90,9 @@ function toRunnerConfigOverrides(config) {
   if (config.artifactDirectory !== void 0) {
     overrides.artifactDirectory = config.artifactDirectory;
   }
+  if (config.maxConcurrency !== void 0) {
+    overrides.maxConcurrency = config.maxConcurrency;
+  }
   if (Object.keys(discovery).length > 0) {
     overrides.discovery = discovery;
   }
@@ -313,6 +317,7 @@ var Metric = {
     const def = {
       id: config.id,
       name: config.name,
+      aggregate: config.aggregate,
       format: config.format,
       make: (data) => ({ id: config.id, data })
     };
@@ -332,6 +337,7 @@ var Score = {
       id: config.id,
       name: config.name,
       displayStrategy: config.displayStrategy,
+      aggregate: config.aggregate,
       format: config.format,
       make: (data, options) => {
         const passed = options?.definePassed !== void 0 ? options.definePassed(data) : void 0;
@@ -350,23 +356,62 @@ function getScoreById(id) {
   return registry2.get(id);
 }
 
+// src/evals/aggregators.ts
+function aggregateAverage(values) {
+  if (values.length === 0) {
+    return { value: 0 };
+  }
+  const sum = values.reduce((s, v) => s + v.value, 0);
+  return { value: sum / values.length };
+}
+function aggregateAll(values) {
+  return { passed: values.length > 0 && values.every((v) => v.passed) };
+}
+function aggregateTokenCountSum(values) {
+  const initial = {
+    input: 0,
+    output: 0,
+    inputCached: 0,
+    outputCached: 0
+  };
+  return values.reduce(
+    (acc, v) => ({
+      input: acc.input + (v.input ?? 0),
+      output: acc.output + (v.output ?? 0),
+      inputCached: acc.inputCached + (v.inputCached ?? 0),
+      outputCached: acc.outputCached + (v.outputCached ?? 0)
+    }),
+    initial
+  );
+}
+function aggregateLatencyAverage(values) {
+  if (values.length === 0) {
+    return { ms: 0 };
+  }
+  const sum = values.reduce((s, v) => s + v.ms, 0);
+  return { ms: sum / values.length };
+}
+
 // src/evals/metrics/standard.ts
 Metric.of({
   id: "token-count",
   name: "Tokens",
-  format: (data) => {
+  aggregate: aggregateTokenCountSum,
+  format: (data, options) => {
     const input = data.input ?? 0;
     const output = data.output ?? 0;
     const inputCached = data.inputCached ?? 0;
     const outputCached = data.outputCached ?? 0;
     const cached = inputCached + outputCached;
-    return `in:${input} out:${output} cached:${cached}`;
+    const base = `in:${input} out:${output} cached:${cached}`;
+    return options?.isAggregated ? `Total: ${base}` : base;
   }
 });
 Metric.of({
   id: "latency",
   name: "Latency",
-  format: (data) => `${data.ms}ms`
+  aggregate: aggregateLatencyAverage,
+  format: (data, options) => options?.isAggregated ? `Avg: ${data.ms}ms` : `${data.ms}ms`
 });
 
 // src/evals/scores/standard.ts
@@ -374,16 +419,36 @@ Score.of({
   id: "percent",
   name: "Score",
   displayStrategy: "bar",
-  format: (data) => data.value.toFixed(2)
+  format: (data, options) => options?.isAggregated ? `Avg: ${data.value.toFixed(2)}` : data.value.toFixed(2),
+  aggregate: aggregateAverage
 });
 Score.of({
   id: "binary",
   name: "Result",
   displayStrategy: "passFail",
-  format: (data) => data.passed ? "PASSED" : "NOT PASSED"
+  format: (data, options) => options?.isAggregated ? data.passed ? "All: PASSED" : "Some: FAILED" : data.passed ? "PASSED" : "NOT PASSED",
+  aggregate: aggregateAll
 });
 
 // src/runner/score-utils.ts
+function aggregateScoreItems(items) {
+  if (items.length === 0)
+    return void 0;
+  const def = getScoreById(items[0].id);
+  if (!def?.aggregate)
+    return items[items.length - 1];
+  const aggregated = def.aggregate(items.map((i) => i.data));
+  return { ...items[0], data: aggregated };
+}
+function aggregateMetricItems(items) {
+  if (items.length === 0)
+    return void 0;
+  const def = getMetricById(items[0].id);
+  if (!def?.aggregate)
+    return items[items.length - 1];
+  const aggregated = def.aggregate(items.map((i) => i.data));
+  return { ...items[0], data: aggregated };
+}
 function toNumericScoreFromScores(scores) {
   for (const item of scores) {
     const def = getScoreById(item.id);
@@ -462,6 +527,105 @@ function createArtifactPath(artifactDirectory, datasetId, runId) {
     `${datasetId}_${runId}_${nowIsoForFile()}.jsonl`
   );
 }
+function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent, persistenceQueue, updateSnapshot, completedRef, passedRef, failedRef) {
+  return effect.Effect.gen(function* () {
+    const reruns = typeof testCaseItem.testCase.getReruns === "function" ? testCaseItem.testCase.getReruns() : 1;
+    const rerunPassed = [];
+    for (let r = 0; r < reruns; r++) {
+      const started = Date.now();
+      const evaluatorScores = [];
+      let testCaseError;
+      const output = readOutput(testCaseItem.testCase);
+      for (const { id: evaluatorId, evaluator } of task.evaluators) {
+        const evaluateFn = evaluator.getEvaluateFn();
+        if (!evaluateFn) {
+          continue;
+        }
+        try {
+          const logs = [];
+          const logDiff = (expected, actual, options) => {
+            logs.push(createDiffLogEntry(expected, actual, options));
+          };
+          const ctx = yield* effect.Effect.promise(
+            () => Promise.resolve(evaluator.resolveContext())
+          );
+          const result = yield* effect.Effect.promise(
+            () => Promise.resolve(
+              evaluateFn({
+                input: testCaseItem.testCase.getInput(),
+                ctx,
+                output,
+                logDiff
+              })
+            )
+          );
+          const { scores, metrics } = normalizeResult(result);
+          const passed2 = computeEvaluatorPassed(evaluator, result, scores);
+          evaluatorScores.push({
+            evaluatorId,
+            scores,
+            passed: passed2,
+            metrics,
+            logs: logs.length > 0 ? logs : void 0
+          });
+        } catch (error) {
+          testCaseError = error instanceof Error ? error.message : "Evaluator execution failed";
+          evaluatorScores.push({
+            evaluatorId,
+            scores: [],
+            passed: false
+          });
+        }
+      }
+      const rerunPassedThis = evaluatorScores.every((s) => s.passed);
+      rerunPassed.push(rerunPassedThis);
+      const completedEvaluations = yield* effect.Ref.modify(completedRef, (n) => [
+        n + 1,
+        n + 1
+      ]);
+      const progressEvent = {
+        type: "TestCaseProgress",
+        runId: task.runId,
+        testCaseId: testCaseItem.id,
+        testCaseName: testCaseItem.testCase.getName(),
+        completedTestCases: completedEvaluations,
+        totalTestCases: totalEvaluations,
+        rerunIndex: r + 1,
+        rerunTotal: reruns,
+        passed: rerunPassedThis,
+        durationMs: Date.now() - started,
+        evaluatorScores,
+        output,
+        errorMessage: testCaseError
+      };
+      updateSnapshot(task.runId, (snapshot) => ({
+        ...snapshot,
+        completedTestCases: completedEvaluations
+      }));
+      yield* publishEvent(progressEvent);
+      yield* effect.Queue.offer(persistenceQueue, {
+        runId: task.runId,
+        artifactPath: task.snapshot.artifactPath,
+        payload: progressEvent
+      });
+    }
+    const testCasePassed = rerunPassed.every(Boolean);
+    if (testCasePassed) {
+      yield* effect.Ref.update(passedRef, (n) => n + 1);
+    } else {
+      yield* effect.Ref.update(failedRef, (n) => n + 1);
+    }
+    const [passed, failed] = yield* effect.Effect.all([
+      effect.Ref.get(passedRef),
+      effect.Ref.get(failedRef)
+    ]);
+    updateSnapshot(task.runId, (snapshot) => ({
+      ...snapshot,
+      passedTestCases: passed,
+      failedTestCases: failed
+    }));
+  });
+}
 var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => effect.Effect.gen(function* () {
   const startedAt = Date.now();
   updateSnapshot(task.runId, (snapshot) => ({
@@ -474,104 +638,51 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => e
     runId: task.runId,
     startedAt
   });
-  let completedTestCases = 0;
-  let passedTestCases = 0;
-  let failedTestCases = 0;
-  for (const testCaseItem of task.testCases) {
-    const started = Date.now();
-    const evaluatorScores = [];
-    let testCaseError;
-    const output = readOutput(testCaseItem.testCase);
-    for (const { id: evaluatorId, evaluator } of task.evaluators) {
-      const evaluateFn = evaluator.getEvaluateFn();
-      if (!evaluateFn) {
-        continue;
-      }
-      try {
-        const logs = [];
-        const logDiff = (expected, actual, options) => {
-          logs.push(createDiffLogEntry(expected, actual, options));
-        };
-        const ctx = yield* effect.Effect.promise(
-          () => Promise.resolve(evaluator.resolveContext())
-        );
-        const result = yield* effect.Effect.promise(
-          () => Promise.resolve(
-            evaluateFn({
-              input: testCaseItem.testCase.getInput(),
-              ctx,
-              output,
-              logDiff
-            })
-          )
-        );
-        const { scores, metrics } = normalizeResult(result);
-        const passed = computeEvaluatorPassed(evaluator, result, scores);
-        evaluatorScores.push({
-          evaluatorId,
-          scores,
-          passed,
-          metrics,
-          logs: logs.length > 0 ? logs : void 0
-        });
-      } catch (error) {
-        testCaseError = error instanceof Error ? error.message : "Evaluator execution failed";
-        evaluatorScores.push({
-          evaluatorId,
-          scores: [],
-          passed: false
-        });
-      }
-    }
-    const testCasePassed = evaluatorScores.every((s) => s.passed);
-    completedTestCases += 1;
-    if (testCasePassed) {
-      passedTestCases += 1;
-    } else {
-      failedTestCases += 1;
-    }
-    const progressEvent = {
-      type: "TestCaseProgress",
-      runId: task.runId,
-      testCaseId: testCaseItem.id,
-      testCaseName: testCaseItem.testCase.getName(),
-      completedTestCases,
-      totalTestCases: task.testCases.length,
-      passed: testCasePassed,
-      durationMs: Date.now() - started,
-      evaluatorScores,
-      output,
-      errorMessage: testCaseError
-    };
-    updateSnapshot(task.runId, (snapshot) => ({
-      ...snapshot,
-      completedTestCases,
-      passedTestCases,
-      failedTestCases
-    }));
-    yield* publishEvent(progressEvent);
-    yield* effect.Queue.offer(persistenceQueue, {
-      runId: task.runId,
-      artifactPath: task.snapshot.artifactPath,
-      payload: progressEvent
-    });
-  }
+  const totalEvaluations = task.testCases.reduce(
+    (sum, tc) => sum + (typeof tc.testCase.getReruns === "function" ? tc.testCase.getReruns() : 1),
+    0
+  );
+  const maxConcurrency = Math.max(1, task.maxConcurrency ?? 1);
+  const completedRef = yield* effect.Ref.make(0);
+  const passedRef = yield* effect.Ref.make(0);
+  const failedRef = yield* effect.Ref.make(0);
+  const processTestCase = (testCaseItem) => processOneTestCase(
+    task,
+    testCaseItem,
+    totalEvaluations,
+    publishEvent,
+    persistenceQueue,
+    updateSnapshot,
+    completedRef,
+    passedRef,
+    failedRef
+  );
+  yield* effect.Effect.forEach(
+    task.testCases,
+    processTestCase,
+    maxConcurrency > 1 ? { concurrency: maxConcurrency } : void 0
+  );
+  const [completedEvaluations, passedUniqueTestCases, failedUniqueTestCases] = yield* effect.Effect.all([
+    effect.Ref.get(completedRef),
+    effect.Ref.get(passedRef),
+    effect.Ref.get(failedRef)
+  ]);
   const finishedAt = Date.now();
   const completedEvent = {
     type: "RunCompleted",
     runId: task.runId,
     finishedAt,
-    passedTestCases,
-    failedTestCases,
+    passedTestCases: passedUniqueTestCases,
+    failedTestCases: failedUniqueTestCases,
     totalTestCases: task.testCases.length,
     artifactPath: task.snapshot.artifactPath
   };
   updateSnapshot(task.runId, (snapshot) => ({
     ...snapshot,
     status: "completed",
-    completedTestCases,
-    passedTestCases,
-    failedTestCases,
+    completedTestCases: completedEvaluations,
+    passedTestCases: passedUniqueTestCases,
+    failedTestCases: failedUniqueTestCases,
     finishedAt
   }));
   yield* publishEvent(completedEvent);
@@ -586,6 +697,126 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => e
     artifactPath: task.snapshot.artifactPath
   });
 });
+async function loadRunSnapshotsFromArtifacts(config) {
+  const baseDir = path.resolve(config.artifactDirectory);
+  let entries;
+  try {
+    entries = await promises.readdir(baseDir);
+  } catch {
+    return [];
+  }
+  const jsonlFiles = entries.filter((name) => name.endsWith(".jsonl"));
+  const snapshots = [];
+  for (const fileName of jsonlFiles) {
+    const filePath = path.join(baseDir, fileName);
+    try {
+      const snapshot = await parseArtifactToSnapshot(filePath, config);
+      if (snapshot) {
+        snapshots.push(snapshot);
+      }
+    } catch {
+    }
+  }
+  return snapshots.sort((a, b) => b.queuedAt - a.queuedAt);
+}
+async function parseArtifactToSnapshot(filePath, _config) {
+  const content = await promises.readFile(filePath, "utf8");
+  const lines = content.split("\n").filter((line) => line.trim().length > 0);
+  if (lines.length === 0) {
+    return null;
+  }
+  let runQueued = null;
+  let runCompleted = null;
+  let runFailed = null;
+  let runStarted = null;
+  for (const line of lines) {
+    try {
+      const event = JSON.parse(line);
+      const type = event.type;
+      if (type === "RunQueued") {
+        runQueued = {
+          runId: event.runId,
+          datasetId: event.datasetId,
+          datasetName: event.datasetName,
+          evaluatorIds: event.evaluatorIds,
+          totalTestCases: event.totalTestCases ?? 0,
+          artifactPath: event.artifactPath ?? filePath,
+          ts: event.ts
+        };
+      }
+      if (type === "RunStarted") {
+        runStarted = { startedAt: event.startedAt };
+      }
+      if (type === "RunCompleted") {
+        runCompleted = {
+          passedTestCases: event.passedTestCases,
+          failedTestCases: event.failedTestCases,
+          totalTestCases: event.totalTestCases,
+          finishedAt: event.finishedAt
+        };
+      }
+      if (type === "RunFailed") {
+        runFailed = {
+          finishedAt: event.finishedAt,
+          errorMessage: event.errorMessage
+        };
+      }
+    } catch {
+    }
+  }
+  if (!runQueued) {
+    return null;
+  }
+  const artifactPath = filePath;
+  const status = runFailed ? "failed" : runCompleted ? "completed" : runStarted ? "running" : "queued";
+  const progress = aggregateTestCaseProgress(lines);
+  const completedTestCases = runCompleted ? runQueued.totalTestCases : progress.completedTestCases;
+  const passedTestCases = runCompleted?.passedTestCases ?? progress.passedTestCases;
+  const failedTestCases = runCompleted?.failedTestCases ?? progress.failedTestCases;
+  return {
+    runId: runQueued.runId,
+    datasetId: runQueued.datasetId,
+    datasetName: runQueued.datasetName,
+    evaluatorIds: runQueued.evaluatorIds,
+    queuedAt: runQueued.ts ?? 0,
+    startedAt: runStarted?.startedAt,
+    finishedAt: runCompleted?.finishedAt ?? runFailed?.finishedAt,
+    totalTestCases: runQueued.totalTestCases,
+    completedTestCases,
+    passedTestCases,
+    failedTestCases,
+    status,
+    artifactPath,
+    errorMessage: runFailed?.errorMessage
+  };
+}
+function aggregateTestCaseProgress(lines) {
+  let completedTestCases = 0;
+  const testCasePassedBy = /* @__PURE__ */ new Map();
+  for (const line of lines) {
+    try {
+      const event = JSON.parse(line);
+      if (event.type === "TestCaseProgress") {
+        const ev = event;
+        completedTestCases = ev.completedTestCases ?? completedTestCases;
+        const id = ev.testCaseId;
+        const current = testCasePassedBy.get(id);
+        testCasePassedBy.set(id, current === void 0 ? ev.passed : current && ev.passed);
+      }
+    } catch {
+    }
+  }
+  let passedTestCases = 0;
+  let failedTestCases = 0;
+  for (const passed of testCasePassedBy.values()) {
+    if (passed) {
+      passedTestCases += 1;
+    } else {
+      failedTestCases += 1;
+    }
+  }
+  return { completedTestCases, passedTestCases, failedTestCases };
+}
 async function appendJsonLine(artifactPath, payload) {
   await promises.mkdir(path.dirname(artifactPath), { recursive: true });
   await promises.appendFile(artifactPath, `${JSON.stringify(payload)}
@@ -778,6 +1009,10 @@ var EffectRunner = class {
       throw new Error("No evaluators selected for run");
     }
     const selectedTestCases = await this.collectDatasetTestCases(request.datasetId);
+    const totalEvaluations = selectedTestCases.reduce(
+      (sum, tc) => sum + (typeof tc.testCase.getReruns === "function" ? tc.testCase.getReruns() : 1),
+      0
+    );
     const runId = `run-${crypto.randomUUID()}`;
     const artifactPath = createArtifactPath(
       this.config.artifactDirectory,
@@ -790,7 +1025,7 @@ var EffectRunner = class {
       datasetName: dataset.dataset.getName(),
       evaluatorIds: selectedEvaluators.map((item) => item.id),
       queuedAt: Date.now(),
-      totalTestCases: selectedTestCases.length,
+      totalTestCases: totalEvaluations,
       completedTestCases: 0,
       passedTestCases: 0,
       failedTestCases: 0,
@@ -804,7 +1039,7 @@ var EffectRunner = class {
       datasetId: request.datasetId,
       datasetName: dataset.dataset.getName(),
       evaluatorIds: selectedEvaluators.map((item) => item.id),
-      totalTestCases: selectedTestCases.length,
+      totalTestCases: totalEvaluations,
       artifactPath
     };
     await effect.Effect.runPromise(this.publishEvent(queuedEvent));
@@ -815,6 +1050,7 @@ var EffectRunner = class {
         payload: queuedEvent
       })
     );
+    const maxConcurrency = request.concurrency ?? this.config.maxConcurrency ?? 1;
     await effect.Effect.runPromise(
       effect.Queue.offer(this.runQueue, {
         runId,
@@ -822,7 +1058,8 @@ var EffectRunner = class {
         dataset: dataset.dataset,
         evaluators: selectedEvaluators,
         testCases: selectedTestCases,
-        snapshot
+        snapshot,
+        maxConcurrency
       })
     );
     return snapshot;
@@ -842,6 +1079,9 @@ var EffectRunner = class {
       (a, b) => b.queuedAt - a.queuedAt
     );
   }
+  async loadRunSnapshotsFromArtifacts() {
+    return loadRunSnapshotsFromArtifacts(this.config);
+  }
   async shutdown() {
     await effect.Effect.runPromise(effect.Fiber.interrupt(this.schedulerFiber));
     await effect.Effect.runPromise(effect.Fiber.interrupt(this.persistenceFiber));
@@ -973,7 +1213,7 @@ function GenerateView({
         return;
       }
       const { writeFile: writeFile2 } = await import('fs/promises');
-      const { join: join3, parse: parse2, resolve: resolve4 } = await import('path');
+      const { join: join4, parse: parse2, resolve: resolve5 } = await import('path');
       const testCases = await runner.collectDatasetTestCases(dataset.id);
       const payload = testCases.map((item) => {
         const tc = item.testCase;
@@ -983,9 +1223,9 @@ function GenerateView({
           output: typeof tc.getOutput === "function" ? tc.getOutput() : void 0
         };
       });
-      const absoluteDatasetPath = resolve4(process.cwd(), dataset.filePath);
+      const absoluteDatasetPath = resolve5(process.cwd(), dataset.filePath);
       const parsed = parse2(absoluteDatasetPath);
-      const outputPath = join3(parsed.dir, `${parsed.name}.cases.json`);
+      const outputPath = join4(parsed.dir, `${parsed.name}.cases.json`);
       await writeFile2(
         outputPath,
         `${JSON.stringify(payload, null, 2)}
@@ -1060,7 +1300,7 @@ async function generateDatasetJsonCommandPlain(runner, datasetName) {
   console.log(`Wrote ${outputPath}`);
 }
 async function generateDatasetJsonCommandInk(runner, datasetName) {
-  return new Promise((resolve4, reject) => {
+  return new Promise((resolve5, reject) => {
     const app = ink.render(
       React2__default.default.createElement(GenerateView, {
         runner,
@@ -1070,7 +1310,7 @@ async function generateDatasetJsonCommandInk(runner, datasetName) {
           if (err) {
             reject(err);
           } else {
-            resolve4();
+            resolve5();
           }
         }
       })
@@ -1137,13 +1377,62 @@ function createBar(value, max = 100, width = 20) {
   const filled = Math.round(safe / max * width);
   return "\u2588".repeat(filled) + "\u2591".repeat(width - filled);
 }
-function formatScorePart(item, scoreToColor2) {
+function aggregateEvaluatorScores(events, nameById) {
+  if (events.length === 0)
+    return [];
+  const evaluatorIds = new Set(
+    events.flatMap((e) => e.evaluatorScores.map((x) => x.evaluatorId))
+  );
+  const result = [];
+  for (const evaluatorId of evaluatorIds) {
+    const scoreIdToItems = /* @__PURE__ */ new Map();
+    const metricIdToItems = /* @__PURE__ */ new Map();
+    for (const ev of events) {
+      const es = ev.evaluatorScores.find((x) => x.evaluatorId === evaluatorId);
+      for (const s of es?.scores ?? []) {
+        const list = scoreIdToItems.get(s.id) ?? [];
+        list.push(s);
+        scoreIdToItems.set(s.id, list);
+      }
+      for (const m of es?.metrics ?? []) {
+        const list = metricIdToItems.get(m.id) ?? [];
+        list.push(m);
+        metricIdToItems.set(m.id, list);
+      }
+    }
+    const aggregatedScores = [];
+    for (const items of scoreIdToItems.values()) {
+      const agg = aggregateScoreItems(items);
+      if (agg)
+        aggregatedScores.push(agg);
+    }
+    const aggregatedMetrics = Array.from(metricIdToItems.entries()).map(([id, items]) => aggregateMetricItems(items)).filter((m) => m !== void 0);
+    const passed = events.every((ev) => {
+      const es = ev.evaluatorScores.find((x) => x.evaluatorId === evaluatorId);
+      return es?.passed ?? false;
+    });
+    const lastEvent = events[events.length - 1];
+    const lastEs = lastEvent?.evaluatorScores.find(
+      (x) => x.evaluatorId === evaluatorId
+    );
+    result.push({
+      evaluatorId,
+      evaluatorName: nameById.get(evaluatorId) ?? evaluatorId,
+      scores: aggregatedScores,
+      passed,
+      metrics: aggregatedMetrics.length > 0 ? aggregatedMetrics : void 0,
+      logs: lastEs?.logs
+    });
+  }
+  return result;
+}
+function formatScorePart(item, scoreToColor2, options) {
   const def = getScoreById(item.id);
   if (!def) {
     const numeric = toNumericScore(item.data);
     return numeric !== void 0 ? `${numeric.toFixed(2)}` : "n/a";
   }
-  const formatted = def.format(item.data);
+  const formatted = def.format(item.data, options);
   if (def.displayStrategy === "bar") {
     const numeric = typeof item.data === "object" && item.data !== null && "value" in item.data ? item.data.value : toNumericScore(item.data);
     if (typeof numeric === "number" && Number.isFinite(numeric)) {
@@ -1163,6 +1452,7 @@ function RunView({
   );
   const [runInfo, setRunInfo] = React2.useState(null);
   const [testCases, setTestCases] = React2.useState([]);
+  const [completedEvaluations, setCompletedEvaluations] = React2.useState(0);
   const [summary, setSummary] = React2.useState(null);
   const [evaluatorNameById, setEvaluatorNameById] = React2.useState(/* @__PURE__ */ new Map());
   const runEval = React2.useCallback(async () => {
@@ -1189,20 +1479,17 @@ function RunView({
       return;
     }
     const nameById = new Map(
-      evaluators.map((item) => [
-        item.id,
-        item.evaluator.getName() ?? item.id
-      ])
+      evaluators.map((item) => [item.id, item.evaluator.getName() ?? item.id])
     );
     setEvaluatorNameById(nameById);
     const aggregates = /* @__PURE__ */ new Map();
     let overallScoreTotal = 0;
     let overallScoreCount = 0;
-    const done = new Promise((resolve4) => {
+    const done = new Promise((resolve5) => {
       const unsubscribe = runner.subscribeRunEvents((event) => {
         if (event.type === "TestCaseProgress") {
           const numericScores = event.evaluatorScores.map((item) => toNumericScoreFromScores(item.scores)).filter((item) => item !== void 0);
-          const averageScore = numericScores.length > 0 ? numericScores.reduce((sum, v) => sum + v, 0) / numericScores.length : void 0;
+          numericScores.length > 0 ? numericScores.reduce((sum, v) => sum + v, 0) / numericScores.length : void 0;
           for (const item of event.evaluatorScores) {
             const numeric = toNumericScoreFromScores(item.scores);
             if (numeric !== void 0) {
@@ -1222,15 +1509,10 @@ function RunView({
               overallScoreCount += 1;
             }
           }
-          setTestCases((prev) => [
-            ...prev,
-            {
-              name: event.testCaseName,
-              completedTestCases: event.completedTestCases,
-              totalTestCases: event.totalTestCases,
-              durationMs: event.durationMs,
-              passed: event.passed,
-              averageScore,
+          setTestCases((prev) => {
+            const byId = new Map(prev.map((tc) => [tc.testCaseId, tc]));
+            const existing = byId.get(event.testCaseId);
+            const newEvent = {
               evaluatorScores: event.evaluatorScores.map((item) => ({
                 evaluatorId: item.evaluatorId,
                 evaluatorName: nameById.get(item.evaluatorId) ?? item.evaluatorId,
@@ -1238,13 +1520,37 @@ function RunView({
                 passed: item.passed,
                 metrics: item.metrics,
                 logs: item.logs
-              }))
-            }
-          ]);
+              })),
+              passed: event.passed,
+              durationMs: event.durationMs
+            };
+            const events = existing ? [...existing.events, newEvent] : [newEvent];
+            const isAggregated = events.length > 1;
+            const aggregatedEvaluatorScores = aggregateEvaluatorScores(
+              events,
+              nameById
+            );
+            const merged = {
+              name: event.testCaseName,
+              testCaseId: event.testCaseId,
+              completedTestCases: event.completedTestCases,
+              totalTestCases: event.totalTestCases,
+              rerunIndex: event.rerunIndex,
+              rerunTotal: event.rerunTotal,
+              durationMs: events.reduce((s, e) => s + e.durationMs, 0),
+              passed: events.every((e) => e.passed),
+              events,
+              aggregatedEvaluatorScores,
+              isAggregated
+            };
+            byId.set(event.testCaseId, merged);
+            setCompletedEvaluations(event.completedTestCases);
+            return Array.from(byId.values());
+          });
         }
         if (event.type === "RunCompleted" || event.type === "RunFailed") {
           unsubscribe();
-          resolve4(event);
+          resolve5(event);
         }
       });
     });
@@ -1255,9 +1561,7 @@ function RunView({
     setRunInfo({
       runId: snapshot.runId,
       datasetName: snapshot.datasetName,
-      evaluatorNames: evaluators.map(
-        (e) => e.evaluator.getName() ?? e.id
-      ),
+      evaluatorNames: evaluators.map((e) => e.evaluator.getName() ?? e.id),
       totalTestCases: snapshot.totalTestCases
     });
     setPhase("running");
@@ -1285,29 +1589,41 @@ function RunView({
     /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { marginBottom: 1, children: /* @__PURE__ */ jsxRuntime.jsx(Banner, {}) }),
     runInfo && /* @__PURE__ */ jsxRuntime.jsxs(ink.Box, { flexDirection: "column", marginBottom: 1, children: [
       /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { children: [
-        /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "cyan", bold: true, children: "Run " }),
+        /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "cyan", bold: true, children: [
+          "Run",
+          " "
+        ] }),
         /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "gray", children: runInfo.runId })
       ] }),
       /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { children: [
-        /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "cyan", bold: true, children: "Dataset " }),
+        /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "cyan", bold: true, children: [
+          "Dataset",
+          " "
+        ] }),
         runInfo.datasetName
       ] }),
       /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { children: [
-        /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "cyan", bold: true, children: "Evaluators " }),
+        /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "cyan", bold: true, children: [
+          "Evaluators",
+          " "
+        ] }),
         runInfo.evaluatorNames.join(", ")
       ] }),
       /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { children: [
-        /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "cyan", bold: true, children: "Test cases " }),
+        /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "cyan", bold: true, children: [
+          "Test cases",
+          " "
+        ] }),
         runInfo.totalTestCases
       ] })
     ] }),
     phase === "running" && /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { marginBottom: 1, children: /* @__PURE__ */ jsxRuntime.jsx(
       Spinner,
       {
-        label: `Evaluations ${testCases.length}/${runInfo?.totalTestCases ?? 0}`
+        label: `Evaluations ${completedEvaluations}/${runInfo?.totalTestCases ?? 0}`
       }
     ) }),
-    testCases.length > 0 && /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { flexDirection: "column", marginBottom: 1, children: testCases.map((tc, i) => /* @__PURE__ */ jsxRuntime.jsxs(ink.Box, { flexDirection: "column", marginBottom: 0, children: [
+    testCases.length > 0 && /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { flexDirection: "column", marginBottom: 1, children: testCases.map((tc) => /* @__PURE__ */ jsxRuntime.jsxs(ink.Box, { flexDirection: "column", marginBottom: 0, children: [
       /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { children: [
         /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "cyan", children: [
           "[",
@@ -1318,49 +1634,78 @@ function RunView({
         ] }),
         " ",
         tc.name,
+        " ",
+        /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "cyan", children: [
+          "(",
+          tc.rerunIndex,
+          "/",
+          tc.rerunTotal,
+          ")"
+        ] }),
         /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
           " (",
           tc.durationMs,
           "ms)"
         ] })
       ] }),
-      tc.evaluatorScores.map((item) => /* @__PURE__ */ jsxRuntime.jsxs(ink.Box, { flexDirection: "column", marginLeft: 2, children: [
-        /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { children: [
-          item.evaluatorName,
-          ":",
-          " ",
-          /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: item.passed ? "green" : "red", bold: true, children: item.passed ? "PASS" : "FAIL" }),
-          " ",
-          item.scores.map((s) => /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: scoreColor(toNumericScore(s.data) ?? 0), children: [
-            formatScorePart(s),
-            " "
-          ] }, s.id)),
-          item.metrics?.map((m) => {
-            const def = getMetricById(m.id);
-            if (!def)
-              return null;
-            const formatted = def.format(m.data);
-            return /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
-              "[",
-              def.name ? `${def.name}: ` : "",
-              formatted,
-              "]",
-              " "
-            ] }, m.id);
-          })
-        ] }),
-        !item.passed && item.logs && item.logs.length > 0 && /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { marginLeft: 2, flexDirection: "column", children: item.logs.map(
-          (log, logIdx) => log.type === "diff" ? /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { flexDirection: "column", children: getDiffLines(log).map(({ type, line }, lineIdx) => /* @__PURE__ */ jsxRuntime.jsx(
-            ink.Text,
-            {
-              color: type === "remove" ? "red" : type === "add" ? "green" : "gray",
-              children: line
-            },
-            lineIdx
-          )) }, logIdx) : null
-        ) })
-      ] }, item.evaluatorId))
-    ] }, i)) }),
+      tc.aggregatedEvaluatorScores.map((item) => /* @__PURE__ */ jsxRuntime.jsxs(
+        ink.Box,
+        {
+          flexDirection: "column",
+          marginLeft: 2,
+          children: [
+            /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { children: [
+              item.evaluatorName,
+              ":",
+              " ",
+              /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: item.passed ? "green" : "red", bold: true, children: item.passed ? "PASS" : "FAIL" }),
+              " ",
+              item.scores.map((s) => /* @__PURE__ */ jsxRuntime.jsxs(
+                ink.Text,
+                {
+                  color: scoreColor(toNumericScore(s.data) ?? 0),
+                  children: [
+                    formatScorePart(s, scoreColor, {
+                      isAggregated: tc.isAggregated
+                    }),
+                    " "
+                  ]
+                },
+                s.id
+              )),
+              item.metrics?.map((m) => {
+                const def = getMetricById(m.id);
+                if (!def)
+                  return null;
+                const formatted = def.format(m.data, {
+                  isAggregated: tc.isAggregated
+                });
+                return /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
+                  "[",
+                  def.name ? `${def.name}: ` : "",
+                  formatted,
+                  "]",
+                  " "
+                ] }, m.id);
+              })
+            ] }),
+            !item.passed && item.logs && item.logs.length > 0 && /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { marginLeft: 2, flexDirection: "column", children: item.logs.map(
+              (log, logIdx) => log.type === "diff" ? /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { flexDirection: "column", children: getDiffLines(log).map(
+                ({ type, line }, lineIdx) => /* @__PURE__ */ jsxRuntime.jsx(
+                  ink.Text,
+                  {
+                    color: type === "remove" ? "red" : type === "add" ? "green" : "gray",
+                    children: line
+                  },
+                  lineIdx
+                )
+              ) }, logIdx) : null
+            ) })
+          ]
+        },
+        item.evaluatorId
+      ))
+    ] }, tc.testCaseId)) }),
     phase === "completed" && summary && /* @__PURE__ */ jsxRuntime.jsxs(ink.Box, { flexDirection: "column", children: [
       /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "cyan", bold: true, children: "Run Summary" }),
       /* @__PURE__ */ jsxRuntime.jsxs(ink.Box, { marginTop: 1, children: [
@@ -1407,7 +1752,8 @@ function RunView({
             name.padEnd(28),
             " avg=",
             /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: scoreColor(mean), children: mean.toFixed(2) }),
-            " passed=",
+            " ",
+            "passed=",
             agg.passed,
             " failed=",
             agg.failed
@@ -1416,28 +1762,38 @@ function RunView({
       ] }),
       /* @__PURE__ */ jsxRuntime.jsxs(ink.Box, { marginTop: 1, flexDirection: "column", children: [
         /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "magenta", children: "test case scores" }),
-        testCases.map((tc, i) => /* @__PURE__ */ jsxRuntime.jsxs(ink.Box, { children: [
-          /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: tc.passed ? "green" : "red", children: tc.passed ? "PASS" : "FAIL" }),
-          /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { children: [
-            " ",
-            tc.name.padEnd(24)
-          ] }),
-          tc.averageScore !== void 0 ? /* @__PURE__ */ jsxRuntime.jsxs(jsxRuntime.Fragment, { children: [
-            /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: scoreColor(tc.averageScore), children: [
-              "score=",
-              tc.averageScore.toFixed(2)
+        testCases.map((tc) => {
+          const numericScores = tc.aggregatedEvaluatorScores.flatMap(
+            (item) => item.scores.map((s) => toNumericScoreFromScores([s])).filter((n) => n !== void 0)
+          );
+          const averageScore = numericScores.length > 0 ? numericScores.reduce((a, b) => a + b, 0) / numericScores.length : void 0;
+          const firstScore = tc.aggregatedEvaluatorScores[0]?.scores[0];
+          const scoreLabel = firstScore && tc.isAggregated ? formatScorePart(firstScore, scoreColor, {
+            isAggregated: true
+          }) : averageScore !== void 0 ? averageScore.toFixed(2) : "n/a";
+          return /* @__PURE__ */ jsxRuntime.jsxs(ink.Box, { children: [
+            /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: tc.passed ? "green" : "red", children: tc.passed ? "PASS" : "FAIL" }),
+            /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { children: [
+              " ",
+              tc.name.padEnd(24)
             ] }),
+            averageScore !== void 0 ? /* @__PURE__ */ jsxRuntime.jsxs(jsxRuntime.Fragment, { children: [
+              /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: scoreColor(averageScore), children: [
+                "score=",
+                scoreLabel
+              ] }),
+              /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
+                " ",
+                createBar(averageScore, 100, 14)
+              ] })
+            ] }) : /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "gray", children: "score=n/a" }),
             /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
-              " ",
-              createBar(tc.averageScore, 100, 14)
+              " (",
+              tc.durationMs,
+              "ms)"
             ] })
-          ] }) : /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "gray", children: "score=n/a" }),
-          /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
-            " (",
-            tc.durationMs,
-            "ms)"
-          ] })
-        ] }, i))
+          ] }, tc.testCaseId);
+        })
       ] }),
       /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { marginTop: 1, children: /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
         "artifact: ",
@@ -1448,6 +1804,51 @@ function RunView({
 }
 
 // src/cli-simple/run.ts
+function buildTestCaseSummaries(byId) {
+  const summaries = [];
+  for (const { name, events } of byId.values()) {
+    const passed = events.every((e) => e.passed);
+    const durationMs = events.reduce((sum, e) => sum + e.durationMs, 0);
+    const isAggregated = events.length > 1;
+    const numericScores = [];
+    let firstAggregatedScore;
+    for (const evaluatorScores of events[0]?.evaluatorScores ?? []) {
+      const scoreIdToItems = /* @__PURE__ */ new Map();
+      for (const ev of events) {
+        const es = ev.evaluatorScores.find(
+          (x) => x.evaluatorId === evaluatorScores.evaluatorId
+        );
+        for (const s of es?.scores ?? []) {
+          const list = scoreIdToItems.get(s.id) ?? [];
+          list.push(s);
+          scoreIdToItems.set(s.id, list);
+        }
+      }
+      for (const items of scoreIdToItems.values()) {
+        const agg = aggregateScoreItems(items);
+        if (agg) {
+          const n = toNumericScoreFromScores([agg]);
+          if (n !== void 0) {
+            numericScores.push(n);
+            if (firstAggregatedScore === void 0) {
+              firstAggregatedScore = agg;
+            }
+          }
+        }
+      }
+    }
+    const averageScore = numericScores.length > 0 ? numericScores.reduce((a, b) => a + b, 0) / numericScores.length : void 0;
+    summaries.push({
+      name,
+      averageScore,
+      aggregatedScoreItem: firstAggregatedScore,
+      isAggregated,
+      durationMs,
+      passed
+    });
+  }
+  return summaries;
+}
 var ansi2 = {
   reset: "\x1B[0m",
   bold: "\x1B[1m",
@@ -1482,7 +1883,50 @@ function createBar2(value, max = 100, width = 20) {
   const filled = Math.round(safe / max * width);
   return `${"\u2588".repeat(filled)}${"\u2591".repeat(width - filled)}`;
 }
-function formatEvaluatorScoreLine(name, scores, passed, metrics) {
+function aggregateEvaluatorScoresFromEvents(events, evaluatorNameById) {
+  if (events.length === 0)
+    return [];
+  const evaluatorIds = new Set(
+    events.flatMap((e) => e.evaluatorScores.map((x) => x.evaluatorId))
+  );
+  const result = [];
+  for (const evaluatorId of evaluatorIds) {
+    const scoreIdToItems = /* @__PURE__ */ new Map();
+    const metricIdToItems = /* @__PURE__ */ new Map();
+    for (const ev of events) {
+      const es = ev.evaluatorScores.find((x) => x.evaluatorId === evaluatorId);
+      for (const s of es?.scores ?? []) {
+        const list = scoreIdToItems.get(s.id) ?? [];
+        list.push(s);
+        scoreIdToItems.set(s.id, list);
+      }
+      for (const m of es?.metrics ?? []) {
+        const list = metricIdToItems.get(m.id) ?? [];
+        list.push(m);
+        metricIdToItems.set(m.id, list);
+      }
+    }
+    const aggregatedScores = [];
+    for (const items of scoreIdToItems.values()) {
+      const agg = aggregateScoreItems(items);
+      if (agg)
+        aggregatedScores.push(agg);
+    }
+    const aggregatedMetrics = Array.from(metricIdToItems.entries()).map(([, items]) => aggregateMetricItems(items)).filter((m) => m !== void 0);
+    const passed = events.every((ev) => {
+      const es = ev.evaluatorScores.find((x) => x.evaluatorId === evaluatorId);
+      return es?.passed ?? false;
+    });
+    result.push({
+      evaluatorId,
+      scores: aggregatedScores,
+      passed,
+      metrics: aggregatedMetrics.length > 0 ? aggregatedMetrics : void 0
+    });
+  }
+  return result;
+}
+function formatEvaluatorScoreLine(name, scores, passed, metrics, options) {
   const passLabel = passed ? colorize("PASS", `${ansi2.bold}${ansi2.green}`) : colorize("FAIL", `${ansi2.bold}${ansi2.red}`);
   const scoreParts = [];
   for (const item of scores) {
@@ -1494,7 +1938,7 @@ function formatEvaluatorScoreLine(name, scores, passed, metrics) {
       );
       continue;
     }
-    const formatted = def.format(item.data);
+    const formatted = def.format(item.data, options);
     switch (def.displayStrategy) {
       case "bar": {
         const numeric = typeof item.data === "object" && item.data !== null && "value" in item.data ? item.data.value : toNumericScore(item.data);
@@ -1527,7 +1971,7 @@ function formatEvaluatorScoreLine(name, scores, passed, metrics) {
     for (const { id, data } of metrics) {
       const def = getMetricById(id);
       if (def) {
-        const formatted = def.format(data);
+        const formatted = def.format(data, options);
         metricParts.push(
           def.name ? `[${def.name}: ${formatted}]` : `[${formatted}]`
         );
@@ -1560,7 +2004,7 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
     evaluators.map((item) => [item.id, item.evaluator.getName() ?? item.id])
   );
   const aggregates = /* @__PURE__ */ new Map();
-  const testCaseSummaries = [];
+  const testCaseByTestId = /* @__PURE__ */ new Map();
   let overallScoreTotal = 0;
   let overallScoreCount = 0;
   let completedCount = 0;
@@ -1574,6 +2018,11 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
     }
     process.stdout.write("\r\x1B[2K");
   }
+  function cursorUp(n) {
+    if (!process.stdout.isTTY || n <= 0)
+      return;
+    process.stdout.write(`\x1B[${n}A`);
+  }
   function drawSpinner() {
     if (!process.stdout.isTTY || runFinished) {
       return;
@@ -1587,38 +2036,28 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
       )} ${colorize("(live)", ansi2.dim)}`
     );
   }
+  let lastPrintedTestCaseId = null;
+  let lastPrintedLineCount = 0;
   let spinnerTimer;
-  const done = new Promise((resolve4) => {
+  const done = new Promise((resolve5) => {
     const unsubscribe = runner.subscribeRunEvents((event) => {
       if (event.type === "TestCaseProgress") {
         completedCount = event.completedTestCases;
         const numericScores = event.evaluatorScores.map((item) => toNumericScoreFromScores(item.scores)).filter((item) => item !== void 0);
         const averageScore = numericScores.length > 0 ? numericScores.reduce((sum, value) => sum + value, 0) / numericScores.length : void 0;
-        clearLine();
-        console.log(
-          `${colorize(`[${event.completedTestCases}/${event.totalTestCases}]`, ansi2.cyan)} ${event.testCaseName} ${colorize(`(${event.durationMs}ms)`, ansi2.dim)}`
-        );
+        const testCaseId = event.testCaseId;
+        const existing = testCaseByTestId.get(testCaseId) ?? {
+          name: event.testCaseName,
+          events: []
+        };
+        existing.events.push({
+          averageScore,
+          passed: event.passed,
+          durationMs: event.durationMs,
+          evaluatorScores: event.evaluatorScores
+        });
+        testCaseByTestId.set(testCaseId, existing);
         for (const item of event.evaluatorScores) {
-          const name = evaluatorNameById.get(item.evaluatorId) ?? item.evaluatorId;
-          console.log(
-            formatEvaluatorScoreLine(
-              name,
-              item.scores,
-              item.passed,
-              item.metrics
-            )
-          );
-          if (!item.passed && item.logs && item.logs.length > 0) {
-            for (const log of item.logs) {
-              if (log.type === "diff") {
-                const useColor = process.stdout.isTTY;
-                for (const { type, line } of getDiffLines(log)) {
-                  const colored = useColor && type === "remove" ? colorize(`      ${line}`, ansi2.red) : useColor && type === "add" ? colorize(`      ${line}`, ansi2.green) : `      ${line}`;
-                  console.log(colored);
-                }
-              }
-            }
-          }
           const numeric = toNumericScoreFromScores(item.scores);
           if (numeric !== void 0) {
             const current = aggregates.get(item.evaluatorId) ?? {
@@ -1637,19 +2076,67 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
             overallScoreCount += 1;
           }
         }
-        testCaseSummaries.push({
-          name: event.testCaseName,
-          averageScore,
-          durationMs: event.durationMs,
-          passed: event.passed
-        });
+        const isSameTestCase = lastPrintedTestCaseId === testCaseId;
+        const isLastRerun = event.rerunIndex >= event.rerunTotal;
+        const isNonTty = !process.stdout.isTTY;
+        const skipPrintNonTty = isNonTty && event.rerunTotal > 1 && !isLastRerun;
+        if (isSameTestCase && lastPrintedLineCount > 0 && !skipPrintNonTty) {
+          cursorUp(lastPrintedLineCount);
+        }
+        const aggregatedScores = aggregateEvaluatorScoresFromEvents(
+          existing.events);
+        const isAggregated = existing.events.length > 1;
+        const durationMs = existing.events.reduce(
+          (s, e) => s + e.durationMs,
+          0
+        );
+        existing.events.every((e) => e.passed);
+        const lines = [];
+        lines.push(
+          `${colorize(`[${event.completedTestCases}/${event.totalTestCases}]`, ansi2.cyan)} ${event.testCaseName} ${colorize(`(${event.rerunIndex}/${event.rerunTotal})`, ansi2.cyan)} ${colorize(`(${durationMs}ms)`, ansi2.dim)}`
+        );
+        for (const item of aggregatedScores) {
+          const name = evaluatorNameById.get(item.evaluatorId) ?? item.evaluatorId;
+          lines.push(
+            formatEvaluatorScoreLine(
+              name,
+              item.scores,
+              item.passed,
+              item.metrics,
+              { isAggregated }
+            )
+          );
+          const lastEvent = existing.events[existing.events.length - 1];
+          const lastEs = lastEvent?.evaluatorScores.find(
+            (x) => x.evaluatorId === item.evaluatorId
+          );
+          if (!item.passed && lastEs?.logs && lastEs.logs.length > 0) {
+            for (const log of lastEs.logs) {
+              if (log.type === "diff") {
+                const useColor = process.stdout.isTTY;
+                for (const { type, line } of getDiffLines(log)) {
+                  const colored = useColor && type === "remove" ? colorize(`      ${line}`, ansi2.red) : useColor && type === "add" ? colorize(`      ${line}`, ansi2.green) : `      ${line}`;
+                  lines.push(colored);
+                }
+              }
+            }
+          }
+        }
+        if (!skipPrintNonTty) {
+          for (let i = 0; i < lines.length; i++) {
+            process.stdout.write(`\r\x1B[2K${lines[i]}
+`);
+          }
+          lastPrintedTestCaseId = testCaseId;
+          lastPrintedLineCount = lines.length;
+        }
         drawSpinner();
       }
       if (event.type === "RunCompleted" || event.type === "RunFailed") {
         runFinished = true;
         clearLine();
         unsubscribe();
-        resolve4(event);
+        resolve5(event);
       }
     });
   });
@@ -1706,6 +2193,7 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
       getEvaluatorSummaryLine(evaluatorName, aggregates.get(evaluatorId))
     );
   }
+  const testCaseSummaries = buildTestCaseSummaries(testCaseByTestId);
   if (testCaseSummaries.length > 0) {
     console.log(colorize("- test case scores:", ansi2.magenta));
     for (const summary of testCaseSummaries) {
@@ -1716,9 +2204,13 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
         );
         continue;
       }
+      const scoreLabel = summary.isAggregated && summary.aggregatedScoreItem ? getScoreById(summary.aggregatedScoreItem.id)?.format(
+        summary.aggregatedScoreItem.data,
+        { isAggregated: true }
+      ) ?? summary.averageScore.toFixed(2) : summary.averageScore.toFixed(2);
       console.log(
         `  ${status} ${summary.name.padEnd(24)} score=${colorize(
-          summary.averageScore.toFixed(2),
+          scoreLabel,
           scoreToColor(summary.averageScore)
         )} ${colorize(createBar2(summary.averageScore, 100, 14), ansi2.dim)} ${colorize(`(${summary.durationMs}ms)`, ansi2.dim)}`
       );
@@ -1727,7 +2219,7 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
   console.log(`- artifact: ${colorize(finalEvent.artifactPath, ansi2.dim)}`);
 }
 async function runSimpleEvalCommandInk(runner, datasetName, evaluatorPattern) {
-  return new Promise((resolve4, reject) => {
+  return new Promise((resolve5, reject) => {
     const app = ink.render(
       React2__default.default.createElement(RunView, {
         runner,
@@ -1738,7 +2230,7 @@ async function runSimpleEvalCommandInk(runner, datasetName, evaluatorPattern) {
           if (err) {
             reject(err);
           } else {
-            resolve4();
+            resolve5();
           }
         }
       })