@m4trix/evals 0.12.0 → 0.14.0

package/dist/cli.js

@@ -1,16 +1,17 @@
 #!/usr/bin/env node
 import { withFullScreen, useScreenSize } from 'fullscreen-ink';
-import React, { useState, useReducer, useEffect, useMemo } from 'react';
+import React2, { useState, useRef, useReducer, useEffect, useMemo } from 'react';
 import { useApp, useInput, Box, Text } from 'ink';
 import { jsx, jsxs, Fragment } from 'react/jsx-runtime';
 import { resolve, relative, join, dirname } from 'path';
-import { diffString } from 'json-diff';
+import { LineGraph } from '@pppp606/ink-chart';
 import { randomUUID } from 'crypto';
-import { Effect, PubSub, Queue, Fiber } from 'effect';
+import { Effect, PubSub, Queue, Fiber, Ref } from 'effect';
 import { existsSync } from 'fs';
 import * as jitiModule from 'jiti';
 import { readdir, readFile, mkdir, appendFile } from 'fs/promises';
 import { pathToFileURL } from 'url';
+import { diffLines } from 'diff';
 
 var SEP = "  ";
 var ARROW = "\u203A";
@@ -78,7 +79,7 @@ function getBreadcrumbText(state, datasetName, runLabel) {
 // src/cli/components/Footer.tsx
 function getFooterText(state) {
   if (state.level === "datasets") {
-    return "\u2191\u2193 move  Enter open  / search  Tab focus  q quit";
+    return state.focus === "right" ? "\u2191\u2193 scroll  Tab focus left  / search  q quit" : "\u2191\u2193 move  Enter open  Tab focus right  / search  q quit";
   }
   if (state.level === "runs") {
     return "\u2191\u2193 move  Enter details  Backspace datasets  Tab focus  q quit";
@@ -610,6 +611,7 @@ function createInitialState(data, args) {
     datasetMenuIndex,
     runMenuIndex,
     detailsScrollOffset: 0,
+    overviewScrollOffset: 0,
     selectedEvaluatorIds: data.evaluators.slice(0, 2).map((item) => item.id),
     evaluatorMenuIndex: 0,
     searchQuery,
@@ -625,8 +627,11 @@ function reduceCliState(state, action) {
     if (state.level === "details" && state.focus === "right") {
       return { ...state, detailsScrollOffset: Math.max(0, state.detailsScrollOffset - 1) };
     }
+    if (state.level === "datasets" && state.focus === "right") {
+      return { ...state, overviewScrollOffset: Math.max(0, state.overviewScrollOffset - 1) };
+    }
     if (state.level === "datasets") {
-      return { ...state, datasetMenuIndex: Math.max(0, state.datasetMenuIndex - 1) };
+      return { ...state, datasetMenuIndex: Math.max(0, state.datasetMenuIndex - 1), overviewScrollOffset: 0 };
     }
     if (state.level === "runs") {
       return { ...state, runMenuIndex: Math.max(0, state.runMenuIndex - 1) };
@@ -643,8 +648,11 @@ function reduceCliState(state, action) {
     if (state.level === "details" && state.focus === "right") {
       return { ...state, detailsScrollOffset: Math.min(action.max, state.detailsScrollOffset + 1) };
     }
+    if (state.level === "datasets" && state.focus === "right") {
+      return { ...state, overviewScrollOffset: Math.min(action.max, state.overviewScrollOffset + 1) };
+    }
     if (state.level === "datasets") {
-      return { ...state, datasetMenuIndex: Math.min(action.max, state.datasetMenuIndex + 1) };
+      return { ...state, datasetMenuIndex: Math.min(action.max, state.datasetMenuIndex + 1), overviewScrollOffset: 0 };
     }
     if (state.level === "runs") {
       return { ...state, runMenuIndex: Math.min(action.max, state.runMenuIndex + 1) };
@@ -720,249 +728,6 @@ function reduceCliState(state, action) {
   }
   return state;
 }
-var LEFT_PANE_WIDTH2 = 44;
-function DatasetsView({
-  state,
-  filteredDatasets,
-  selectedDataset
-}) {
-  const leftFocused = state.focus === "left";
-  const rightFocused = state.focus === "right";
-  return /* @__PURE__ */ jsxs(Fragment, { children: [
-    /* @__PURE__ */ jsxs(Pane, { width: LEFT_PANE_WIDTH2, focused: leftFocused, children: [
-      /* @__PURE__ */ jsx(SectionHeader, { children: "Datasets" }),
-      /* @__PURE__ */ jsx(
-        ListItem,
-        {
-          selected: state.datasetMenuIndex === 0,
-          label: "New evaluation",
-          itemKey: "datasets-new-eval"
-        }
-      ),
-      filteredDatasets.map((dataset, index) => /* @__PURE__ */ jsx(
-        ListItem,
-        {
-          selected: state.datasetMenuIndex === index + 1,
-          label: dataset.name,
-          itemKey: `dataset-${dataset.id}`
-        },
-        dataset.id
-      ))
-    ] }),
-    /* @__PURE__ */ jsxs(Pane, { flexGrow: 1, marginLeft: 1, focused: rightFocused, children: [
-      /* @__PURE__ */ jsx(SectionHeader, { children: "Overview" }),
-      /* @__PURE__ */ jsx(Text, { color: "gray", children: selectedDataset?.overview ?? "Select a dataset to inspect prior runs." })
-    ] })
-  ] });
-}
-function RunsView({
-  state,
-  dataset,
-  selectedRun
-}) {
-  const runs = dataset?.runs ?? [];
-  const rightFocused = state.focus === "right";
-  return /* @__PURE__ */ jsxs(Fragment, { children: [
-    /* @__PURE__ */ jsx(RunsSidebar, { state, dataset, runs }),
-    /* @__PURE__ */ jsx(Pane, { flexGrow: 1, marginLeft: 1, focused: rightFocused, children: !selectedRun ? /* @__PURE__ */ jsx(Text, { color: "gray", children: "Select a run to see summary metrics." }) : /* @__PURE__ */ jsxs(Box, { flexDirection: "column", children: [
-      /* @__PURE__ */ jsxs(Text, { children: [
-        /* @__PURE__ */ jsx(Text, { color: "gray", children: "Run:" }),
-        " ",
-        selectedRun.label,
-        " ",
-        /* @__PURE__ */ jsx(StatusText, { status: selectedRun.status })
-      ] }),
-      /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
-        "Commit: ",
-        selectedRun.meta.commit,
-        "  Branch: ",
-        selectedRun.meta.branch,
-        " ",
-        "Seed: ",
-        selectedRun.meta.seed
-      ] }),
-      /* @__PURE__ */ jsx(Text, { children: " " }),
-      /* @__PURE__ */ jsx(SectionHeader, { children: "Overall" }),
-      /* @__PURE__ */ jsx(
-        TextBar,
-        {
-          label: "pass rate",
-          value: selectedRun.performance.passRate,
-          format: (v) => `${v}%`
-        }
-      ),
-      /* @__PURE__ */ jsx(
-        TextBar,
-        {
-          label: "avg score",
-          value: Math.round(selectedRun.performance.avgScore * 100)
-        }
-      ),
-      /* @__PURE__ */ jsx(Text, { children: " " }),
-      /* @__PURE__ */ jsx(SectionHeader, { children: "Dimensions" }),
-      selectedRun.dimensions.map((dimension) => /* @__PURE__ */ jsx(
-        TextBar,
-        {
-          label: dimension.name,
-          value: dimension.score
-        },
-        dimension.name
-      )),
-      /* @__PURE__ */ jsx(Text, { children: " " }),
-      /* @__PURE__ */ jsx(SectionHeader, { children: "Latency trend" }),
-      /* @__PURE__ */ jsx(
-        Sparkline,
-        {
-          data: selectedRun.performance.latencyHistoryMs ?? [
-            selectedRun.performance.latencyAvgMs - 40,
-            selectedRun.performance.latencyAvgMs - 10,
-            selectedRun.performance.latencyAvgMs + 20,
-            selectedRun.performance.latencyP95Ms - 80,
-            selectedRun.performance.latencyP95Ms
-          ],
-          width: 24
-        }
-      )
-    ] }) })
-  ] });
-}
-
-// src/evals/metric.ts
-var registry = /* @__PURE__ */ new Map();
-var Metric = {
-  of(config) {
-    const def = {
-      id: config.id,
-      name: config.name,
-      format: config.format,
-      make: (data) => ({ id: config.id, data })
-    };
-    registry.set(config.id, def);
-    return def;
-  }
-};
-function getMetricById(id) {
-  return registry.get(id);
-}
-
-// src/evals/score.ts
-var registry2 = /* @__PURE__ */ new Map();
-var Score = {
-  of(config) {
-    const def = {
-      id: config.id,
-      name: config.name,
-      displayStrategy: config.displayStrategy,
-      format: config.format,
-      make: (data, options) => {
-        const passed = options?.definePassed !== void 0 ? options.definePassed(data) : void 0;
-        return {
-          id: config.id,
-          data,
-          ...passed !== void 0 && { passed }
-        };
-      }
-    };
-    registry2.set(config.id, def);
-    return def;
-  }
-};
-function getScoreById(id) {
-  return registry2.get(id);
-}
-
-// src/evals/metrics/standard.ts
-Metric.of({
-  id: "token-count",
-  name: "Tokens",
-  format: (data) => {
-    const input = data.input ?? 0;
-    const output = data.output ?? 0;
-    const inputCached = data.inputCached ?? 0;
-    const outputCached = data.outputCached ?? 0;
-    const cached = inputCached + outputCached;
-    return `in:${input} out:${output} cached:${cached}`;
-  }
-});
-Metric.of({
-  id: "latency",
-  name: "Latency",
-  format: (data) => `${data.ms}ms`
-});
-
-// src/evals/scores/standard.ts
-Score.of({
-  id: "percent",
-  name: "Score",
-  displayStrategy: "bar",
-  format: (data) => data.value.toFixed(2)
-});
-Score.of({
-  id: "binary",
-  name: "Result",
-  displayStrategy: "passFail",
-  format: (data) => data.passed ? "PASSED" : "NOT PASSED"
-});
-function createDiffLogEntry(expected, actual, options) {
-  const diff = diffString(expected, actual, { color: false });
-  return {
-    type: "diff",
-    label: options?.label,
-    expected,
-    actual,
-    diff: diff || "(no differences)"
-  };
-}
-function getDiffLines(entry) {
-  const raw = diffString(entry.expected, entry.actual, { color: false }) || "(no differences)";
-  return raw.split("\n").map((line) => {
-    const trimmed = line.trimStart();
-    if (trimmed.startsWith("-") && !trimmed.startsWith("---")) {
-      return { type: "remove", line };
-    }
-    if (trimmed.startsWith("+") && !trimmed.startsWith("+++")) {
-      return { type: "add", line };
-    }
-    return { type: "context", line };
-  });
-}
-
-// src/runner/score-utils.ts
-function toNumericScoreFromScores(scores) {
-  for (const item of scores) {
-    const def = getScoreById(item.id);
-    if (def && def.displayStrategy === "bar" && typeof item.data === "object" && item.data !== null && "value" in item.data) {
-      const value = item.data.value;
-      if (typeof value === "number" && Number.isFinite(value)) {
-        return value;
-      }
-    }
-    const numeric = toNumericScore(item.data);
-    if (numeric !== void 0) {
-      return numeric;
-    }
-  }
-  return void 0;
-}
-function toNumericScore(value) {
-  if (typeof value === "number" && Number.isFinite(value)) {
-    return value;
-  }
-  if (typeof value !== "object" || value === null) {
-    return void 0;
-  }
-  const obj = value;
-  if ("score" in obj && typeof obj.score === "number" && Number.isFinite(obj.score)) {
-    return obj.score;
-  }
-  const numberValues = Object.values(value).filter(
-    (entry) => typeof entry === "number" && Number.isFinite(entry)
-  );
-  if (numberValues.length === 0) {
-    return void 0;
-  }
-  return numberValues.reduce((sum, entry) => sum + entry, 0) / numberValues.length;
-}
 
 // src/runner/config.ts
 var defaultRunnerConfig = {
@@ -983,7 +748,8 @@ var defaultRunnerConfig = {
     ],
     excludeDirectories: ["node_modules", "dist", ".next", ".git", ".pnpm-store"]
   },
-  artifactDirectory: ".eval-results"
+  artifactDirectory: ".eval-results",
+  maxConcurrency: 1
 };
 function toRunnerConfigOverrides(config) {
   if (!config) {
@@ -1016,6 +782,9 @@ function toRunnerConfigOverrides(config) {
   if (config.artifactDirectory !== void 0) {
     overrides.artifactDirectory = config.artifactDirectory;
   }
+  if (config.maxConcurrency !== void 0) {
+    overrides.maxConcurrency = config.maxConcurrency;
+  }
   if (Object.keys(discovery).length > 0) {
     overrides.discovery = discovery;
   }
@@ -1190,25 +959,261 @@ async function collectEvaluatorsFromFiles(config) {
   );
   return found.flat();
 }
-async function collectTestCasesFromFiles(config) {
-  const files = await walkDirectory(config.rootDir, config.excludeDirectories);
-  const matched = files.filter(
-    (filePath) => hasOneSuffix(filePath, config.testCaseSuffixes)
-  );
-  const found = await Promise.all(
-    matched.map(async (absolutePath) => {
-      const exports = await loadModuleExports(absolutePath);
-      const testCases = exports.filter(isTestCaseLike);
-      const relPath = relative(config.rootDir, absolutePath);
-      return testCases.map((testCase) => ({
-        id: toId("test-case", relPath, testCase.getName()),
-        filePath: relPath,
-        testCase
-      }));
-    })
+async function collectTestCasesFromFiles(config) {
+  const files = await walkDirectory(config.rootDir, config.excludeDirectories);
+  const matched = files.filter(
+    (filePath) => hasOneSuffix(filePath, config.testCaseSuffixes)
+  );
+  const found = await Promise.all(
+    matched.map(async (absolutePath) => {
+      const exports = await loadModuleExports(absolutePath);
+      const testCases = exports.filter(isTestCaseLike);
+      const relPath = relative(config.rootDir, absolutePath);
+      return testCases.map((testCase) => ({
+        id: toId("test-case", relPath, testCase.getName()),
+        filePath: relPath,
+        testCase
+      }));
+    })
+  );
+  return found.flat();
+}
+function toJsonLines(value) {
+  try {
+    return JSON.stringify(value, null, 2);
+  } catch {
+    return String(value);
+  }
+}
+function formatDiffString(changes) {
+  const lines = [];
+  for (const part of changes) {
+    const prefix = part.added ? "+" : part.removed ? "-" : " ";
+    const partLines = part.value.split("\n");
+    if (partLines[partLines.length - 1] === "") {
+      partLines.pop();
+    }
+    for (const line of partLines) {
+      lines.push(`${prefix} ${line}`);
+    }
+  }
+  return lines.join("\n");
+}
+function createDiffString(expected, actual) {
+  const expectedStr = toJsonLines(expected);
+  const actualStr = toJsonLines(actual);
+  const changes = diffLines(expectedStr, actualStr);
+  return formatDiffString(changes);
+}
+function createDiffLogEntry(expected, actual, options) {
+  const diff = createDiffString(expected, actual);
+  return {
+    type: "diff",
+    label: options?.label,
+    expected,
+    actual,
+    diff: diff || "(no differences)"
+  };
+}
+function getDiffLines(entry) {
+  const raw = createDiffString(entry.expected, entry.actual) || "(no differences)";
+  return raw.split("\n").map((line) => {
+    const trimmed = line.trimStart();
+    if (trimmed.startsWith("-") && !trimmed.startsWith("---")) {
+      return { type: "remove", line };
+    }
+    if (trimmed.startsWith("+") && !trimmed.startsWith("+++")) {
+      return { type: "add", line };
+    }
+    return { type: "context", line };
+  });
+}
+
+// src/evals/metric.ts
+var registry = /* @__PURE__ */ new Map();
+var Metric = {
+  of(config) {
+    const def = {
+      id: config.id,
+      name: config.name,
+      aggregate: config.aggregate,
+      format: config.format,
+      make: (data) => ({ id: config.id, data })
+    };
+    registry.set(config.id, def);
+    return def;
+  }
+};
+function getMetricById(id) {
+  return registry.get(id);
+}
+
+// src/evals/score.ts
+var registry2 = /* @__PURE__ */ new Map();
+var Score = {
+  of(config) {
+    const def = {
+      id: config.id,
+      name: config.name,
+      displayStrategy: config.displayStrategy,
+      aggregate: config.aggregate,
+      format: config.format,
+      make: (data, options) => {
+        const passed = options?.definePassed !== void 0 ? options.definePassed(data) : void 0;
+        return {
+          id: config.id,
+          data,
+          ...passed !== void 0 && { passed }
+        };
+      }
+    };
+    registry2.set(config.id, def);
+    return def;
+  }
+};
+function getScoreById(id) {
+  return registry2.get(id);
+}
+
+// src/evals/aggregators.ts
+function aggregateAverageWithVariance(values) {
+  if (values.length === 0) {
+    return { value: 0, count: 0 };
+  }
+  const sum = values.reduce((s, v) => s + v.value, 0);
+  const sumSq = values.reduce((s, v) => s + v.value * v.value, 0);
+  const mean = sum / values.length;
+  let stdDev;
+  if (values.length >= 2) {
+    const variance = (sumSq - values.length * mean * mean) / (values.length - 1);
+    stdDev = variance > 0 ? Math.sqrt(variance) : 0;
+  }
+  return { value: mean, stdDev, count: values.length };
+}
+function aggregateAll(values) {
+  const total = values.length;
+  const passedCount = values.filter((v) => v.passed).length;
+  return {
+    passed: total > 0 && values.every((v) => v.passed),
+    passedCount,
+    totalCount: total
+  };
+}
+function aggregateTokenCountSum(values) {
+  const initial = {
+    input: 0,
+    output: 0,
+    inputCached: 0,
+    outputCached: 0
+  };
+  return values.reduce(
+    (acc, v) => ({
+      input: acc.input + (v.input ?? 0),
+      output: acc.output + (v.output ?? 0),
+      inputCached: acc.inputCached + (v.inputCached ?? 0),
+      outputCached: acc.outputCached + (v.outputCached ?? 0)
+    }),
+    initial
+  );
+}
+function aggregateLatencyAverage(values) {
+  if (values.length === 0) {
+    return { ms: 0 };
+  }
+  const sum = values.reduce((s, v) => s + v.ms, 0);
+  return { ms: sum / values.length };
+}
+
+// src/evals/metrics/standard.ts
+Metric.of({
+  id: "token-count",
+  name: "Tokens",
+  aggregate: aggregateTokenCountSum,
+  format: (data, options) => {
+    const input = data.input ?? 0;
+    const output = data.output ?? 0;
+    const inputCached = data.inputCached ?? 0;
+    const outputCached = data.outputCached ?? 0;
+    const cached = inputCached + outputCached;
+    const base = `in:${input} out:${output} cached:${cached}`;
+    return options?.isAggregated ? `Total: ${base}` : base;
+  }
+});
+Metric.of({
+  id: "latency",
+  name: "Latency",
+  aggregate: aggregateLatencyAverage,
+  format: (data, options) => options?.isAggregated ? `Avg: ${data.ms}ms` : `${data.ms}ms`
+});
+
+// src/evals/scores/standard.ts
+Score.of({
+  id: "percent",
+  name: "Score",
+  displayStrategy: "bar",
+  format: (data, options) => {
+    if (options?.isAggregated) {
+      return data.stdDev != null ? `Avg: ${data.value.toFixed(2)} \xB1 ${data.stdDev.toFixed(2)}` : `Avg: ${data.value.toFixed(2)}`;
+    }
+    return data.value.toFixed(2);
+  },
+  aggregate: aggregateAverageWithVariance
+});
+Score.of({
+  id: "binary",
+  name: "Result",
+  displayStrategy: "passFail",
+  format: (data, options) => {
+    if (options?.isAggregated) {
+      const base = data.passed ? "All: PASSED" : "Some: FAILED";
+      if (data.passedCount != null && data.totalCount != null && data.totalCount > 1) {
+        return `${base} (${data.passedCount}/${data.totalCount})`;
+      }
+      return base;
+    }
+    return data.passed ? "PASSED" : "NOT PASSED";
+  },
+  aggregate: aggregateAll
+});
+
+// src/runner/score-utils.ts
+function toNumericScoreFromScores(scores) {
+  for (const item of scores) {
+    const def = getScoreById(item.id);
+    if (def && def.displayStrategy === "bar" && typeof item.data === "object" && item.data !== null && "value" in item.data) {
+      const value = item.data.value;
+      if (typeof value === "number" && Number.isFinite(value)) {
+        return value;
+      }
+    }
+    const numeric = toNumericScore(item.data);
+    if (numeric !== void 0) {
+      return numeric;
+    }
+  }
+  return void 0;
+}
+function toNumericScore(value) {
+  if (typeof value === "number" && Number.isFinite(value)) {
+    return value;
+  }
+  if (typeof value !== "object" || value === null) {
+    return void 0;
+  }
+  const obj = value;
+  if ("score" in obj && typeof obj.score === "number" && Number.isFinite(obj.score)) {
+    return obj.score;
+  }
+  const numberValues = Object.values(value).filter(
+    (entry) => typeof entry === "number" && Number.isFinite(entry)
   );
-  return found.flat();
+  if (numberValues.length === 0) {
+    return void 0;
+  }
+  return numberValues.reduce((sum, entry) => sum + entry, 0) / numberValues.length;
 }
+
+// src/runner/execution.ts
 function computeEvaluatorPassed(evaluator, result, scores) {
   const scoresWithPassed = scores.filter((s) => "passed" in s && s.passed !== void 0);
   if (scoresWithPassed.length > 0) {
@@ -1250,6 +1255,105 @@ function createArtifactPath(artifactDirectory, datasetId, runId) {
     `${datasetId}_${runId}_${nowIsoForFile()}.jsonl`
   );
 }
+function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent, persistenceQueue, updateSnapshot, completedRef, passedRef, failedRef) {
+  return Effect.gen(function* () {
+    const reruns = typeof testCaseItem.testCase.getReruns === "function" ? testCaseItem.testCase.getReruns() : 1;
+    const rerunPassed = [];
+    for (let r = 0; r < reruns; r++) {
+      const started = Date.now();
+      const evaluatorScores = [];
+      let testCaseError;
+      const output = readOutput(testCaseItem.testCase);
+      for (const { id: evaluatorId, evaluator } of task.evaluators) {
+        const evaluateFn = evaluator.getEvaluateFn();
+        if (!evaluateFn) {
+          continue;
+        }
+        try {
+          const logs = [];
+          const logDiff = (expected, actual, options) => {
+            logs.push(createDiffLogEntry(expected, actual, options));
+          };
+          const ctx = yield* Effect.promise(
+            () => Promise.resolve(evaluator.resolveContext())
+          );
+          const result = yield* Effect.promise(
+            () => Promise.resolve(
+              evaluateFn({
+                input: testCaseItem.testCase.getInput(),
+                ctx,
+                output,
+                logDiff
+              })
+            )
+          );
+          const { scores, metrics } = normalizeResult(result);
+          const passed2 = computeEvaluatorPassed(evaluator, result, scores);
+          evaluatorScores.push({
+            evaluatorId,
+            scores,
+            passed: passed2,
+            metrics,
+            logs: logs.length > 0 ? logs : void 0
+          });
+        } catch (error) {
+          testCaseError = error instanceof Error ? error.message : "Evaluator execution failed";
+          evaluatorScores.push({
+            evaluatorId,
+            scores: [],
+            passed: false
+          });
+        }
+      }
+      const rerunPassedThis = evaluatorScores.every((s) => s.passed);
+      rerunPassed.push(rerunPassedThis);
+      const completedEvaluations = yield* Ref.modify(completedRef, (n) => [
+        n + 1,
+        n + 1
+      ]);
+      const progressEvent = {
+        type: "TestCaseProgress",
+        runId: task.runId,
+        testCaseId: testCaseItem.id,
+        testCaseName: testCaseItem.testCase.getName(),
+        completedTestCases: completedEvaluations,
+        totalTestCases: totalEvaluations,
+        rerunIndex: r + 1,
+        rerunTotal: reruns,
+        passed: rerunPassedThis,
+        durationMs: Date.now() - started,
+        evaluatorScores,
+        output,
+        errorMessage: testCaseError
+      };
+      updateSnapshot(task.runId, (snapshot) => ({
+        ...snapshot,
+        completedTestCases: completedEvaluations
+      }));
+      yield* publishEvent(progressEvent);
+      yield* Queue.offer(persistenceQueue, {
+        runId: task.runId,
+        artifactPath: task.snapshot.artifactPath,
+        payload: progressEvent
+      });
+    }
+    const testCasePassed = rerunPassed.every(Boolean);
+    if (testCasePassed) {
+      yield* Ref.update(passedRef, (n) => n + 1);
+    } else {
+      yield* Ref.update(failedRef, (n) => n + 1);
+    }
+    const [passed, failed] = yield* Effect.all([
+      Ref.get(passedRef),
+      Ref.get(failedRef)
+    ]);
+    updateSnapshot(task.runId, (snapshot) => ({
+      ...snapshot,
+      passedTestCases: passed,
+      failedTestCases: failed
+    }));
+  });
+}
 var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => Effect.gen(function* () {
   const startedAt = Date.now();
   updateSnapshot(task.runId, (snapshot) => ({
@@ -1262,104 +1366,51 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
     runId: task.runId,
     startedAt
   });
-  let completedTestCases = 0;
-  let passedTestCases = 0;
-  let failedTestCases = 0;
-  for (const testCaseItem of task.testCases) {
-    const started = Date.now();
-    const evaluatorScores = [];
-    let testCaseError;
-    const output = readOutput(testCaseItem.testCase);
-    for (const { id: evaluatorId, evaluator } of task.evaluators) {
-      const evaluateFn = evaluator.getEvaluateFn();
-      if (!evaluateFn) {
-        continue;
-      }
-      try {
-        const logs = [];
-        const logDiff = (expected, actual, options) => {
-          logs.push(createDiffLogEntry(expected, actual, options));
-        };
-        const ctx = yield* Effect.promise(
-          () => Promise.resolve(evaluator.resolveContext())
-        );
-        const result = yield* Effect.promise(
-          () => Promise.resolve(
-            evaluateFn({
-              input: testCaseItem.testCase.getInput(),
-              ctx,
-              output,
-              logDiff
-            })
-          )
-        );
-        const { scores, metrics } = normalizeResult(result);
-        const passed = computeEvaluatorPassed(evaluator, result, scores);
-        evaluatorScores.push({
-          evaluatorId,
-          scores,
-          passed,
-          metrics,
-          logs: logs.length > 0 ? logs : void 0
-        });
-      } catch (error) {
-        testCaseError = error instanceof Error ? error.message : "Evaluator execution failed";
-        evaluatorScores.push({
-          evaluatorId,
-          scores: [],
-          passed: false
-        });
-      }
-    }
-    const testCasePassed = evaluatorScores.every((s) => s.passed);
-    completedTestCases += 1;
-    if (testCasePassed) {
-      passedTestCases += 1;
-    } else {
-      failedTestCases += 1;
-    }
-    const progressEvent = {
-      type: "TestCaseProgress",
-      runId: task.runId,
-      testCaseId: testCaseItem.id,
-      testCaseName: testCaseItem.testCase.getName(),
-      completedTestCases,
-      totalTestCases: task.testCases.length,
-      passed: testCasePassed,
-      durationMs: Date.now() - started,
-      evaluatorScores,
-      output,
-      errorMessage: testCaseError
-    };
-    updateSnapshot(task.runId, (snapshot) => ({
-      ...snapshot,
-      completedTestCases,
-      passedTestCases,
-      failedTestCases
-    }));
-    yield* publishEvent(progressEvent);
-    yield* Queue.offer(persistenceQueue, {
-      runId: task.runId,
-      artifactPath: task.snapshot.artifactPath,
-      payload: progressEvent
-    });
-  }
+  const totalEvaluations = task.testCases.reduce(
+    (sum, tc) => sum + (typeof tc.testCase.getReruns === "function" ? tc.testCase.getReruns() : 1),
+    0
+  );
+  const maxConcurrency = Math.max(1, task.maxConcurrency ?? 1);
+  const completedRef = yield* Ref.make(0);
+  const passedRef = yield* Ref.make(0);
+  const failedRef = yield* Ref.make(0);
+  const processTestCase = (testCaseItem) => processOneTestCase(
+    task,
+    testCaseItem,
+    totalEvaluations,
+    publishEvent,
+    persistenceQueue,
+    updateSnapshot,
+    completedRef,
+    passedRef,
+    failedRef
+  );
+  yield* Effect.forEach(
+    task.testCases,
+    processTestCase,
+    maxConcurrency > 1 ? { concurrency: maxConcurrency } : void 0
+  );
+  const [completedEvaluations, passedUniqueTestCases, failedUniqueTestCases] = yield* Effect.all([
+    Ref.get(completedRef),
+    Ref.get(passedRef),
+    Ref.get(failedRef)
+  ]);
   const finishedAt = Date.now();
   const completedEvent = {
     type: "RunCompleted",
     runId: task.runId,
     finishedAt,
-    passedTestCases,
-    failedTestCases,
+    passedTestCases: passedUniqueTestCases,
+    failedTestCases: failedUniqueTestCases,
     totalTestCases: task.testCases.length,
     artifactPath: task.snapshot.artifactPath
   };
   updateSnapshot(task.runId, (snapshot) => ({
     ...snapshot,
     status: "completed",
-    completedTestCases,
-    passedTestCases,
-    failedTestCases,
+    completedTestCases: completedEvaluations,
+    passedTestCases: passedUniqueTestCases,
+    failedTestCases: failedUniqueTestCases,
     finishedAt
   }));
   yield* publishEvent(completedEvent);
@@ -1447,7 +1498,7 @@ async function parseArtifactToSnapshot(filePath, _config) {
   const artifactPath = filePath;
   const status = runFailed ? "failed" : runCompleted ? "completed" : runStarted ? "running" : "queued";
   const progress = aggregateTestCaseProgress(lines);
-  const completedTestCases = runCompleted?.totalTestCases ?? progress.completedTestCases;
+  const completedTestCases = runCompleted ? runQueued.totalTestCases : progress.completedTestCases;
   const passedTestCases = runCompleted?.passedTestCases ?? progress.passedTestCases;
   const failedTestCases = runCompleted?.failedTestCases ?? progress.failedTestCases;
   return {
@@ -1469,23 +1520,29 @@ async function parseArtifactToSnapshot(filePath, _config) {
 }
 function aggregateTestCaseProgress(lines) {
   let completedTestCases = 0;
-  let passedTestCases = 0;
-  let failedTestCases = 0;
+  const testCasePassedBy = /* @__PURE__ */ new Map();
   for (const line of lines) {
     try {
       const event = JSON.parse(line);
       if (event.type === "TestCaseProgress") {
         const ev = event;
         completedTestCases = ev.completedTestCases ?? completedTestCases;
-        if (ev.passed) {
-          passedTestCases += 1;
-        } else {
-          failedTestCases += 1;
-        }
+        const id = ev.testCaseId;
+        const current = testCasePassedBy.get(id);
+        testCasePassedBy.set(id, current === void 0 ? ev.passed : current && ev.passed);
       }
     } catch {
     }
   }
+  let passedTestCases = 0;
+  let failedTestCases = 0;
+  for (const passed of testCasePassedBy.values()) {
+    if (passed) {
+      passedTestCases += 1;
+    } else {
+      failedTestCases += 1;
+    }
+  }
   return { completedTestCases, passedTestCases, failedTestCases };
 }
 async function parseArtifactFile(artifactPath) {
@@ -1503,6 +1560,8 @@ async function parseArtifactFile(artifactPath) {
             testCaseName: ev.testCaseName,
             completedTestCases: ev.completedTestCases,
             totalTestCases: ev.totalTestCases,
+            rerunIndex: ev.rerunIndex,
+            rerunTotal: ev.rerunTotal,
             passed: ev.passed,
             durationMs: ev.durationMs,
             evaluatorScores: ev.evaluatorScores ?? []
@@ -1708,6 +1767,10 @@ var EffectRunner = class {
       throw new Error("No evaluators selected for run");
     }
     const selectedTestCases = await this.collectDatasetTestCases(request.datasetId);
+    const totalEvaluations = selectedTestCases.reduce(
+      (sum, tc) => sum + (typeof tc.testCase.getReruns === "function" ? tc.testCase.getReruns() : 1),
+      0
+    );
     const runId = `run-${randomUUID()}`;
     const artifactPath = createArtifactPath(
       this.config.artifactDirectory,
@@ -1720,7 +1783,7 @@ var EffectRunner = class {
       datasetName: dataset.dataset.getName(),
       evaluatorIds: selectedEvaluators.map((item) => item.id),
       queuedAt: Date.now(),
-      totalTestCases: selectedTestCases.length,
+      totalTestCases: totalEvaluations,
       completedTestCases: 0,
       passedTestCases: 0,
       failedTestCases: 0,
@@ -1734,7 +1797,7 @@ var EffectRunner = class {
       datasetId: request.datasetId,
       datasetName: dataset.dataset.getName(),
       evaluatorIds: selectedEvaluators.map((item) => item.id),
-      totalTestCases: selectedTestCases.length,
+      totalTestCases: totalEvaluations,
       artifactPath
     };
     await Effect.runPromise(this.publishEvent(queuedEvent));
@@ -1745,6 +1808,7 @@ var EffectRunner = class {
         payload: queuedEvent
       })
     );
+    const maxConcurrency = request.concurrency ?? this.config.maxConcurrency ?? 1;
     await Effect.runPromise(
       Queue.offer(this.runQueue, {
         runId,
@@ -1752,7 +1816,8 @@ var EffectRunner = class {
         dataset: dataset.dataset,
         evaluators: selectedEvaluators,
         testCases: selectedTestCases,
-        snapshot
+        snapshot,
+        maxConcurrency
       })
     );
     return snapshot;
@@ -1819,6 +1884,240 @@ var EffectRunner = class {
     );
   }
 };
+var LEFT_PANE_WIDTH2 = 44;
+var MAX_RUNS_FOR_CHART = 12;
+var MAX_RUNS_FOR_TREND = 20;
+var TREND_BATCH_SIZE = 4;
+function extractRunAverageScore(testCases) {
+  const scores = [];
+  for (const tc of testCases) {
+    for (const es of tc.evaluatorScores) {
+      const n = toNumericScoreFromScores(es.scores);
+      if (n !== void 0) {
+        scores.push(n);
+      }
+    }
+  }
+  if (scores.length === 0)
+    return void 0;
+  return scores.reduce((a, b) => a + b, 0) / scores.length;
+}
+async function loadRunScores(runs) {
+  const results = [];
+  for (const run of runs) {
+    const artifact = run.meta?.artifact;
+    if (!artifact)
+      continue;
+    try {
+      const path = resolve(artifact);
+      const testCases = await parseArtifactFile(path);
+      const avg = extractRunAverageScore(testCases);
+      if (avg !== void 0) {
+        results.push({
+          runId: run.id,
+          label: run.label,
+          value: avg
+        });
+      }
+    } catch {
+    }
+  }
+  return results;
+}
+function batchAverage(values, batchSize) {
+  const batches = [];
+  for (let i = 0; i < values.length; i += batchSize) {
+    const slice = values.slice(i, i + batchSize);
+    if (slice.length > 0) {
+      batches.push(slice.reduce((a, b) => a + b, 0) / slice.length);
+    }
+  }
+  return batches;
+}
+var OVERVIEW_PAGE_SIZE = 15;
+function DatasetsView({
+  state,
+  filteredDatasets,
+  selectedDataset,
+  overviewRowCountRef
+}) {
+  const leftFocused = state.focus === "left";
+  const rightFocused = state.focus === "right";
+  const [runScores, setRunScores] = useState([]);
+  const [loading, setLoading] = useState(false);
+  useEffect(() => {
+    if (!selectedDataset?.runs?.length) {
+      setRunScores([]);
+      return;
+    }
+    setLoading(true);
+    const runs = selectedDataset.runs.slice(0, MAX_RUNS_FOR_TREND);
+    loadRunScores(runs).then(setRunScores).finally(() => setLoading(false));
+  }, [selectedDataset?.id, selectedDataset?.runs?.length]);
+  const barData = runScores.slice(0, MAX_RUNS_FOR_CHART).reverse();
+  const trendValues = runScores.slice(0, MAX_RUNS_FOR_TREND).map((r) => r.value).reverse();
+  const trendBatched = batchAverage(trendValues, TREND_BATCH_SIZE);
+  const overviewRows = useMemo(() => {
+    const rows = [];
+    rows.push(
+      /* @__PURE__ */ jsx(Text, { color: "gray", children: selectedDataset?.overview ?? "Select a dataset to inspect prior runs." }, "overview")
+    );
+    if (selectedDataset && selectedDataset.runs.length > 0) {
+      if (loading) {
+        rows.push(
+          /* @__PURE__ */ jsx(Text, { color: "gray", children: "Loading run scores\u2026" }, "loading")
+        );
+      } else if (runScores.length > 0) {
+        rows.push(
+          /* @__PURE__ */ jsx(Text, { color: "gray", children: "Scores (last runs)" }, "scores-header")
+        );
+        for (const d of barData) {
+          rows.push(
+            /* @__PURE__ */ jsx(
+              TextBar,
+              {
+                label: d.label,
+                value: d.value,
+                labelWidth: 14,
+                barWidth: 24,
+                max: 100,
+                format: (v) => v.toFixed(1)
+              },
+              d.runId
+            )
+          );
+        }
+        if (trendBatched.length > 0) {
+          rows.push(
+            /* @__PURE__ */ jsx(Text, { color: "gray", children: "Avg trend (last 20, batched by 4)" }, "trend-header")
+          );
+          rows.push(
+            /* @__PURE__ */ jsx(Box, { children: /* @__PURE__ */ jsx(
+              LineGraph,
+              {
+                data: [{ values: trendBatched, color: "cyan" }],
+                height: 5,
+                width: 45,
+                showYAxis: true,
+                xLabels: ["older", "newer"]
+              }
+            ) }, "trend-graph")
+          );
+        }
+      }
+    }
+    return rows;
+  }, [
+    selectedDataset?.overview,
+    selectedDataset?.runs?.length,
+    loading,
+    runScores,
+    barData,
+    trendBatched
+  ]);
+  if (overviewRowCountRef) {
+    overviewRowCountRef.current = overviewRows.length;
+  }
+  const offset = Math.max(0, state.overviewScrollOffset);
+  const visibleRows = overviewRows.slice(offset, offset + OVERVIEW_PAGE_SIZE);
+  return /* @__PURE__ */ jsxs(Fragment, { children: [
+    /* @__PURE__ */ jsxs(Pane, { width: LEFT_PANE_WIDTH2, focused: leftFocused, children: [
+      /* @__PURE__ */ jsx(SectionHeader, { children: "Datasets" }),
+      /* @__PURE__ */ jsx(
+        ListItem,
+        {
+          selected: state.datasetMenuIndex === 0,
+          label: "New evaluation",
+          itemKey: "datasets-new-eval"
+        }
+      ),
+      filteredDatasets.map((dataset, index) => /* @__PURE__ */ jsx(
+        ListItem,
+        {
+          selected: state.datasetMenuIndex === index + 1,
+          label: dataset.name,
+          itemKey: `dataset-${dataset.id}`
+        },
+        dataset.id
+      ))
+    ] }),
+    /* @__PURE__ */ jsxs(Pane, { flexGrow: 1, marginLeft: 1, focused: rightFocused, children: [
+      /* @__PURE__ */ jsx(SectionHeader, { children: "Overview" }),
+      /* @__PURE__ */ jsx(Box, { flexDirection: "column", children: visibleRows.map((row, i) => /* @__PURE__ */ jsx(Box, { children: row }, offset + i)) })
+    ] })
+  ] });
+}
+function RunsView({
+  state,
+  dataset,
+  selectedRun
+}) {
+  const runs = dataset?.runs ?? [];
+  const rightFocused = state.focus === "right";
+  return /* @__PURE__ */ jsxs(Fragment, { children: [
+    /* @__PURE__ */ jsx(RunsSidebar, { state, dataset, runs }),
+    /* @__PURE__ */ jsx(Pane, { flexGrow: 1, marginLeft: 1, focused: rightFocused, children: !selectedRun ? /* @__PURE__ */ jsx(Text, { color: "gray", children: "Select a run to see summary metrics." }) : /* @__PURE__ */ jsxs(Box, { flexDirection: "column", children: [
+      /* @__PURE__ */ jsxs(Text, { children: [
+        /* @__PURE__ */ jsx(Text, { color: "gray", children: "Run:" }),
+        " ",
+        selectedRun.label,
+        " ",
+        /* @__PURE__ */ jsx(StatusText, { status: selectedRun.status })
+      ] }),
+      /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
+        "Commit: ",
+        selectedRun.meta.commit,
+        "  Branch: ",
+        selectedRun.meta.branch,
+        " ",
+        "Seed: ",
+        selectedRun.meta.seed
+      ] }),
+      /* @__PURE__ */ jsx(Text, { children: " " }),
+      /* @__PURE__ */ jsx(SectionHeader, { children: "Overall" }),
+      /* @__PURE__ */ jsx(
+        TextBar,
+        {
+          label: "pass rate",
+          value: selectedRun.performance.passRate,
+          format: (v) => `${v}%`
+        }
+      ),
+      /* @__PURE__ */ jsx(
+        TextBar,
+        {
+          label: "avg score",
+          value: Math.round(selectedRun.performance.avgScore * 100)
+        }
+      ),
+      /* @__PURE__ */ jsx(Text, { children: " " }),
+      /* @__PURE__ */ jsx(SectionHeader, { children: "Dimensions" }),
+      selectedRun.dimensions.map((dimension) => /* @__PURE__ */ jsx(
+        TextBar,
+        {
+          label: dimension.name,
+          value: dimension.score
+        },
+        dimension.name
+      )),
+      /* @__PURE__ */ jsx(Text, { children: " " }),
+      /* @__PURE__ */ jsx(SectionHeader, { children: "Latency trend" }),
+      /* @__PURE__ */ jsx(
+        Sparkline,
+        {
+          data: selectedRun.performance.latencyHistoryMs ?? [
+            selectedRun.performance.latencyAvgMs - 40,
+            selectedRun.performance.latencyAvgMs - 10,
+            selectedRun.performance.latencyAvgMs + 20,
+            selectedRun.performance.latencyP95Ms - 80,
+            selectedRun.performance.latencyP95Ms
+          ],
+          width: 24
+        }
+      )
+    ] }) })
+  ] });
+}
 var DETAILS_PAGE_SIZE = 20;
 function scoreColor(score) {
   if (score >= 80)
@@ -1827,7 +2126,7 @@ function scoreColor(score) {
     return "yellow";
   return "red";
 }
-function formatScorePart(item, scoreToColor) {
+function formatScorePart(item) {
   const def = getScoreById(item.id);
   if (!def) {
     const numeric = toNumericScore(item.data);
@@ -1857,7 +2156,7 @@ function CheckRow({
     " ",
     /* @__PURE__ */ jsx(Text, { color, bold: true, children: status }),
     detail ? /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
-      "  (",
+      " (",
       detail,
       ")"
     ] }) : null
@@ -1877,21 +2176,21 @@ function buildDetailRows(run, testCases, evaluatorNameById) {
     /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
       "Model: ",
       meta.model,
-      "   Provider: ",
+      " Provider: ",
       meta.provider
     ] }, "meta-1"),
     /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
       "Commit: ",
       meta.commit,
-      "  Branch: ",
+      " Branch: ",
       meta.branch,
-      "  Seed: ",
+      " Seed: ",
       meta.seed
     ] }, "meta-2"),
     /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
       "Duration: ",
       meta.duration,
-      "   Concurrency: ",
+      " Concurrency: ",
       meta.concurrency
     ] }, "meta-3"),
     /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
@@ -1903,7 +2202,15 @@ function buildDetailRows(run, testCases, evaluatorNameById) {
     ...dimensions.map((d) => /* @__PURE__ */ jsx(TextBar, { label: d.name, value: d.score }, `dim-${d.name}`)),
     /* @__PURE__ */ jsx(Text, { children: " " }, "sp2"),
     /* @__PURE__ */ jsx(SectionHeader, { children: "Checks (boolean)" }, "checks-h"),
-    ...checks.map((c) => /* @__PURE__ */ jsx(CheckRow, { name: c.name, passed: c.passed, detail: c.detail }, `chk-${c.name}`)),
+    ...checks.map((c) => /* @__PURE__ */ jsx(
+      CheckRow,
+      {
+        name: c.name,
+        passed: c.passed,
+        detail: c.detail
+      },
+      `chk-${c.name}`
+    )),
     /* @__PURE__ */ jsx(Text, { children: " " }, "sp3"),
     /* @__PURE__ */ jsx(SectionHeader, { children: "Performance" }, "perf-h"),
     /* @__PURE__ */ jsx(
@@ -1916,16 +2223,16 @@ function buildDetailRows(run, testCases, evaluatorNameById) {
       "perf-rate"
     ),
     /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
-      "latency avg     ",
+      "latency avg ",
       performance.latencyAvgMs,
-      "ms   p95 ",
+      "ms p95 ",
       performance.latencyP95Ms,
       "ms"
     ] }, "perf-lat"),
     /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
-      "tokens avg      ",
+      "tokens avg ",
       performance.tokensAvg,
-      "   p95 ",
+      " p95 ",
       performance.tokensP95
     ] }, "perf-tok"),
     /* @__PURE__ */ jsx(Text, { children: " " }, "sp4"),
@@ -1949,6 +2256,7 @@ function buildDetailRows(run, testCases, evaluatorNameById) {
     rows.push(/* @__PURE__ */ jsx(Text, { children: " " }, "sp6"));
     rows.push(/* @__PURE__ */ jsx(SectionHeader, { children: "Test cases" }, "tc-h"));
     for (const tc of testCases) {
+      const rerunPart = tc.rerunTotal != null && tc.rerunIndex != null ? ` (${tc.rerunIndex}/${tc.rerunTotal})` : "";
       rows.push(
         /* @__PURE__ */ jsxs(Text, { children: [
           /* @__PURE__ */ jsxs(Text, { color: "cyan", children: [
@@ -1960,12 +2268,13 @@ function buildDetailRows(run, testCases, evaluatorNameById) {
           ] }),
           " ",
           tc.testCaseName,
+          rerunPart ? /* @__PURE__ */ jsx(Text, { color: "cyan", children: rerunPart }) : null,
           /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
             " (",
             tc.durationMs,
             "ms)"
           ] })
-        ] }, `tc-${tc.testCaseId}`)
+        ] }, `tc-${tc.testCaseId}-${tc.rerunIndex ?? 0}`)
       );
       for (const item of tc.evaluatorScores) {
         const name = evaluatorNameById.get(item.evaluatorId) ?? item.evaluatorId;
@@ -2034,7 +2343,7 @@ function RunDetailsView({
   const runs = dataset?.runs ?? [];
   const rightFocused = state.focus === "right";
   const [testCases, setTestCases] = useState([]);
-  const evaluatorNameById = React.useMemo(
+  const evaluatorNameById = React2.useMemo(
     () => new Map(evaluators.map((e) => [e.id, e.name])),
     [evaluators]
   );
@@ -2057,7 +2366,7 @@ function RunDetailsView({
   const visible = rows.slice(offset, offset + DETAILS_PAGE_SIZE);
   return /* @__PURE__ */ jsxs(Fragment, { children: [
     /* @__PURE__ */ jsx(RunsSidebar, { state, dataset, runs }),
-    /* @__PURE__ */ jsx(Pane, { flexGrow: 1, marginLeft: 1, focused: rightFocused, children: /* @__PURE__ */ jsx(Box, { flexDirection: "column", children: visible.map((row, i) => /* @__PURE__ */ jsx(React.Fragment, { children: row }, i)) }) })
+    /* @__PURE__ */ jsx(Pane, { flexGrow: 1, marginLeft: 1, focused: rightFocused, children: /* @__PURE__ */ jsx(Box, { flexDirection: "column", children: visible.map((row, i) => /* @__PURE__ */ jsx(React2.Fragment, { children: row }, i)) }) })
   ] });
 }
 var LEFT_PANE_WIDTH3 = 44;
@@ -2139,6 +2448,7 @@ function EvalsCliApp({
   const { width: stdoutWidth, height: stdoutHeight } = useScreenSize();
   const [liveData, setLiveData] = useState(data);
   const [runtimeMessage, setRuntimeMessage] = useState();
+  const overviewRowCountRef = useRef(0);
   const [state, dispatch] = useReducer(
     reduceCliState,
     createInitialState(data, args)
@@ -2218,7 +2528,16 @@ function EvalsCliApp({
       return;
     }
     if (key.downArrow) {
-      const max = clampedState.level === "datasets" ? filteredDatasets.length : clampedState.level === "runs" ? selectedDataset?.runs.length ?? 0 : clampedState.level === "new-evaluation" ? Math.max(0, visibleEvaluators.length - 1) : 100;
+      let max;
+      if (clampedState.level === "datasets") {
+        max = clampedState.focus === "right" ? Math.max(0, overviewRowCountRef.current - OVERVIEW_PAGE_SIZE) : filteredDatasets.length;
+      } else if (clampedState.level === "runs") {
+        max = selectedDataset?.runs.length ?? 0;
+      } else if (clampedState.level === "new-evaluation") {
+        max = Math.max(0, visibleEvaluators.length - 1);
+      } else {
+        max = 100;
+      }
       dispatch({ type: "MOVE_DOWN", max });
       return;
     }
@@ -2236,7 +2555,7 @@ function EvalsCliApp({
       }
       return;
     }
-    if (isBackKey(key)) {
+    if (isBackKey(key) || input === "\x7F" || input === "\b") {
       dispatch({ type: "BACK" });
       return;
     }
@@ -2289,7 +2608,8 @@ function EvalsCliApp({
         {
           state: clampedState,
           filteredDatasets,
-          selectedDataset
+          selectedDataset,
+          overviewRowCountRef
         }
       );
     }