@m4trix/evals 0.13.0 → 0.15.0

package/dist/cli.cjs

@@ -2,17 +2,18 @@
 'use strict';
 
 var fullscreenInk = require('fullscreen-ink');
-var React = require('react');
+var React2 = require('react');
 var ink = require('ink');
 var jsxRuntime = require('react/jsx-runtime');
 var path = require('path');
-var jsonDiff = require('json-diff');
+var inkChart = require('@pppp606/ink-chart');
 var crypto = require('crypto');
 var effect = require('effect');
 var fs = require('fs');
 var jitiModule = require('jiti');
 var promises = require('fs/promises');
 var url = require('url');
+var diff = require('diff');
 
 var _documentCurrentScript = typeof document !== 'undefined' ? document.currentScript : null;
 function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
@@ -35,7 +36,7 @@ function _interopNamespace(e) {
   return Object.freeze(n);
 }
 
-var React__default = /*#__PURE__*/_interopDefault(React);
+var React2__default = /*#__PURE__*/_interopDefault(React2);
 var jitiModule__namespace = /*#__PURE__*/_interopNamespace(jitiModule);
 
 var SEP = "  ";
@@ -104,7 +105,7 @@ function getBreadcrumbText(state, datasetName, runLabel) {
 // src/cli/components/Footer.tsx
 function getFooterText(state) {
   if (state.level === "datasets") {
-    return "\u2191\u2193 move  Enter open  / search  Tab focus  q quit";
+    return state.focus === "right" ? "\u2191\u2193 scroll  Tab focus left  / search  q quit" : "\u2191\u2193 move  Enter open  Tab focus right  / search  q quit";
   }
   if (state.level === "runs") {
     return "\u2191\u2193 move  Enter details  Backspace datasets  Tab focus  q quit";
@@ -636,6 +637,7 @@ function createInitialState(data, args) {
     datasetMenuIndex,
     runMenuIndex,
     detailsScrollOffset: 0,
+    overviewScrollOffset: 0,
     selectedEvaluatorIds: data.evaluators.slice(0, 2).map((item) => item.id),
     evaluatorMenuIndex: 0,
     searchQuery,
@@ -651,8 +653,11 @@ function reduceCliState(state, action) {
     if (state.level === "details" && state.focus === "right") {
       return { ...state, detailsScrollOffset: Math.max(0, state.detailsScrollOffset - 1) };
     }
+    if (state.level === "datasets" && state.focus === "right") {
+      return { ...state, overviewScrollOffset: Math.max(0, state.overviewScrollOffset - 1) };
+    }
     if (state.level === "datasets") {
-      return { ...state, datasetMenuIndex: Math.max(0, state.datasetMenuIndex - 1) };
+      return { ...state, datasetMenuIndex: Math.max(0, state.datasetMenuIndex - 1), overviewScrollOffset: 0 };
     }
     if (state.level === "runs") {
       return { ...state, runMenuIndex: Math.max(0, state.runMenuIndex - 1) };
@@ -669,8 +674,11 @@ function reduceCliState(state, action) {
     if (state.level === "details" && state.focus === "right") {
       return { ...state, detailsScrollOffset: Math.min(action.max, state.detailsScrollOffset + 1) };
     }
+    if (state.level === "datasets" && state.focus === "right") {
+      return { ...state, overviewScrollOffset: Math.min(action.max, state.overviewScrollOffset + 1) };
+    }
     if (state.level === "datasets") {
-      return { ...state, datasetMenuIndex: Math.min(action.max, state.datasetMenuIndex + 1) };
+      return { ...state, datasetMenuIndex: Math.min(action.max, state.datasetMenuIndex + 1), overviewScrollOffset: 0 };
     }
     if (state.level === "runs") {
       return { ...state, runMenuIndex: Math.min(action.max, state.runMenuIndex + 1) };
@@ -746,292 +754,6 @@ function reduceCliState(state, action) {
   }
   return state;
 }
-var LEFT_PANE_WIDTH2 = 44;
-function DatasetsView({
-  state,
-  filteredDatasets,
-  selectedDataset
-}) {
-  const leftFocused = state.focus === "left";
-  const rightFocused = state.focus === "right";
-  return /* @__PURE__ */ jsxRuntime.jsxs(jsxRuntime.Fragment, { children: [
-    /* @__PURE__ */ jsxRuntime.jsxs(Pane, { width: LEFT_PANE_WIDTH2, focused: leftFocused, children: [
-      /* @__PURE__ */ jsxRuntime.jsx(SectionHeader, { children: "Datasets" }),
-      /* @__PURE__ */ jsxRuntime.jsx(
-        ListItem,
-        {
-          selected: state.datasetMenuIndex === 0,
-          label: "New evaluation",
-          itemKey: "datasets-new-eval"
-        }
-      ),
-      filteredDatasets.map((dataset, index) => /* @__PURE__ */ jsxRuntime.jsx(
-        ListItem,
-        {
-          selected: state.datasetMenuIndex === index + 1,
-          label: dataset.name,
-          itemKey: `dataset-${dataset.id}`
-        },
-        dataset.id
-      ))
-    ] }),
-    /* @__PURE__ */ jsxRuntime.jsxs(Pane, { flexGrow: 1, marginLeft: 1, focused: rightFocused, children: [
-      /* @__PURE__ */ jsxRuntime.jsx(SectionHeader, { children: "Overview" }),
-      /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "gray", children: selectedDataset?.overview ?? "Select a dataset to inspect prior runs." })
-    ] })
-  ] });
-}
-function RunsView({
-  state,
-  dataset,
-  selectedRun
-}) {
-  const runs = dataset?.runs ?? [];
-  const rightFocused = state.focus === "right";
-  return /* @__PURE__ */ jsxRuntime.jsxs(jsxRuntime.Fragment, { children: [
-    /* @__PURE__ */ jsxRuntime.jsx(RunsSidebar, { state, dataset, runs }),
-    /* @__PURE__ */ jsxRuntime.jsx(Pane, { flexGrow: 1, marginLeft: 1, focused: rightFocused, children: !selectedRun ? /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "gray", children: "Select a run to see summary metrics." }) : /* @__PURE__ */ jsxRuntime.jsxs(ink.Box, { flexDirection: "column", children: [
-      /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { children: [
-        /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "gray", children: "Run:" }),
-        " ",
-        selectedRun.label,
-        " ",
-        /* @__PURE__ */ jsxRuntime.jsx(StatusText, { status: selectedRun.status })
-      ] }),
-      /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
-        "Commit: ",
-        selectedRun.meta.commit,
-        "  Branch: ",
-        selectedRun.meta.branch,
-        " ",
-        "Seed: ",
-        selectedRun.meta.seed
-      ] }),
-      /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { children: " " }),
-      /* @__PURE__ */ jsxRuntime.jsx(SectionHeader, { children: "Overall" }),
-      /* @__PURE__ */ jsxRuntime.jsx(
-        TextBar,
-        {
-          label: "pass rate",
-          value: selectedRun.performance.passRate,
-          format: (v) => `${v}%`
-        }
-      ),
-      /* @__PURE__ */ jsxRuntime.jsx(
-        TextBar,
-        {
-          label: "avg score",
-          value: Math.round(selectedRun.performance.avgScore * 100)
-        }
-      ),
-      /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { children: " " }),
-      /* @__PURE__ */ jsxRuntime.jsx(SectionHeader, { children: "Dimensions" }),
-      selectedRun.dimensions.map((dimension) => /* @__PURE__ */ jsxRuntime.jsx(
-        TextBar,
-        {
-          label: dimension.name,
-          value: dimension.score
-        },
-        dimension.name
-      )),
-      /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { children: " " }),
-      /* @__PURE__ */ jsxRuntime.jsx(SectionHeader, { children: "Latency trend" }),
-      /* @__PURE__ */ jsxRuntime.jsx(
-        Sparkline,
-        {
-          data: selectedRun.performance.latencyHistoryMs ?? [
-            selectedRun.performance.latencyAvgMs - 40,
-            selectedRun.performance.latencyAvgMs - 10,
-            selectedRun.performance.latencyAvgMs + 20,
-            selectedRun.performance.latencyP95Ms - 80,
-            selectedRun.performance.latencyP95Ms
-          ],
-          width: 24
-        }
-      )
-    ] }) })
-  ] });
-}
-
-// src/evals/metric.ts
-var registry = /* @__PURE__ */ new Map();
-var Metric = {
-  of(config) {
-    const def = {
-      id: config.id,
-      name: config.name,
-      aggregate: config.aggregate,
-      format: config.format,
-      make: (data) => ({ id: config.id, data })
-    };
-    registry.set(config.id, def);
-    return def;
-  }
-};
-function getMetricById(id) {
-  return registry.get(id);
-}
-
-// src/evals/score.ts
-var registry2 = /* @__PURE__ */ new Map();
-var Score = {
-  of(config) {
-    const def = {
-      id: config.id,
-      name: config.name,
-      displayStrategy: config.displayStrategy,
-      aggregate: config.aggregate,
-      format: config.format,
-      make: (data, options) => {
-        const passed = options?.definePassed !== void 0 ? options.definePassed(data) : void 0;
-        return {
-          id: config.id,
-          data,
-          ...passed !== void 0 && { passed }
-        };
-      }
-    };
-    registry2.set(config.id, def);
-    return def;
-  }
-};
-function getScoreById(id) {
-  return registry2.get(id);
-}
-
-// src/evals/aggregators.ts
-function aggregateAverage(values) {
-  if (values.length === 0) {
-    return { value: 0 };
-  }
-  const sum = values.reduce((s, v) => s + v.value, 0);
-  return { value: sum / values.length };
-}
-function aggregateAll(values) {
-  return { passed: values.length > 0 && values.every((v) => v.passed) };
-}
-function aggregateTokenCountSum(values) {
-  const initial = {
-    input: 0,
-    output: 0,
-    inputCached: 0,
-    outputCached: 0
-  };
-  return values.reduce(
-    (acc, v) => ({
-      input: acc.input + (v.input ?? 0),
-      output: acc.output + (v.output ?? 0),
-      inputCached: acc.inputCached + (v.inputCached ?? 0),
-      outputCached: acc.outputCached + (v.outputCached ?? 0)
-    }),
-    initial
-  );
-}
-function aggregateLatencyAverage(values) {
-  if (values.length === 0) {
-    return { ms: 0 };
-  }
-  const sum = values.reduce((s, v) => s + v.ms, 0);
-  return { ms: sum / values.length };
-}
-
-// src/evals/metrics/standard.ts
-Metric.of({
-  id: "token-count",
-  name: "Tokens",
-  aggregate: aggregateTokenCountSum,
-  format: (data, options) => {
-    const input = data.input ?? 0;
-    const output = data.output ?? 0;
-    const inputCached = data.inputCached ?? 0;
-    const outputCached = data.outputCached ?? 0;
-    const cached = inputCached + outputCached;
-    const base = `in:${input} out:${output} cached:${cached}`;
-    return options?.isAggregated ? `Total: ${base}` : base;
-  }
-});
-Metric.of({
-  id: "latency",
-  name: "Latency",
-  aggregate: aggregateLatencyAverage,
-  format: (data, options) => options?.isAggregated ? `Avg: ${data.ms}ms` : `${data.ms}ms`
-});
-
-// src/evals/scores/standard.ts
-Score.of({
-  id: "percent",
-  name: "Score",
-  displayStrategy: "bar",
-  format: (data, options) => options?.isAggregated ? `Avg: ${data.value.toFixed(2)}` : data.value.toFixed(2),
-  aggregate: aggregateAverage
-});
-Score.of({
-  id: "binary",
-  name: "Result",
-  displayStrategy: "passFail",
-  format: (data, options) => options?.isAggregated ? data.passed ? "All: PASSED" : "Some: FAILED" : data.passed ? "PASSED" : "NOT PASSED",
-  aggregate: aggregateAll
-});
-function createDiffLogEntry(expected, actual, options) {
-  const diff = jsonDiff.diffString(expected, actual, { color: false });
-  return {
-    type: "diff",
-    label: options?.label,
-    expected,
-    actual,
-    diff: diff || "(no differences)"
-  };
-}
-function getDiffLines(entry) {
-  const raw = jsonDiff.diffString(entry.expected, entry.actual, { color: false }) || "(no differences)";
-  return raw.split("\n").map((line) => {
-    const trimmed = line.trimStart();
-    if (trimmed.startsWith("-") && !trimmed.startsWith("---")) {
-      return { type: "remove", line };
-    }
-    if (trimmed.startsWith("+") && !trimmed.startsWith("+++")) {
-      return { type: "add", line };
-    }
-    return { type: "context", line };
-  });
-}
-
-// src/runner/score-utils.ts
-function toNumericScoreFromScores(scores) {
-  for (const item of scores) {
-    const def = getScoreById(item.id);
-    if (def && def.displayStrategy === "bar" && typeof item.data === "object" && item.data !== null && "value" in item.data) {
-      const value = item.data.value;
-      if (typeof value === "number" && Number.isFinite(value)) {
-        return value;
-      }
-    }
-    const numeric = toNumericScore(item.data);
-    if (numeric !== void 0) {
-      return numeric;
-    }
-  }
-  return void 0;
-}
-function toNumericScore(value) {
-  if (typeof value === "number" && Number.isFinite(value)) {
-    return value;
-  }
-  if (typeof value !== "object" || value === null) {
-    return void 0;
-  }
-  const obj = value;
-  if ("score" in obj && typeof obj.score === "number" && Number.isFinite(obj.score)) {
-    return obj.score;
-  }
-  const numberValues = Object.values(value).filter(
-    (entry) => typeof entry === "number" && Number.isFinite(entry)
-  );
-  if (numberValues.length === 0) {
-    return void 0;
-  }
-  return numberValues.reduce((sum, entry) => sum + entry, 0) / numberValues.length;
-}
 
 // src/runner/config.ts
 var defaultRunnerConfig = {
@@ -1213,75 +935,311 @@ async function loadModuleExports(filePath) {
       if (!createJiti2) {
         throw new Error("Failed to initialize jiti TypeScript loader");
       }
-      jitiLoader = createJiti2((typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('out.js', document.baseURI).href)), {
-        interopDefault: true,
-        moduleCache: true
-      });
+      jitiLoader = createJiti2((typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('out.js', document.baseURI).href)), {
+        interopDefault: true,
+        moduleCache: true
+      });
+    }
+    const loaded2 = jitiLoader.import ? await jitiLoader.import(filePath) : await Promise.resolve(jitiLoader(filePath));
+    return Object.values(loaded2);
+  }
+  const moduleUrl = url.pathToFileURL(filePath).href;
+  const loaded = await import(moduleUrl);
+  return Object.values(loaded);
+}
+async function collectDatasetsFromFiles(config) {
+  const files = await walkDirectory(config.rootDir, config.excludeDirectories);
+  const matched = files.filter(
+    (filePath) => hasOneSuffix(filePath, config.datasetSuffixes)
+  );
+  const found = await Promise.all(
+    matched.map(async (absolutePath) => {
+      const exports = await loadModuleExports(absolutePath);
+      const datasets = exports.filter(isDatasetLike);
+      const relPath = path.relative(config.rootDir, absolutePath);
+      return datasets.map((dataset) => ({
+        id: toId("dataset", relPath, dataset.getName()),
+        filePath: relPath,
+        dataset
+      }));
+    })
+  );
+  return found.flat();
+}
+async function collectEvaluatorsFromFiles(config) {
+  const files = await walkDirectory(config.rootDir, config.excludeDirectories);
+  const matched = files.filter(
+    (filePath) => hasOneSuffix(filePath, config.evaluatorSuffixes)
+  );
+  const found = await Promise.all(
+    matched.map(async (absolutePath) => {
+      const exports = await loadModuleExports(absolutePath);
+      const evaluators = exports.filter(isEvaluatorLike);
+      const relPath = path.relative(config.rootDir, absolutePath);
+      return evaluators.map((evaluator) => ({
+        id: toId("evaluator", relPath, evaluator.getName()),
+        filePath: relPath,
+        evaluator
+      }));
+    })
+  );
+  return found.flat();
+}
+async function collectTestCasesFromFiles(config) {
+  const files = await walkDirectory(config.rootDir, config.excludeDirectories);
+  const matched = files.filter(
+    (filePath) => hasOneSuffix(filePath, config.testCaseSuffixes)
+  );
+  const found = await Promise.all(
+    matched.map(async (absolutePath) => {
+      const exports = await loadModuleExports(absolutePath);
+      const testCases = exports.filter(isTestCaseLike);
+      const relPath = path.relative(config.rootDir, absolutePath);
+      return testCases.map((testCase) => ({
+        id: toId("test-case", relPath, testCase.getName()),
+        filePath: relPath,
+        testCase
+      }));
+    })
+  );
+  return found.flat();
+}
+function toJsonLines(value) {
+  try {
+    return JSON.stringify(value, null, 2);
+  } catch {
+    return String(value);
+  }
+}
+function formatDiffString(changes) {
+  const lines = [];
+  for (const part of changes) {
+    const prefix = part.added ? "+" : part.removed ? "-" : " ";
+    const partLines = part.value.split("\n");
+    if (partLines[partLines.length - 1] === "") {
+      partLines.pop();
+    }
+    for (const line of partLines) {
+      lines.push(`${prefix} ${line}`);
+    }
+  }
+  return lines.join("\n");
+}
+function createDiffString(expected, actual) {
+  const expectedStr = toJsonLines(expected);
+  const actualStr = toJsonLines(actual);
+  const changes = diff.diffLines(expectedStr, actualStr);
+  return formatDiffString(changes);
+}
+function createDiffLogEntry(expected, actual, options) {
+  const diff = createDiffString(expected, actual);
+  return {
+    type: "diff",
+    label: options?.label,
+    expected,
+    actual,
+    diff: diff || "(no differences)"
+  };
+}
+function getDiffLines(entry) {
+  const raw = createDiffString(entry.expected, entry.actual) || "(no differences)";
+  return raw.split("\n").map((line) => {
+    const trimmed = line.trimStart();
+    if (trimmed.startsWith("-") && !trimmed.startsWith("---")) {
+      return { type: "remove", line };
+    }
+    if (trimmed.startsWith("+") && !trimmed.startsWith("+++")) {
+      return { type: "add", line };
+    }
+    return { type: "context", line };
+  });
+}
+
+// src/evals/metric.ts
+var registry = /* @__PURE__ */ new Map();
+var Metric = {
+  of(config) {
+    const def = {
+      id: config.id,
+      name: config.name,
+      aggregate: config.aggregate,
+      format: config.format,
+      make: (data) => ({ id: config.id, data })
+    };
+    registry.set(config.id, def);
+    return def;
+  }
+};
+function getMetricById(id) {
+  return registry.get(id);
+}
+
+// src/evals/score.ts
+var registry2 = /* @__PURE__ */ new Map();
+var Score = {
+  of(config) {
+    const def = {
+      id: config.id,
+      name: config.name,
+      displayStrategy: config.displayStrategy,
+      aggregate: config.aggregate,
+      format: config.format,
+      make: (data, options) => {
+        const passed = options?.definePassed !== void 0 ? options.definePassed(data) : void 0;
+        return {
+          id: config.id,
+          data,
+          ...passed !== void 0 && { passed }
+        };
+      }
+    };
+    registry2.set(config.id, def);
+    return def;
+  }
+};
+function getScoreById(id) {
+  return registry2.get(id);
+}
+
+// src/evals/aggregators.ts
+function aggregateAverageWithVariance(values) {
+  if (values.length === 0) {
+    return { value: 0, count: 0 };
+  }
+  const sum = values.reduce((s, v) => s + v.value, 0);
+  const sumSq = values.reduce((s, v) => s + v.value * v.value, 0);
+  const mean = sum / values.length;
+  let stdDev;
+  if (values.length >= 2) {
+    const variance = (sumSq - values.length * mean * mean) / (values.length - 1);
+    stdDev = variance > 0 ? Math.sqrt(variance) : 0;
+  }
+  return { value: mean, stdDev, count: values.length };
+}
+function aggregateAll(values) {
+  const total = values.length;
+  const passedCount = values.filter((v) => v.passed).length;
+  return {
+    passed: total > 0 && values.every((v) => v.passed),
+    passedCount,
+    totalCount: total
+  };
+}
+function aggregateTokenCountSum(values) {
+  const initial = {
+    input: 0,
+    output: 0,
+    inputCached: 0,
+    outputCached: 0
+  };
+  return values.reduce(
+    (acc, v) => ({
+      input: acc.input + (v.input ?? 0),
+      output: acc.output + (v.output ?? 0),
+      inputCached: acc.inputCached + (v.inputCached ?? 0),
+      outputCached: acc.outputCached + (v.outputCached ?? 0)
+    }),
+    initial
+  );
+}
+function aggregateLatencyAverage(values) {
+  if (values.length === 0) {
+    return { ms: 0 };
+  }
+  const sum = values.reduce((s, v) => s + v.ms, 0);
+  return { ms: sum / values.length };
+}
+
+// src/evals/metrics/standard.ts
+Metric.of({
+  id: "token-count",
+  name: "Tokens",
+  aggregate: aggregateTokenCountSum,
+  format: (data, options) => {
+    const input = data.input ?? 0;
+    const output = data.output ?? 0;
+    const inputCached = data.inputCached ?? 0;
+    const outputCached = data.outputCached ?? 0;
+    const cached = inputCached + outputCached;
+    const base = `in:${input} out:${output} cached:${cached}`;
+    return options?.isAggregated ? `Total: ${base}` : base;
+  }
+});
+Metric.of({
+  id: "latency",
+  name: "Latency",
+  aggregate: aggregateLatencyAverage,
+  format: (data, options) => options?.isAggregated ? `Avg: ${data.ms}ms` : `${data.ms}ms`
+});
+
+// src/evals/scores/standard.ts
+Score.of({
+  id: "percent",
+  name: "Score",
+  displayStrategy: "bar",
+  format: (data, options) => {
+    if (options?.isAggregated) {
+      return data.stdDev != null ? `Avg: ${data.value.toFixed(2)} \xB1 ${data.stdDev.toFixed(2)}` : `Avg: ${data.value.toFixed(2)}`;
+    }
+    return data.value.toFixed(2);
+  },
+  aggregate: aggregateAverageWithVariance
+});
+Score.of({
+  id: "binary",
+  name: "Result",
+  displayStrategy: "passFail",
+  format: (data, options) => {
+    if (options?.isAggregated) {
+      const base = data.passed ? "All: PASSED" : "Some: FAILED";
+      if (data.passedCount != null && data.totalCount != null && data.totalCount > 1) {
+        return `${base} (${data.passedCount}/${data.totalCount})`;
+      }
+      return base;
+    }
+    return data.passed ? "PASSED" : "NOT PASSED";
+  },
+  aggregate: aggregateAll
+});
+
+// src/runner/score-utils.ts
+function toNumericScoreFromScores(scores) {
+  for (const item of scores) {
+    const def = getScoreById(item.id);
+    if (def && def.displayStrategy === "bar" && typeof item.data === "object" && item.data !== null && "value" in item.data) {
+      const value = item.data.value;
+      if (typeof value === "number" && Number.isFinite(value)) {
+        return value;
+      }
+    }
+    const numeric = toNumericScore(item.data);
+    if (numeric !== void 0) {
+      return numeric;
     }
-    const loaded2 = jitiLoader.import ? await jitiLoader.import(filePath) : await Promise.resolve(jitiLoader(filePath));
-    return Object.values(loaded2);
   }
-  const moduleUrl = url.pathToFileURL(filePath).href;
-  const loaded = await import(moduleUrl);
-  return Object.values(loaded);
-}
-async function collectDatasetsFromFiles(config) {
-  const files = await walkDirectory(config.rootDir, config.excludeDirectories);
-  const matched = files.filter(
-    (filePath) => hasOneSuffix(filePath, config.datasetSuffixes)
-  );
-  const found = await Promise.all(
-    matched.map(async (absolutePath) => {
-      const exports = await loadModuleExports(absolutePath);
-      const datasets = exports.filter(isDatasetLike);
-      const relPath = path.relative(config.rootDir, absolutePath);
-      return datasets.map((dataset) => ({
-        id: toId("dataset", relPath, dataset.getName()),
-        filePath: relPath,
-        dataset
-      }));
-    })
-  );
-  return found.flat();
-}
-async function collectEvaluatorsFromFiles(config) {
-  const files = await walkDirectory(config.rootDir, config.excludeDirectories);
-  const matched = files.filter(
-    (filePath) => hasOneSuffix(filePath, config.evaluatorSuffixes)
-  );
-  const found = await Promise.all(
-    matched.map(async (absolutePath) => {
-      const exports = await loadModuleExports(absolutePath);
-      const evaluators = exports.filter(isEvaluatorLike);
-      const relPath = path.relative(config.rootDir, absolutePath);
-      return evaluators.map((evaluator) => ({
-        id: toId("evaluator", relPath, evaluator.getName()),
-        filePath: relPath,
-        evaluator
-      }));
-    })
-  );
-  return found.flat();
+  return void 0;
 }
-async function collectTestCasesFromFiles(config) {
-  const files = await walkDirectory(config.rootDir, config.excludeDirectories);
-  const matched = files.filter(
-    (filePath) => hasOneSuffix(filePath, config.testCaseSuffixes)
-  );
-  const found = await Promise.all(
-    matched.map(async (absolutePath) => {
-      const exports = await loadModuleExports(absolutePath);
-      const testCases = exports.filter(isTestCaseLike);
-      const relPath = path.relative(config.rootDir, absolutePath);
-      return testCases.map((testCase) => ({
-        id: toId("test-case", relPath, testCase.getName()),
-        filePath: relPath,
-        testCase
-      }));
-    })
+function toNumericScore(value) {
+  if (typeof value === "number" && Number.isFinite(value)) {
+    return value;
+  }
+  if (typeof value !== "object" || value === null) {
+    return void 0;
+  }
+  const obj = value;
+  if ("score" in obj && typeof obj.score === "number" && Number.isFinite(obj.score)) {
+    return obj.score;
+  }
+  const numberValues = Object.values(value).filter(
+    (entry) => typeof entry === "number" && Number.isFinite(entry)
   );
-  return found.flat();
+  if (numberValues.length === 0) {
+    return void 0;
+  }
+  return numberValues.reduce((sum, entry) => sum + entry, 0) / numberValues.length;
 }
+
+// src/runner/execution.ts
 function computeEvaluatorPassed(evaluator, result, scores) {
   const scoresWithPassed = scores.filter((s) => "passed" in s && s.passed !== void 0);
   if (scoresWithPassed.length > 0) {
@@ -1952,6 +1910,240 @@ var EffectRunner = class {
     );
   }
 };
+var LEFT_PANE_WIDTH2 = 44;
+var MAX_RUNS_FOR_CHART = 12;
+var MAX_RUNS_FOR_TREND = 20;
+var TREND_BATCH_SIZE = 4;
+function extractRunAverageScore(testCases) {
+  const scores = [];
+  for (const tc of testCases) {
+    for (const es of tc.evaluatorScores) {
+      const n = toNumericScoreFromScores(es.scores);
+      if (n !== void 0) {
+        scores.push(n);
+      }
+    }
+  }
+  if (scores.length === 0)
+    return void 0;
+  return scores.reduce((a, b) => a + b, 0) / scores.length;
+}
+async function loadRunScores(runs) {
+  const results = [];
+  for (const run of runs) {
+    const artifact = run.meta?.artifact;
+    if (!artifact)
+      continue;
+    try {
+      const path$1 = path.resolve(artifact);
+      const testCases = await parseArtifactFile(path$1);
+      const avg = extractRunAverageScore(testCases);
+      if (avg !== void 0) {
+        results.push({
+          runId: run.id,
+          label: run.label,
+          value: avg
+        });
+      }
+    } catch {
+    }
+  }
+  return results;
+}
+function batchAverage(values, batchSize) {
+  const batches = [];
+  for (let i = 0; i < values.length; i += batchSize) {
+    const slice = values.slice(i, i + batchSize);
+    if (slice.length > 0) {
+      batches.push(slice.reduce((a, b) => a + b, 0) / slice.length);
+    }
+  }
+  return batches;
+}
+var OVERVIEW_PAGE_SIZE = 15;
+function DatasetsView({
+  state,
+  filteredDatasets,
+  selectedDataset,
+  overviewRowCountRef
+}) {
+  const leftFocused = state.focus === "left";
+  const rightFocused = state.focus === "right";
+  const [runScores, setRunScores] = React2.useState([]);
+  const [loading, setLoading] = React2.useState(false);
+  React2.useEffect(() => {
+    if (!selectedDataset?.runs?.length) {
+      setRunScores([]);
+      return;
+    }
+    setLoading(true);
+    const runs = selectedDataset.runs.slice(0, MAX_RUNS_FOR_TREND);
+    loadRunScores(runs).then(setRunScores).finally(() => setLoading(false));
+  }, [selectedDataset?.id, selectedDataset?.runs?.length]);
+  const barData = runScores.slice(0, MAX_RUNS_FOR_CHART).reverse();
+  const trendValues = runScores.slice(0, MAX_RUNS_FOR_TREND).map((r) => r.value).reverse();
+  const trendBatched = batchAverage(trendValues, TREND_BATCH_SIZE);
+  const overviewRows = React2.useMemo(() => {
+    const rows = [];
+    rows.push(
+      /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "gray", children: selectedDataset?.overview ?? "Select a dataset to inspect prior runs." }, "overview")
+    );
+    if (selectedDataset && selectedDataset.runs.length > 0) {
+      if (loading) {
+        rows.push(
+          /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "gray", children: "Loading run scores\u2026" }, "loading")
+        );
+      } else if (runScores.length > 0) {
+        rows.push(
+          /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "gray", children: "Scores (last runs)" }, "scores-header")
+        );
+        for (const d of barData) {
+          rows.push(
+            /* @__PURE__ */ jsxRuntime.jsx(
+              TextBar,
+              {
+                label: d.label,
+                value: d.value,
+                labelWidth: 14,
+                barWidth: 24,
+                max: 100,
+                format: (v) => v.toFixed(1)
+              },
+              d.runId
+            )
+          );
+        }
+        if (trendBatched.length > 0) {
+          rows.push(
+            /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "gray", children: "Avg trend (last 20, batched by 4)" }, "trend-header")
+          );
+          rows.push(
+            /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { children: /* @__PURE__ */ jsxRuntime.jsx(
+              inkChart.LineGraph,
+              {
+                data: [{ values: trendBatched, color: "cyan" }],
+                height: 5,
+                width: 45,
+                showYAxis: true,
+                xLabels: ["older", "newer"]
+              }
+            ) }, "trend-graph")
+          );
+        }
+      }
+    }
+    return rows;
+  }, [
+    selectedDataset?.overview,
+    selectedDataset?.runs?.length,
+    loading,
+    runScores,
+    barData,
+    trendBatched
+  ]);
+  if (overviewRowCountRef) {
+    overviewRowCountRef.current = overviewRows.length;
+  }
+  const offset = Math.max(0, state.overviewScrollOffset);
+  const visibleRows = overviewRows.slice(offset, offset + OVERVIEW_PAGE_SIZE);
+  return /* @__PURE__ */ jsxRuntime.jsxs(jsxRuntime.Fragment, { children: [
+    /* @__PURE__ */ jsxRuntime.jsxs(Pane, { width: LEFT_PANE_WIDTH2, focused: leftFocused, children: [
+      /* @__PURE__ */ jsxRuntime.jsx(SectionHeader, { children: "Datasets" }),
+      /* @__PURE__ */ jsxRuntime.jsx(
+        ListItem,
+        {
+          selected: state.datasetMenuIndex === 0,
+          label: "New evaluation",
+          itemKey: "datasets-new-eval"
+        }
+      ),
+      filteredDatasets.map((dataset, index) => /* @__PURE__ */ jsxRuntime.jsx(
+        ListItem,
+        {
+          selected: state.datasetMenuIndex === index + 1,
+          label: dataset.name,
+          itemKey: `dataset-${dataset.id}`
+        },
+        dataset.id
+      ))
+    ] }),
+    /* @__PURE__ */ jsxRuntime.jsxs(Pane, { flexGrow: 1, marginLeft: 1, focused: rightFocused, children: [
+      /* @__PURE__ */ jsxRuntime.jsx(SectionHeader, { children: "Overview" }),
+      /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { flexDirection: "column", children: visibleRows.map((row, i) => /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { children: row }, offset + i)) })
+    ] })
+  ] });
+}
+function RunsView({
+  state,
+  dataset,
+  selectedRun
+}) {
+  const runs = dataset?.runs ?? [];
+  const rightFocused = state.focus === "right";
+  return /* @__PURE__ */ jsxRuntime.jsxs(jsxRuntime.Fragment, { children: [
+    /* @__PURE__ */ jsxRuntime.jsx(RunsSidebar, { state, dataset, runs }),
+    /* @__PURE__ */ jsxRuntime.jsx(Pane, { flexGrow: 1, marginLeft: 1, focused: rightFocused, children: !selectedRun ? /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "gray", children: "Select a run to see summary metrics." }) : /* @__PURE__ */ jsxRuntime.jsxs(ink.Box, { flexDirection: "column", children: [
+      /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { children: [
+        /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "gray", children: "Run:" }),
+        " ",
+        selectedRun.label,
+        " ",
+        /* @__PURE__ */ jsxRuntime.jsx(StatusText, { status: selectedRun.status })
+      ] }),
+      /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
+        "Commit: ",
+        selectedRun.meta.commit,
+        "  Branch: ",
+        selectedRun.meta.branch,
+        " ",
+        "Seed: ",
+        selectedRun.meta.seed
+      ] }),
+      /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { children: " " }),
+      /* @__PURE__ */ jsxRuntime.jsx(SectionHeader, { children: "Overall" }),
+      /* @__PURE__ */ jsxRuntime.jsx(
+        TextBar,
+        {
+          label: "pass rate",
+          value: selectedRun.performance.passRate,
+          format: (v) => `${v}%`
+        }
+      ),
+      /* @__PURE__ */ jsxRuntime.jsx(
+        TextBar,
+        {
+          label: "avg score",
+          value: Math.round(selectedRun.performance.avgScore * 100)
+        }
+      ),
+      /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { children: " " }),
+      /* @__PURE__ */ jsxRuntime.jsx(SectionHeader, { children: "Dimensions" }),
+      selectedRun.dimensions.map((dimension) => /* @__PURE__ */ jsxRuntime.jsx(
+        TextBar,
+        {
+          label: dimension.name,
+          value: dimension.score
+        },
+        dimension.name
+      )),
+      /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { children: " " }),
+      /* @__PURE__ */ jsxRuntime.jsx(SectionHeader, { children: "Latency trend" }),
+      /* @__PURE__ */ jsxRuntime.jsx(
+        Sparkline,
+        {
+          data: selectedRun.performance.latencyHistoryMs ?? [
+            selectedRun.performance.latencyAvgMs - 40,
+            selectedRun.performance.latencyAvgMs - 10,
+            selectedRun.performance.latencyAvgMs + 20,
+            selectedRun.performance.latencyP95Ms - 80,
+            selectedRun.performance.latencyP95Ms
+          ],
+          width: 24
+        }
+      )
+    ] }) })
+  ] });
+}
 var DETAILS_PAGE_SIZE = 20;
 function scoreColor(score) {
   if (score >= 80)
@@ -1960,7 +2152,7 @@ function scoreColor(score) {
     return "yellow";
   return "red";
 }
-function formatScorePart(item, scoreToColor) {
+function formatScorePart(item) {
   const def = getScoreById(item.id);
   if (!def) {
     const numeric = toNumericScore(item.data);
@@ -1990,7 +2182,7 @@ function CheckRow({
     " ",
     /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color, bold: true, children: status }),
     detail ? /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
-      "  (",
+      " (",
       detail,
       ")"
     ] }) : null
@@ -2010,21 +2202,21 @@ function buildDetailRows(run, testCases, evaluatorNameById) {
     /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
       "Model: ",
       meta.model,
-      "   Provider: ",
+      " Provider: ",
       meta.provider
     ] }, "meta-1"),
     /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
       "Commit: ",
       meta.commit,
-      "  Branch: ",
+      " Branch: ",
       meta.branch,
-      "  Seed: ",
+      " Seed: ",
       meta.seed
     ] }, "meta-2"),
     /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
       "Duration: ",
       meta.duration,
-      "   Concurrency: ",
+      " Concurrency: ",
       meta.concurrency
     ] }, "meta-3"),
     /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
@@ -2036,7 +2228,15 @@ function buildDetailRows(run, testCases, evaluatorNameById) {
     ...dimensions.map((d) => /* @__PURE__ */ jsxRuntime.jsx(TextBar, { label: d.name, value: d.score }, `dim-${d.name}`)),
     /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { children: " " }, "sp2"),
     /* @__PURE__ */ jsxRuntime.jsx(SectionHeader, { children: "Checks (boolean)" }, "checks-h"),
-    ...checks.map((c) => /* @__PURE__ */ jsxRuntime.jsx(CheckRow, { name: c.name, passed: c.passed, detail: c.detail }, `chk-${c.name}`)),
+    ...checks.map((c) => /* @__PURE__ */ jsxRuntime.jsx(
+      CheckRow,
+      {
+        name: c.name,
+        passed: c.passed,
+        detail: c.detail
+      },
+      `chk-${c.name}`
+    )),
     /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { children: " " }, "sp3"),
     /* @__PURE__ */ jsxRuntime.jsx(SectionHeader, { children: "Performance" }, "perf-h"),
     /* @__PURE__ */ jsxRuntime.jsx(
@@ -2049,16 +2249,16 @@ function buildDetailRows(run, testCases, evaluatorNameById) {
       "perf-rate"
     ),
     /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
-      "latency avg     ",
+      "latency avg ",
       performance.latencyAvgMs,
-      "ms   p95 ",
+      "ms p95 ",
       performance.latencyP95Ms,
       "ms"
     ] }, "perf-lat"),
     /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
-      "tokens avg      ",
+      "tokens avg ",
       performance.tokensAvg,
-      "   p95 ",
+      " p95 ",
       performance.tokensP95
     ] }, "perf-tok"),
     /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { children: " " }, "sp4"),
@@ -2111,26 +2311,60 @@ function buildDetailRows(run, testCases, evaluatorNameById) {
             ":",
             " ",
             /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: item.passed ? "green" : "red", bold: true, children: item.passed ? "PASS" : "FAIL" }),
-            " ",
-            item.scores.map((s) => /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: scoreColor(toNumericScore(s.data) ?? 0), children: [
-              formatScorePart(s),
-              " "
-            ] }, s.id)),
-            item.metrics?.map((m) => {
-              const def = getMetricById(m.id);
-              if (!def)
-                return null;
-              const formatted = def.format(m.data);
-              return /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
-                "[",
-                def.name ? `${def.name}: ` : "",
-                formatted,
-                "]",
-                " "
-              ] }, m.id);
-            })
+            item.metrics && item.metrics.length > 0 ? /* @__PURE__ */ jsxRuntime.jsxs(jsxRuntime.Fragment, { children: [
+              " ",
+              item.metrics.map((m) => {
+                const def = getMetricById(m.id);
+                if (!def)
+                  return null;
+                const formatted = def.format(m.data);
+                return /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
+                  "[",
+                  def.name ? `${def.name}: ` : "",
+                  formatted,
+                  "]",
+                  " "
+                ] }, m.id);
+              })
+            ] }) : null
           ] }, `tc-${tc.testCaseId}-${item.evaluatorId}`)
         );
+        if (item.scores.length > 0) {
+          for (let sIdx = 0; sIdx < item.scores.length; sIdx++) {
+            const s = item.scores[sIdx];
+            const def = getScoreById(s.id);
+            const scoreLabel = def ? def.name ?? def.id : s.id;
+            rows.push(
+              /* @__PURE__ */ jsxRuntime.jsxs(
+                ink.Text,
+                {
+                  color: scoreColor(toNumericScore(s.data) ?? 0),
+                  children: [
+                    "      ",
+                    scoreLabel,
+                    ": ",
+                    formatScorePart(s)
+                  ]
+                },
+                `tc-${tc.testCaseId}-${item.evaluatorId}-score-${sIdx}`
+              )
+            );
+          }
+        } else {
+          rows.push(
+            /* @__PURE__ */ jsxRuntime.jsxs(
+              ink.Text,
+              {
+                color: "gray",
+                children: [
+                  "      ",
+                  "n/a"
+                ]
+              },
+              `tc-${tc.testCaseId}-${item.evaluatorId}-n/a`
+            )
+          );
+        }
         if (!item.passed && item.logs && item.logs.length > 0) {
           for (let logIdx = 0; logIdx < item.logs.length; logIdx++) {
             const log = item.logs[logIdx];
@@ -2168,12 +2402,12 @@ function RunDetailsView({
 }) {
   const runs = dataset?.runs ?? [];
   const rightFocused = state.focus === "right";
-  const [testCases, setTestCases] = React.useState([]);
-  const evaluatorNameById = React__default.default.useMemo(
+  const [testCases, setTestCases] = React2.useState([]);
+  const evaluatorNameById = React2__default.default.useMemo(
     () => new Map(evaluators.map((e) => [e.id, e.name])),
     [evaluators]
   );
-  React.useEffect(() => {
+  React2.useEffect(() => {
     if (!selectedRun?.meta?.artifact) {
       setTestCases([]);
       return;
@@ -2192,7 +2426,7 @@ function RunDetailsView({
   const visible = rows.slice(offset, offset + DETAILS_PAGE_SIZE);
   return /* @__PURE__ */ jsxRuntime.jsxs(jsxRuntime.Fragment, { children: [
     /* @__PURE__ */ jsxRuntime.jsx(RunsSidebar, { state, dataset, runs }),
-    /* @__PURE__ */ jsxRuntime.jsx(Pane, { flexGrow: 1, marginLeft: 1, focused: rightFocused, children: /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { flexDirection: "column", children: visible.map((row, i) => /* @__PURE__ */ jsxRuntime.jsx(React__default.default.Fragment, { children: row }, i)) }) })
+    /* @__PURE__ */ jsxRuntime.jsx(Pane, { flexGrow: 1, marginLeft: 1, focused: rightFocused, children: /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { flexDirection: "column", children: visible.map((row, i) => /* @__PURE__ */ jsxRuntime.jsx(React2__default.default.Fragment, { children: row }, i)) }) })
   ] });
 }
 var LEFT_PANE_WIDTH3 = 44;
@@ -2272,16 +2506,17 @@ function EvalsCliApp({
 }) {
   const { exit } = ink.useApp();
   const { width: stdoutWidth, height: stdoutHeight } = fullscreenInk.useScreenSize();
-  const [liveData, setLiveData] = React.useState(data);
-  const [runtimeMessage, setRuntimeMessage] = React.useState();
-  const [state, dispatch] = React.useReducer(
+  const [liveData, setLiveData] = React2.useState(data);
+  const [runtimeMessage, setRuntimeMessage] = React2.useState();
+  const overviewRowCountRef = React2.useRef(0);
+  const [state, dispatch] = React2.useReducer(
     reduceCliState,
     createInitialState(data, args)
   );
-  React.useEffect(() => {
+  React2.useEffect(() => {
     setLiveData(data);
   }, [data]);
-  React.useEffect(() => {
+  React2.useEffect(() => {
     if (!runner) {
       return void 0;
     }
@@ -2300,7 +2535,7 @@ function EvalsCliApp({
       }
     });
   }, [runner]);
-  const filteredDatasets = React.useMemo(
+  const filteredDatasets = React2.useMemo(
     () => getFilteredDatasets(liveData, state.searchQuery),
     [liveData, state.searchQuery]
   );
@@ -2353,7 +2588,16 @@ function EvalsCliApp({
       return;
     }
     if (key.downArrow) {
-      const max = clampedState.level === "datasets" ? filteredDatasets.length : clampedState.level === "runs" ? selectedDataset?.runs.length ?? 0 : clampedState.level === "new-evaluation" ? Math.max(0, visibleEvaluators.length - 1) : 100;
+      let max;
+      if (clampedState.level === "datasets") {
+        max = clampedState.focus === "right" ? Math.max(0, overviewRowCountRef.current - OVERVIEW_PAGE_SIZE) : filteredDatasets.length;
+      } else if (clampedState.level === "runs") {
+        max = selectedDataset?.runs.length ?? 0;
+      } else if (clampedState.level === "new-evaluation") {
+        max = Math.max(0, visibleEvaluators.length - 1);
+      } else {
+        max = 100;
+      }
       dispatch({ type: "MOVE_DOWN", max });
       return;
     }
@@ -2371,7 +2615,7 @@ function EvalsCliApp({
       }
       return;
     }
-    if (isBackKey(key)) {
+    if (isBackKey(key) || input === "\x7F" || input === "\b") {
       dispatch({ type: "BACK" });
       return;
     }
@@ -2424,7 +2668,8 @@ function EvalsCliApp({
         {
           state: clampedState,
           filteredDatasets,
-          selectedDataset
+          selectedDataset,
+          overviewRowCountRef
         }
       );
     }