@m4trix/evals 0.11.0 → 0.13.0

package/dist/cli.js

@@ -3,14 +3,14 @@ import { withFullScreen, useScreenSize } from 'fullscreen-ink';
 import React, { useState, useReducer, useEffect, useMemo } from 'react';
 import { useApp, useInput, Box, Text } from 'ink';
 import { jsx, jsxs, Fragment } from 'react/jsx-runtime';
+import { resolve, relative, join, dirname } from 'path';
+import { diffString } from 'json-diff';
 import { randomUUID } from 'crypto';
-import { Effect, PubSub, Queue, Fiber } from 'effect';
+import { Effect, PubSub, Queue, Fiber, Ref } from 'effect';
 import { existsSync } from 'fs';
-import { resolve, relative, join, dirname } from 'path';
 import * as jitiModule from 'jiti';
-import { mkdir, appendFile, readdir } from 'fs/promises';
+import { readdir, readFile, mkdir, appendFile } from 'fs/promises';
 import { pathToFileURL } from 'url';
-import { diffString } from 'json-diff';
 
 var SEP = "  ";
 var ARROW = "\u203A";
@@ -498,11 +498,17 @@ function toEvaluatorOption(item) {
   };
 }
 async function loadRunnerData(runner) {
-  const [datasets, evaluators] = await Promise.all([
+  const [datasets, evaluators, diskSnapshots] = await Promise.all([
     runner.collectDatasets(),
-    runner.collectEvaluators()
+    runner.collectEvaluators(),
+    runner.loadRunSnapshotsFromArtifacts()
   ]);
-  const snapshots = runner.getAllRunSnapshots();
+  const memSnapshots = runner.getAllRunSnapshots();
+  const seen = new Set(memSnapshots.map((s) => s.runId));
+  const fromDisk = diskSnapshots.filter((s) => !seen.has(s.runId));
+  const snapshots = [...memSnapshots, ...fromDisk].sort(
+    (a, b) => b.queuedAt - a.queuedAt
+  );
   if (datasets.length === 0 && evaluators.length === 0) {
     return loadMockData();
   }
@@ -820,458 +826,185 @@ function RunsView({
     ] }) })
   ] });
 }
-var DETAILS_PAGE_SIZE = 20;
-function CheckRow({
-  name,
-  passed,
-  detail
-}) {
-  const status = passed ? "PASSED" : "FAILED";
-  const color = passed ? "green" : "red";
-  return /* @__PURE__ */ jsxs(Text, { children: [
-    /* @__PURE__ */ jsx(Text, { color: "gray", children: name.padEnd(14) }),
-    " ",
-    /* @__PURE__ */ jsx(Text, { color, bold: true, children: status }),
-    detail ? /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
-      "  (",
-      detail,
-      ")"
-    ] }) : null
-  ] });
-}
-function buildDetailRows(run) {
-  const { performance, dimensions, checks, failures, meta } = run;
-  const latencyHistory = performance.latencyHistoryMs ?? [
-    performance.latencyAvgMs - 40,
-    performance.latencyAvgMs - 10,
-    performance.latencyAvgMs + 20,
-    performance.latencyP95Ms - 80,
-    performance.latencyP95Ms
-  ];
-  const rows = [
-    /* @__PURE__ */ jsx(SectionHeader, { children: "Meta" }, "meta-h"),
-    /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
-      "Model: ",
-      meta.model,
-      "   Provider: ",
-      meta.provider
-    ] }, "meta-1"),
-    /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
-      "Commit: ",
-      meta.commit,
-      "  Branch: ",
-      meta.branch,
-      "  Seed: ",
-      meta.seed
-    ] }, "meta-2"),
-    /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
-      "Duration: ",
-      meta.duration,
-      "   Concurrency: ",
-      meta.concurrency
-    ] }, "meta-3"),
-    /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
-      "Artifact: ",
-      meta.artifact
-    ] }, "meta-4"),
-    /* @__PURE__ */ jsx(Text, { children: " " }, "sp1"),
-    /* @__PURE__ */ jsx(SectionHeader, { children: "Scores (0\u2013100)" }, "scores-h"),
-    ...dimensions.map((d) => /* @__PURE__ */ jsx(TextBar, { label: d.name, value: d.score }, `dim-${d.name}`)),
-    /* @__PURE__ */ jsx(Text, { children: " " }, "sp2"),
-    /* @__PURE__ */ jsx(SectionHeader, { children: "Checks (boolean)" }, "checks-h"),
-    ...checks.map((c) => /* @__PURE__ */ jsx(CheckRow, { name: c.name, passed: c.passed, detail: c.detail }, `chk-${c.name}`)),
-    /* @__PURE__ */ jsx(Text, { children: " " }, "sp3"),
-    /* @__PURE__ */ jsx(SectionHeader, { children: "Performance" }, "perf-h"),
-    /* @__PURE__ */ jsx(
-      TextBar,
-      {
-        label: "pass rate",
-        value: performance.passRate,
-        format: (v) => `${v}%`
-      },
-      "perf-rate"
-    ),
-    /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
-      "latency avg     ",
-      performance.latencyAvgMs,
-      "ms   p95 ",
-      performance.latencyP95Ms,
-      "ms"
-    ] }, "perf-lat"),
-    /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
-      "tokens avg      ",
-      performance.tokensAvg,
-      "   p95 ",
-      performance.tokensP95
-    ] }, "perf-tok"),
-    /* @__PURE__ */ jsx(Text, { children: " " }, "sp4"),
-    /* @__PURE__ */ jsx(SectionHeader, { children: "Latency trend" }, "spark-h"),
-    /* @__PURE__ */ jsx(Sparkline, { data: latencyHistory, width: 20 }, "spark")
-  ];
-  if (failures.length > 0) {
-    rows.push(/* @__PURE__ */ jsx(Text, { children: " " }, "sp5"));
-    rows.push(/* @__PURE__ */ jsx(SectionHeader, { children: "Failures (top)" }, "fail-h"));
-    failures.forEach((f, i) => {
-      rows.push(
-        /* @__PURE__ */ jsxs(Text, { color: "red", children: [
-          i + 1,
-          ") ",
-          f.title
-        ] }, `fail-${i}`)
-      );
-    });
+
+// src/evals/metric.ts
+var registry = /* @__PURE__ */ new Map();
+var Metric = {
+  of(config) {
+    const def = {
+      id: config.id,
+      name: config.name,
+      aggregate: config.aggregate,
+      format: config.format,
+      make: (data) => ({ id: config.id, data })
+    };
+    registry.set(config.id, def);
+    return def;
   }
-  return rows;
+};
+function getMetricById(id) {
+  return registry.get(id);
 }
-function RunDetailsView({
-  state,
-  dataset,
-  selectedRun
-}) {
-  const runs = dataset?.runs ?? [];
-  const rightFocused = state.focus === "right";
-  if (!selectedRun) {
-    return /* @__PURE__ */ jsxs(Fragment, { children: [
-      /* @__PURE__ */ jsx(RunsSidebar, { state, dataset, runs }),
-      /* @__PURE__ */ jsx(Pane, { flexGrow: 1, marginLeft: 1, focused: rightFocused, children: /* @__PURE__ */ jsx(Text, { color: "gray", children: "Select a run to inspect details." }) })
-    ] });
+
+// src/evals/score.ts
+var registry2 = /* @__PURE__ */ new Map();
+var Score = {
+  of(config) {
+    const def = {
+      id: config.id,
+      name: config.name,
+      displayStrategy: config.displayStrategy,
+      aggregate: config.aggregate,
+      format: config.format,
+      make: (data, options) => {
+        const passed = options?.definePassed !== void 0 ? options.definePassed(data) : void 0;
+        return {
+          id: config.id,
+          data,
+          ...passed !== void 0 && { passed }
+        };
+      }
+    };
+    registry2.set(config.id, def);
+    return def;
   }
-  const rows = buildDetailRows(selectedRun);
-  const offset = Math.max(0, state.detailsScrollOffset);
-  const visible = rows.slice(offset, offset + DETAILS_PAGE_SIZE);
-  return /* @__PURE__ */ jsxs(Fragment, { children: [
-    /* @__PURE__ */ jsx(RunsSidebar, { state, dataset, runs }),
-    /* @__PURE__ */ jsx(Pane, { flexGrow: 1, marginLeft: 1, focused: rightFocused, children: /* @__PURE__ */ jsx(Box, { flexDirection: "column", children: visible.map((row, i) => /* @__PURE__ */ jsx(React.Fragment, { children: row }, i)) }) })
-  ] });
-}
-var LEFT_PANE_WIDTH3 = 44;
-function NewEvaluationView({
-  state,
-  data,
-  visibleEvaluators
-}) {
-  const selectedCount = state.selectedEvaluatorIds.length;
-  const focusedEvaluator = visibleEvaluators[state.evaluatorMenuIndex];
-  const leftFocused = state.focus === "left";
-  const rightFocused = state.focus === "right";
-  return /* @__PURE__ */ jsxs(Fragment, { children: [
-    /* @__PURE__ */ jsxs(Pane, { width: LEFT_PANE_WIDTH3, focused: leftFocused, children: [
-      /* @__PURE__ */ jsx(SectionHeader, { children: "Available Evaluators" }),
-      /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
-        "Search: ",
-        state.searchQuery || "(none)"
-      ] }),
-      visibleEvaluators.map((evaluator, index) => {
-        const selected = index === state.evaluatorMenuIndex;
-        const inSelection = state.selectedEvaluatorIds.includes(evaluator.id);
-        return /* @__PURE__ */ jsxs(
-          Text,
-          {
-            color: selected ? "cyan" : "gray",
-            bold: selected,
-            children: [
-              selected ? "\u25B8 " : "  ",
-              inSelection ? "[x] " : "[ ] ",
-              evaluator.name
-            ]
-          },
-          evaluator.id
-        );
-      })
-    ] }),
-    /* @__PURE__ */ jsxs(Pane, { flexGrow: 1, marginLeft: 1, focused: rightFocused, children: [
-      /* @__PURE__ */ jsxs(SectionHeader, { children: [
-        "Selected (",
-        selectedCount,
-        ")"
-      ] }),
-      state.selectedEvaluatorIds.map((id, index) => {
-        const evaluator = data.evaluators.find((item) => item.id === id);
-        if (!evaluator)
-          return null;
-        return /* @__PURE__ */ jsxs(Text, { children: [
-          index + 1,
-          ") ",
-          evaluator.name
-        ] }, id);
-      }),
-      /* @__PURE__ */ jsx(SectionHeader, { children: "Config preview" }),
-      /* @__PURE__ */ jsx(Text, { color: "gray", children: focusedEvaluator?.configPreview ?? "Select an evaluator to inspect config." })
-    ] })
-  ] });
+};
+function getScoreById(id) {
+  return registry2.get(id);
 }
-function clampCursor(state, filteredDatasetsLength, selectedRunCount) {
-  const datasetMax = filteredDatasetsLength;
-  const runMax = selectedRunCount;
-  const evaluatorMax = 3;
-  return {
-    ...state,
-    datasetMenuIndex: Math.max(0, Math.min(state.datasetMenuIndex, datasetMax)),
-    runMenuIndex: Math.max(0, Math.min(state.runMenuIndex, runMax)),
-    evaluatorMenuIndex: Math.max(
-      0,
-      Math.min(state.evaluatorMenuIndex, evaluatorMax)
-    )
+
+// src/evals/aggregators.ts
+function aggregateAverage(values) {
+  if (values.length === 0) {
+    return { value: 0 };
+  }
+  const sum = values.reduce((s, v) => s + v.value, 0);
+  return { value: sum / values.length };
+}
+function aggregateAll(values) {
+  return { passed: values.length > 0 && values.every((v) => v.passed) };
+}
+function aggregateTokenCountSum(values) {
+  const initial = {
+    input: 0,
+    output: 0,
+    inputCached: 0,
+    outputCached: 0
   };
-}
-function EvalsCliApp({
-  data,
-  args,
-  runner
-}) {
-  const { exit } = useApp();
-  const { width: stdoutWidth, height: stdoutHeight } = useScreenSize();
-  const [liveData, setLiveData] = useState(data);
-  const [runtimeMessage, setRuntimeMessage] = useState();
-  const [state, dispatch] = useReducer(
-    reduceCliState,
-    createInitialState(data, args)
+  return values.reduce(
+    (acc, v) => ({
+      input: acc.input + (v.input ?? 0),
+      output: acc.output + (v.output ?? 0),
+      inputCached: acc.inputCached + (v.inputCached ?? 0),
+      outputCached: acc.outputCached + (v.outputCached ?? 0)
+    }),
+    initial
   );
-  useEffect(() => {
-    setLiveData(data);
-  }, [data]);
-  useEffect(() => {
-    if (!runner) {
-      return void 0;
-    }
-    return runner.subscribeRunEvents((event) => {
-      setLiveData((current) => applyRunnerEvent(current, event, runner));
-      if (event.type === "RunQueued") {
-        setRuntimeMessage(`Queued ${event.runId} with ${event.totalTestCases} test cases.`);
-      }
-      if (event.type === "RunCompleted") {
-        setRuntimeMessage(
-          `Completed ${event.runId}: ${event.passedTestCases}/${event.totalTestCases} passed.`
-        );
-      }
-      if (event.type === "RunFailed") {
-        setRuntimeMessage(`Run failed: ${event.errorMessage}`);
-      }
-    });
-  }, [runner]);
-  const filteredDatasets = useMemo(
-    () => getFilteredDatasets(liveData, state.searchQuery),
-    [liveData, state.searchQuery]
-  );
-  const clampedState = clampCursor(
-    state,
-    filteredDatasets.length,
-    getDatasetByMenuIndex(filteredDatasets, state.datasetMenuIndex)?.runs.length ?? 0
-  );
-  const selectedDataset = getDatasetByMenuIndex(
-    filteredDatasets,
-    clampedState.datasetMenuIndex
-  );
-  const selectedRun = getRunByMenuIndex(
-    selectedDataset,
-    clampedState.runMenuIndex
-  );
-  const visibleEvaluators = liveData.evaluators.filter(
-    (evaluator) => evaluator.name.toLowerCase().includes(clampedState.searchQuery.toLowerCase())
-  );
-  useInput((input, key) => {
-    if (isQuitInput(input) || key.escape) {
-      exit();
-      return;
-    }
-    if (key.tab) {
-      dispatch({ type: "TOGGLE_FOCUS" });
-      return;
-    }
-    if (isSearchInput(input)) {
-      dispatch({ type: "START_SEARCH" });
-      return;
-    }
-    if (clampedState.searchMode) {
-      if (key.return) {
-        dispatch({ type: "END_SEARCH" });
-        return;
-      }
-      if (isBackKey(key)) {
-        dispatch({ type: "REMOVE_SEARCH_CHAR" });
-        return;
-      }
-      if (isPrintableCharacter(input)) {
-        dispatch({ type: "APPEND_SEARCH", value: input });
-      }
-      return;
-    }
-    if (key.upArrow) {
-      const max = clampedState.level === "details" ? 100 : clampedState.level === "new-evaluation" ? visibleEvaluators.length - 1 : 100;
-      dispatch({ type: "MOVE_UP", max });
-      return;
-    }
-    if (key.downArrow) {
-      const max = clampedState.level === "datasets" ? filteredDatasets.length : clampedState.level === "runs" ? selectedDataset?.runs.length ?? 0 : clampedState.level === "new-evaluation" ? Math.max(0, visibleEvaluators.length - 1) : 100;
-      dispatch({ type: "MOVE_DOWN", max });
-      return;
-    }
-    if (key.return) {
-      dispatch({
-        type: "ENTER",
-        hasDataset: Boolean(selectedDataset),
-        hasRun: Boolean(selectedRun)
-      });
-      if (clampedState.level === "new-evaluation") {
-        const evaluator = visibleEvaluators[clampedState.evaluatorMenuIndex];
-        if (evaluator) {
-          dispatch({ type: "TOGGLE_EVALUATOR", evaluatorId: evaluator.id });
-        }
-      }
-      return;
-    }
-    if (isBackKey(key)) {
-      dispatch({ type: "BACK" });
-      return;
-    }
-    if (input.toLowerCase() === "c") {
-      dispatch({ type: "CLEAR_WARNINGS" });
-      setRuntimeMessage(void 0);
-      return;
+}
+function aggregateLatencyAverage(values) {
+  if (values.length === 0) {
+    return { ms: 0 };
+  }
+  const sum = values.reduce((s, v) => s + v.ms, 0);
+  return { ms: sum / values.length };
+}
+
+// src/evals/metrics/standard.ts
+Metric.of({
+  id: "token-count",
+  name: "Tokens",
+  aggregate: aggregateTokenCountSum,
+  format: (data, options) => {
+    const input = data.input ?? 0;
+    const output = data.output ?? 0;
+    const inputCached = data.inputCached ?? 0;
+    const outputCached = data.outputCached ?? 0;
+    const cached = inputCached + outputCached;
+    const base = `in:${input} out:${output} cached:${cached}`;
+    return options?.isAggregated ? `Total: ${base}` : base;
+  }
+});
+Metric.of({
+  id: "latency",
+  name: "Latency",
+  aggregate: aggregateLatencyAverage,
+  format: (data, options) => options?.isAggregated ? `Avg: ${data.ms}ms` : `${data.ms}ms`
+});
+
+// src/evals/scores/standard.ts
+Score.of({
+  id: "percent",
+  name: "Score",
+  displayStrategy: "bar",
+  format: (data, options) => options?.isAggregated ? `Avg: ${data.value.toFixed(2)}` : data.value.toFixed(2),
+  aggregate: aggregateAverage
+});
+Score.of({
+  id: "binary",
+  name: "Result",
+  displayStrategy: "passFail",
+  format: (data, options) => options?.isAggregated ? data.passed ? "All: PASSED" : "Some: FAILED" : data.passed ? "PASSED" : "NOT PASSED",
+  aggregate: aggregateAll
+});
+function createDiffLogEntry(expected, actual, options) {
+  const diff = diffString(expected, actual, { color: false });
+  return {
+    type: "diff",
+    label: options?.label,
+    expected,
+    actual,
+    diff: diff || "(no differences)"
+  };
+}
+function getDiffLines(entry) {
+  const raw = diffString(entry.expected, entry.actual, { color: false }) || "(no differences)";
+  return raw.split("\n").map((line) => {
+    const trimmed = line.trimStart();
+    if (trimmed.startsWith("-") && !trimmed.startsWith("---")) {
+      return { type: "remove", line };
     }
-    if (input.toLowerCase() === "s" && clampedState.level === "new-evaluation") {
-      if (!runner) {
-        setRuntimeMessage("Runner unavailable: cannot start evaluation.");
-        return;
-      }
-      if (!selectedDataset) {
-        setRuntimeMessage("Select a dataset before starting a new evaluation.");
-        return;
-      }
-      if (clampedState.selectedEvaluatorIds.length === 0) {
-        setRuntimeMessage("Select at least one evaluator before starting.");
-        return;
-      }
-      void runner.runDatasetWith({
-        datasetId: selectedDataset.id,
-        evaluatorIds: clampedState.selectedEvaluatorIds
-      }).then((snapshot) => {
-        setRuntimeMessage(
-          `Started ${snapshot.runId} on ${selectedDataset.name} (${snapshot.totalTestCases} cases).`
-        );
-      }).catch((error) => {
-        setRuntimeMessage(
-          error instanceof Error ? error.message : "Failed to start evaluation."
-        );
-      });
+    if (trimmed.startsWith("+") && !trimmed.startsWith("+++")) {
+      return { type: "add", line };
     }
+    return { type: "context", line };
   });
-  const renderContent = () => {
-    if (clampedState.level === "new-evaluation") {
-      return /* @__PURE__ */ jsx(
-        NewEvaluationView,
-        {
-          state: clampedState,
-          data: liveData,
-          visibleEvaluators
-        }
-      );
-    }
-    if (clampedState.level === "datasets") {
-      return /* @__PURE__ */ jsx(
-        DatasetsView,
-        {
-          state: clampedState,
-          filteredDatasets,
-          selectedDataset
-        }
-      );
-    }
-    if (clampedState.level === "runs") {
-      return /* @__PURE__ */ jsx(
-        RunsView,
-        {
-          state: clampedState,
-          dataset: selectedDataset,
-          selectedRun
-        }
-      );
-    }
-    return /* @__PURE__ */ jsx(
-      RunDetailsView,
-      {
-        state: clampedState,
-        dataset: selectedDataset,
-        selectedRun
+}
+
+// src/runner/score-utils.ts
+function toNumericScoreFromScores(scores) {
+  for (const item of scores) {
+    const def = getScoreById(item.id);
+    if (def && def.displayStrategy === "bar" && typeof item.data === "object" && item.data !== null && "value" in item.data) {
+      const value = item.data.value;
+      if (typeof value === "number" && Number.isFinite(value)) {
+        return value;
       }
-    );
-  };
-  return /* @__PURE__ */ jsxs(
-    Box,
-    {
-      flexDirection: "column",
-      flexGrow: 1,
-      width: stdoutWidth,
-      height: stdoutHeight,
-      children: [
-        /* @__PURE__ */ jsx(
-          Box,
-          {
-            borderStyle: "round",
-            borderColor: "cyan",
-            paddingX: 1,
-            width: stdoutWidth,
-            children: /* @__PURE__ */ jsx(Text, { children: getBreadcrumbText(
-              clampedState,
-              selectedDataset?.name,
-              selectedRun?.label
-            ) })
-          }
-        ),
-        clampedState.startupWarnings.length > 0 && /* @__PURE__ */ jsxs(
-          Box,
-          {
-            marginTop: 1,
-            borderStyle: "round",
-            borderColor: "yellow",
-            paddingX: 1,
-            flexDirection: "column",
-            width: stdoutWidth,
-            children: [
-              /* @__PURE__ */ jsx(Text, { color: "yellow", children: "Startup warnings:" }),
-              clampedState.startupWarnings.map((warning, index) => /* @__PURE__ */ jsx(Text, { children: warning }, `${warning}-${index}`))
-            ]
-          }
-        ),
-        clampedState.searchMode && /* @__PURE__ */ jsxs(
-          Box,
-          {
-            marginTop: 1,
-            borderStyle: "round",
-            borderColor: "magenta",
-            paddingX: 1,
-            width: stdoutWidth,
-            children: [
-              /* @__PURE__ */ jsx(Text, { color: "magenta", bold: true, children: "Search: " }),
-              /* @__PURE__ */ jsx(Text, { color: "white", children: clampedState.searchQuery })
-            ]
-          }
-        ),
-        runtimeMessage && /* @__PURE__ */ jsx(
-          Box,
-          {
-            marginTop: 1,
-            borderStyle: "round",
-            borderColor: "blue",
-            paddingX: 1,
-            width: stdoutWidth,
-            children: /* @__PURE__ */ jsx(Text, { color: "blue", children: runtimeMessage })
-          }
-        ),
-        /* @__PURE__ */ jsx(
-          Box,
-          {
-            marginTop: 1,
-            flexGrow: 1,
-            width: stdoutWidth,
-            flexDirection: "row",
-            children: renderContent()
-          }
-        ),
-        /* @__PURE__ */ jsx(Box, { marginTop: 1, paddingX: 1, children: /* @__PURE__ */ jsx(Text, { color: "gray", children: getFooterText(clampedState) }) })
-      ]
     }
+    const numeric = toNumericScore(item.data);
+    if (numeric !== void 0) {
+      return numeric;
+    }
+  }
+  return void 0;
+}
+function toNumericScore(value) {
+  if (typeof value === "number" && Number.isFinite(value)) {
+    return value;
+  }
+  if (typeof value !== "object" || value === null) {
+    return void 0;
+  }
+  const obj = value;
+  if ("score" in obj && typeof obj.score === "number" && Number.isFinite(obj.score)) {
+    return obj.score;
+  }
+  const numberValues = Object.values(value).filter(
+    (entry) => typeof entry === "number" && Number.isFinite(entry)
   );
+  if (numberValues.length === 0) {
+    return void 0;
+  }
+  return numberValues.reduce((sum, entry) => sum + entry, 0) / numberValues.length;
 }
 
 // src/runner/config.ts
@@ -1293,7 +1026,8 @@ var defaultRunnerConfig = {
     ],
     excludeDirectories: ["node_modules", "dist", ".next", ".git", ".pnpm-store"]
   },
-  artifactDirectory: ".eval-results"
+  artifactDirectory: ".eval-results",
+  maxConcurrency: 1
 };
 function toRunnerConfigOverrides(config) {
   if (!config) {
@@ -1326,6 +1060,9 @@ function toRunnerConfigOverrides(config) {
   if (config.artifactDirectory !== void 0) {
     overrides.artifactDirectory = config.artifactDirectory;
   }
+  if (config.maxConcurrency !== void 0) {
+    overrides.maxConcurrency = config.maxConcurrency;
+  }
   if (Object.keys(discovery).length > 0) {
     overrides.discovery = discovery;
   }
@@ -1519,129 +1256,6 @@ async function collectTestCasesFromFiles(config) {
   );
   return found.flat();
 }
-function createDiffLogEntry(expected, actual, options) {
-  const diff = diffString(expected, actual, { color: false });
-  return {
-    type: "diff",
-    label: options?.label,
-    expected,
-    actual,
-    diff: diff || "(no differences)"
-  };
-}
-
-// src/evals/metric.ts
-var registry = /* @__PURE__ */ new Map();
-var Metric = {
-  of(config) {
-    const def = {
-      id: config.id,
-      name: config.name,
-      format: config.format,
-      make: (data) => ({ id: config.id, data })
-    };
-    registry.set(config.id, def);
-    return def;
-  }
-};
-
-// src/evals/score.ts
-var registry2 = /* @__PURE__ */ new Map();
-var Score = {
-  of(config) {
-    const def = {
-      id: config.id,
-      name: config.name,
-      displayStrategy: config.displayStrategy,
-      format: config.format,
-      make: (data, options) => {
-        const passed = options?.definePassed !== void 0 ? options.definePassed(data) : void 0;
-        return {
-          id: config.id,
-          data,
-          ...passed !== void 0 && { passed }
-        };
-      }
-    };
-    registry2.set(config.id, def);
-    return def;
-  }
-};
-function getScoreById(id) {
-  return registry2.get(id);
-}
-
-// src/evals/metrics/standard.ts
-Metric.of({
-  id: "token-count",
-  name: "Tokens",
-  format: (data) => {
-    const input = data.input ?? 0;
-    const output = data.output ?? 0;
-    const inputCached = data.inputCached ?? 0;
-    const outputCached = data.outputCached ?? 0;
-    const cached = inputCached + outputCached;
-    return `in:${input} out:${output} cached:${cached}`;
-  }
-});
-Metric.of({
-  id: "latency",
-  name: "Latency",
-  format: (data) => `${data.ms}ms`
-});
-
-// src/evals/scores/standard.ts
-Score.of({
-  id: "percent",
-  name: "Score",
-  displayStrategy: "bar",
-  format: (data) => data.value.toFixed(2)
-});
-Score.of({
-  id: "binary",
-  name: "Result",
-  displayStrategy: "passFail",
-  format: (data) => data.passed ? "PASSED" : "NOT PASSED"
-});
-
-// src/runner/score-utils.ts
-function toNumericScoreFromScores(scores) {
-  for (const item of scores) {
-    const def = getScoreById(item.id);
-    if (def && def.displayStrategy === "bar" && typeof item.data === "object" && item.data !== null && "value" in item.data) {
-      const value = item.data.value;
-      if (typeof value === "number" && Number.isFinite(value)) {
-        return value;
-      }
-    }
-    const numeric = toNumericScore(item.data);
-    if (numeric !== void 0) {
-      return numeric;
-    }
-  }
-  return void 0;
-}
-function toNumericScore(value) {
-  if (typeof value === "number" && Number.isFinite(value)) {
-    return value;
-  }
-  if (typeof value !== "object" || value === null) {
-    return void 0;
-  }
-  const obj = value;
-  if ("score" in obj && typeof obj.score === "number" && Number.isFinite(obj.score)) {
-    return obj.score;
-  }
-  const numberValues = Object.values(value).filter(
-    (entry) => typeof entry === "number" && Number.isFinite(entry)
-  );
-  if (numberValues.length === 0) {
-    return void 0;
-  }
-  return numberValues.reduce((sum, entry) => sum + entry, 0) / numberValues.length;
-}
-
-// src/runner/execution.ts
 function computeEvaluatorPassed(evaluator, result, scores) {
   const scoresWithPassed = scores.filter((s) => "passed" in s && s.passed !== void 0);
   if (scoresWithPassed.length > 0) {
@@ -1683,6 +1297,105 @@ function createArtifactPath(artifactDirectory, datasetId, runId) {
     `${datasetId}_${runId}_${nowIsoForFile()}.jsonl`
   );
 }
+function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent, persistenceQueue, updateSnapshot, completedRef, passedRef, failedRef) {
+  return Effect.gen(function* () {
+    const reruns = typeof testCaseItem.testCase.getReruns === "function" ? testCaseItem.testCase.getReruns() : 1;
+    const rerunPassed = [];
+    for (let r = 0; r < reruns; r++) {
+      const started = Date.now();
+      const evaluatorScores = [];
+      let testCaseError;
+      const output = readOutput(testCaseItem.testCase);
+      for (const { id: evaluatorId, evaluator } of task.evaluators) {
+        const evaluateFn = evaluator.getEvaluateFn();
+        if (!evaluateFn) {
+          continue;
+        }
+        try {
+          const logs = [];
+          const logDiff = (expected, actual, options) => {
+            logs.push(createDiffLogEntry(expected, actual, options));
+          };
+          const ctx = yield* Effect.promise(
+            () => Promise.resolve(evaluator.resolveContext())
+          );
+          const result = yield* Effect.promise(
+            () => Promise.resolve(
+              evaluateFn({
+                input: testCaseItem.testCase.getInput(),
+                ctx,
+                output,
+                logDiff
+              })
+            )
+          );
+          const { scores, metrics } = normalizeResult(result);
+          const passed2 = computeEvaluatorPassed(evaluator, result, scores);
+          evaluatorScores.push({
+            evaluatorId,
+            scores,
+            passed: passed2,
+            metrics,
+            logs: logs.length > 0 ? logs : void 0
+          });
+        } catch (error) {
+          testCaseError = error instanceof Error ? error.message : "Evaluator execution failed";
+          evaluatorScores.push({
+            evaluatorId,
+            scores: [],
+            passed: false
+          });
+        }
+      }
+      const rerunPassedThis = evaluatorScores.every((s) => s.passed);
+      rerunPassed.push(rerunPassedThis);
+      const completedEvaluations = yield* Ref.modify(completedRef, (n) => [
+        n + 1,
+        n + 1
+      ]);
+      const progressEvent = {
+        type: "TestCaseProgress",
+        runId: task.runId,
+        testCaseId: testCaseItem.id,
+        testCaseName: testCaseItem.testCase.getName(),
+        completedTestCases: completedEvaluations,
+        totalTestCases: totalEvaluations,
+        rerunIndex: r + 1,
+        rerunTotal: reruns,
+        passed: rerunPassedThis,
+        durationMs: Date.now() - started,
+        evaluatorScores,
+        output,
+        errorMessage: testCaseError
+      };
+      updateSnapshot(task.runId, (snapshot) => ({
+        ...snapshot,
+        completedTestCases: completedEvaluations
+      }));
+      yield* publishEvent(progressEvent);
+      yield* Queue.offer(persistenceQueue, {
+        runId: task.runId,
+        artifactPath: task.snapshot.artifactPath,
+        payload: progressEvent
+      });
+    }
+    const testCasePassed = rerunPassed.every(Boolean);
+    if (testCasePassed) {
+      yield* Ref.update(passedRef, (n) => n + 1);
+    } else {
+      yield* Ref.update(failedRef, (n) => n + 1);
+    }
+    const [passed, failed] = yield* Effect.all([
+      Ref.get(passedRef),
+      Ref.get(failedRef)
+    ]);
+    updateSnapshot(task.runId, (snapshot) => ({
+      ...snapshot,
+      passedTestCases: passed,
+      failedTestCases: failed
+    }));
+  });
+}
 var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => Effect.gen(function* () {
   const startedAt = Date.now();
   updateSnapshot(task.runId, (snapshot) => ({
@@ -1695,118 +1408,215 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
     runId: task.runId,
     startedAt
   });
-  let completedTestCases = 0;
-  let passedTestCases = 0;
-  let failedTestCases = 0;
-  for (const testCaseItem of task.testCases) {
-    const started = Date.now();
-    const evaluatorScores = [];
-    let testCaseError;
-    const output = readOutput(testCaseItem.testCase);
-    for (const { id: evaluatorId, evaluator } of task.evaluators) {
-      const evaluateFn = evaluator.getEvaluateFn();
-      if (!evaluateFn) {
-        continue;
+  const totalEvaluations = task.testCases.reduce(
+    (sum, tc) => sum + (typeof tc.testCase.getReruns === "function" ? tc.testCase.getReruns() : 1),
+    0
+  );
+  const maxConcurrency = Math.max(1, task.maxConcurrency ?? 1);
+  const completedRef = yield* Ref.make(0);
+  const passedRef = yield* Ref.make(0);
+  const failedRef = yield* Ref.make(0);
+  const processTestCase = (testCaseItem) => processOneTestCase(
+    task,
+    testCaseItem,
+    totalEvaluations,
+    publishEvent,
+    persistenceQueue,
+    updateSnapshot,
+    completedRef,
+    passedRef,
+    failedRef
+  );
+  yield* Effect.forEach(
+    task.testCases,
+    processTestCase,
+    maxConcurrency > 1 ? { concurrency: maxConcurrency } : void 0
+  );
+  const [completedEvaluations, passedUniqueTestCases, failedUniqueTestCases] = yield* Effect.all([
+    Ref.get(completedRef),
+    Ref.get(passedRef),
+    Ref.get(failedRef)
+  ]);
+  const finishedAt = Date.now();
+  const completedEvent = {
+    type: "RunCompleted",
+    runId: task.runId,
+    finishedAt,
+    passedTestCases: passedUniqueTestCases,
+    failedTestCases: failedUniqueTestCases,
+    totalTestCases: task.testCases.length,
+    artifactPath: task.snapshot.artifactPath
+  };
+  updateSnapshot(task.runId, (snapshot) => ({
+    ...snapshot,
+    status: "completed",
+    completedTestCases: completedEvaluations,
+    passedTestCases: passedUniqueTestCases,
+    failedTestCases: failedUniqueTestCases,
+    finishedAt
+  }));
+  yield* publishEvent(completedEvent);
+  yield* Queue.offer(persistenceQueue, {
+    runId: task.runId,
+    artifactPath: task.snapshot.artifactPath,
+    payload: completedEvent
+  });
+  yield* publishEvent({
+    type: "ArtifactFlushed",
+    runId: task.runId,
+    artifactPath: task.snapshot.artifactPath
+  });
+});
+async function loadRunSnapshotsFromArtifacts(config) {
+  const baseDir = resolve(config.artifactDirectory);
+  let entries;
+  try {
+    entries = await readdir(baseDir);
+  } catch {
+    return [];
+  }
+  const jsonlFiles = entries.filter((name) => name.endsWith(".jsonl"));
+  const snapshots = [];
+  for (const fileName of jsonlFiles) {
+    const filePath = join(baseDir, fileName);
+    try {
+      const snapshot = await parseArtifactToSnapshot(filePath, config);
+      if (snapshot) {
+        snapshots.push(snapshot);
       }
-      try {
-        const logs = [];
-        const logDiff = (expected, actual, options) => {
-          logs.push(createDiffLogEntry(expected, actual, options));
+    } catch {
+    }
+  }
+  return snapshots.sort((a, b) => b.queuedAt - a.queuedAt);
+}
+async function parseArtifactToSnapshot(filePath, _config) {
+  const content = await readFile(filePath, "utf8");
+  const lines = content.split("\n").filter((line) => line.trim().length > 0);
+  if (lines.length === 0) {
+    return null;
+  }
+  let runQueued = null;
+  let runCompleted = null;
+  let runFailed = null;
+  let runStarted = null;
+  for (const line of lines) {
+    try {
+      const event = JSON.parse(line);
+      const type = event.type;
+      if (type === "RunQueued") {
+        runQueued = {
+          runId: event.runId,
+          datasetId: event.datasetId,
+          datasetName: event.datasetName,
+          evaluatorIds: event.evaluatorIds,
+          totalTestCases: event.totalTestCases ?? 0,
+          artifactPath: event.artifactPath ?? filePath,
+          ts: event.ts
+        };
+      }
+      if (type === "RunStarted") {
+        runStarted = { startedAt: event.startedAt };
+      }
+      if (type === "RunCompleted") {
+        runCompleted = {
+          passedTestCases: event.passedTestCases,
+          failedTestCases: event.failedTestCases,
+          totalTestCases: event.totalTestCases,
+          finishedAt: event.finishedAt
         };
-        const ctx = yield* Effect.promise(
-          () => Promise.resolve(evaluator.resolveContext())
-        );
-        const result = yield* Effect.promise(
-          () => Promise.resolve(
-            evaluateFn({
-              input: testCaseItem.testCase.getInput(),
-              ctx,
-              output,
-              logDiff
-            })
-          )
-        );
-        const { scores, metrics } = normalizeResult(result);
-        const passed = computeEvaluatorPassed(evaluator, result, scores);
-        evaluatorScores.push({
-          evaluatorId,
-          scores,
-          passed,
-          metrics,
-          logs: logs.length > 0 ? logs : void 0
-        });
-      } catch (error) {
-        testCaseError = error instanceof Error ? error.message : "Evaluator execution failed";
-        evaluatorScores.push({
-          evaluatorId,
-          scores: [],
-          passed: false
-        });
       }
+      if (type === "RunFailed") {
+        runFailed = {
+          finishedAt: event.finishedAt,
+          errorMessage: event.errorMessage
+        };
+      }
+    } catch {
     }
-    const testCasePassed = evaluatorScores.every((s) => s.passed);
-    completedTestCases += 1;
-    if (testCasePassed) {
+  }
+  if (!runQueued) {
+    return null;
+  }
+  const artifactPath = filePath;
+  const status = runFailed ? "failed" : runCompleted ? "completed" : runStarted ? "running" : "queued";
+  const progress = aggregateTestCaseProgress(lines);
+  const completedTestCases = runCompleted ? runQueued.totalTestCases : progress.completedTestCases;
+  const passedTestCases = runCompleted?.passedTestCases ?? progress.passedTestCases;
+  const failedTestCases = runCompleted?.failedTestCases ?? progress.failedTestCases;
+  return {
+    runId: runQueued.runId,
+    datasetId: runQueued.datasetId,
+    datasetName: runQueued.datasetName,
+    evaluatorIds: runQueued.evaluatorIds,
+    queuedAt: runQueued.ts ?? 0,
+    startedAt: runStarted?.startedAt,
+    finishedAt: runCompleted?.finishedAt ?? runFailed?.finishedAt,
+    totalTestCases: runQueued.totalTestCases,
+    completedTestCases,
+    passedTestCases,
+    failedTestCases,
+    status,
+    artifactPath,
+    errorMessage: runFailed?.errorMessage
+  };
+}
+function aggregateTestCaseProgress(lines) {
+  let completedTestCases = 0;
+  const testCasePassedBy = /* @__PURE__ */ new Map();
+  for (const line of lines) {
+    try {
+      const event = JSON.parse(line);
+      if (event.type === "TestCaseProgress") {
+        const ev = event;
+        completedTestCases = ev.completedTestCases ?? completedTestCases;
+        const id = ev.testCaseId;
+        const current = testCasePassedBy.get(id);
+        testCasePassedBy.set(id, current === void 0 ? ev.passed : current && ev.passed);
+      }
+    } catch {
+    }
+  }
+  let passedTestCases = 0;
+  let failedTestCases = 0;
+  for (const passed of testCasePassedBy.values()) {
+    if (passed) {
       passedTestCases += 1;
     } else {
       failedTestCases += 1;
     }
-    const progressEvent = {
-      type: "TestCaseProgress",
-      runId: task.runId,
-      testCaseId: testCaseItem.id,
-      testCaseName: testCaseItem.testCase.getName(),
-      completedTestCases,
-      totalTestCases: task.testCases.length,
-      passed: testCasePassed,
-      durationMs: Date.now() - started,
-      evaluatorScores,
-      output,
-      errorMessage: testCaseError
-    };
-    updateSnapshot(task.runId, (snapshot) => ({
-      ...snapshot,
-      completedTestCases,
-      passedTestCases,
-      failedTestCases
-    }));
-    yield* publishEvent(progressEvent);
-    yield* Queue.offer(persistenceQueue, {
-      runId: task.runId,
-      artifactPath: task.snapshot.artifactPath,
-      payload: progressEvent
-    });
   }
-  const finishedAt = Date.now();
-  const completedEvent = {
-    type: "RunCompleted",
-    runId: task.runId,
-    finishedAt,
-    passedTestCases,
-    failedTestCases,
-    totalTestCases: task.testCases.length,
-    artifactPath: task.snapshot.artifactPath
-  };
-  updateSnapshot(task.runId, (snapshot) => ({
-    ...snapshot,
-    status: "completed",
-    completedTestCases,
-    passedTestCases,
-    failedTestCases,
-    finishedAt
-  }));
-  yield* publishEvent(completedEvent);
-  yield* Queue.offer(persistenceQueue, {
-    runId: task.runId,
-    artifactPath: task.snapshot.artifactPath,
-    payload: completedEvent
-  });
-  yield* publishEvent({
-    type: "ArtifactFlushed",
-    runId: task.runId,
-    artifactPath: task.snapshot.artifactPath
-  });
-});
+  return { completedTestCases, passedTestCases, failedTestCases };
+}
+async function parseArtifactFile(artifactPath) {
+  try {
+    const content = await readFile(artifactPath, "utf8");
+    const lines = content.split("\n").filter((line) => line.trim().length > 0);
+    const results = [];
+    for (const line of lines) {
+      try {
+        const event = JSON.parse(line);
+        if (event.type === "TestCaseProgress") {
+          const ev = event;
+          results.push({
+            testCaseId: ev.testCaseId,
+            testCaseName: ev.testCaseName,
+            completedTestCases: ev.completedTestCases,
+            totalTestCases: ev.totalTestCases,
+            rerunIndex: ev.rerunIndex,
+            rerunTotal: ev.rerunTotal,
+            passed: ev.passed,
+            durationMs: ev.durationMs,
+            evaluatorScores: ev.evaluatorScores ?? []
+          });
+        }
+      } catch {
+      }
+    }
+    return results;
+  } catch {
+    return [];
+  }
+}
 async function appendJsonLine(artifactPath, payload) {
   await mkdir(dirname(artifactPath), { recursive: true });
   await appendFile(artifactPath, `${JSON.stringify(payload)}
@@ -1822,291 +1632,873 @@ var createPersistenceWorker = (queue) => Effect.forever(
         ...message.payload
       })
     );
-  })
-);
-
-// src/runner/search.ts
-function matchesAny(value, matchers) {
-  if (!matchers || matchers.length === 0) {
-    return true;
+  })
+);
+
+// src/runner/search.ts
+function matchesAny(value, matchers) {
+  if (!matchers || matchers.length === 0) {
+    return true;
+  }
+  return matchers.some(
+    (matcher) => typeof matcher === "string" ? matcher === value : matcher.test(value)
+  );
+}
+function matchesPath(value, matchers) {
+  if (!matchers || matchers.length === 0) {
+    return true;
+  }
+  return matchers.some((matcher) => {
+    if (typeof matcher === "string") {
+      return value.includes(matcher);
+    }
+    return matcher.test(value);
+  });
+}
+function searchCollectedTestCases(all, query) {
+  if (!query) {
+    return all;
+  }
+  return all.filter((item) => {
+    const tags = item.testCase.getTags();
+    if (query.excludedTags && tags.some((tag) => matchesAny(tag, query.excludedTags))) {
+      return false;
+    }
+    if (query.excludedPaths && matchesPath(item.filePath, query.excludedPaths)) {
+      return false;
+    }
+    const includedTagsMatch = !query.includedTags || query.includedTags.length === 0 || tags.some((tag) => matchesAny(tag, query.includedTags));
+    const includedPathsMatch = !query.includedPaths || query.includedPaths.length === 0 || matchesPath(item.filePath, query.includedPaths);
+    return includedTagsMatch && includedPathsMatch;
+  });
+}
+
+// src/runner/api.ts
+function parseRegexLiteral(pattern) {
+  if (!pattern.startsWith("/")) {
+    return void 0;
+  }
+  const lastSlash = pattern.lastIndexOf("/");
+  if (lastSlash <= 0) {
+    return void 0;
+  }
+  return {
+    source: pattern.slice(1, lastSlash),
+    flags: pattern.slice(lastSlash + 1)
+  };
+}
+function createNameMatcher(pattern) {
+  const normalizedPattern = pattern.trim();
+  const regexLiteral = parseRegexLiteral(normalizedPattern);
+  if (regexLiteral) {
+    const regex = new RegExp(regexLiteral.source, regexLiteral.flags);
+    return (value) => regex.test(value);
+  }
+  if (normalizedPattern.includes("*")) {
+    const escaped = normalizedPattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
+    const regex = new RegExp(`^${escaped}$`, "i");
+    return (value) => regex.test(value);
+  }
+  return (value) => value.toLowerCase() === normalizedPattern.toLowerCase();
+}
+function mergeRunnerOverrides(base, next) {
+  if (!base) {
+    return next;
+  }
+  if (!next) {
+    return base;
+  }
+  const discovery = base.discovery || next.discovery ? {
+    ...base.discovery ?? {},
+    ...next.discovery ?? {}
+  } : void 0;
+  return {
+    ...base,
+    ...next,
+    discovery
+  };
+}
+function createRunner(overrides) {
+  const fileOverrides = loadRunnerConfigFile();
+  const merged = mergeRunnerOverrides(fileOverrides, overrides);
+  return new EffectRunner(withRunnerConfig(merged));
+}
+var EffectRunner = class {
+  constructor(config) {
+    this.eventBus = Effect.runSync(PubSub.unbounded());
+    this.runQueue = Effect.runSync(Queue.unbounded());
+    this.persistenceQueue = Effect.runSync(
+      Queue.unbounded()
+    );
+    this.snapshots = /* @__PURE__ */ new Map();
+    this.listeners = /* @__PURE__ */ new Set();
+    this.datasetsById = /* @__PURE__ */ new Map();
+    this.evaluatorsById = /* @__PURE__ */ new Map();
+    this.schedulerFiber = Effect.runFork(
+      this.createSchedulerEffect()
+    );
+    this.persistenceFiber = Effect.runFork(
+      createPersistenceWorker(this.persistenceQueue)
+    );
+    this.config = config;
+  }
+  async collectDatasets() {
+    const datasets = await collectDatasetsFromFiles(this.config.discovery);
+    this.datasetsById.clear();
+    for (const dataset of datasets) {
+      this.datasetsById.set(dataset.id, dataset);
+    }
+    return datasets;
+  }
+  async collectEvaluators() {
+    const evaluators = await collectEvaluatorsFromFiles(this.config.discovery);
+    this.evaluatorsById.clear();
+    for (const evaluator of evaluators) {
+      this.evaluatorsById.set(evaluator.id, evaluator);
+    }
+    return evaluators;
+  }
+  async resolveDatasetByName(name) {
+    if (this.datasetsById.size === 0) {
+      await this.collectDatasets();
+    }
+    const normalized = name.trim().toLowerCase();
+    return Array.from(this.datasetsById.values()).find(
+      (item) => item.dataset.getName().toLowerCase() === normalized
+    );
+  }
+  async resolveEvaluatorsByNamePattern(pattern) {
+    if (this.evaluatorsById.size === 0) {
+      await this.collectEvaluators();
+    }
+    const matcher = createNameMatcher(pattern);
+    return Array.from(this.evaluatorsById.values()).filter(
+      (item) => matcher(item.evaluator.getName() ?? "")
+    );
+  }
+  async searchTestCases(query) {
+    const testCases = await collectTestCasesFromFiles(this.config.discovery);
+    return searchCollectedTestCases(testCases, query);
+  }
+  async collectDatasetTestCases(datasetId) {
+    if (this.datasetsById.size === 0) {
+      await this.collectDatasets();
+    }
+    const dataset = this.datasetsById.get(datasetId);
+    if (!dataset) {
+      throw new Error(`Unknown dataset: ${datasetId}`);
+    }
+    const allTestCases = await collectTestCasesFromFiles(this.config.discovery);
+    return allTestCases.filter(
+      (testCase) => dataset.dataset.matchesTestCase(testCase.testCase, testCase.filePath)
+    );
+  }
+  async runDatasetWith(request) {
+    if (this.datasetsById.size === 0) {
+      await this.collectDatasets();
+    }
+    if (this.evaluatorsById.size === 0) {
+      await this.collectEvaluators();
+    }
+    const dataset = this.datasetsById.get(request.datasetId);
+    if (!dataset) {
+      throw new Error(`Unknown dataset: ${request.datasetId}`);
+    }
+    const selectedEvaluators = request.evaluatorIds.map((id) => this.evaluatorsById.get(id)).filter((value) => Boolean(value)).map((value) => ({ id: value.id, evaluator: value.evaluator }));
+    if (selectedEvaluators.length === 0) {
+      throw new Error("No evaluators selected for run");
+    }
+    const selectedTestCases = await this.collectDatasetTestCases(request.datasetId);
+    const totalEvaluations = selectedTestCases.reduce(
+      (sum, tc) => sum + (typeof tc.testCase.getReruns === "function" ? tc.testCase.getReruns() : 1),
+      0
+    );
+    const runId = `run-${randomUUID()}`;
+    const artifactPath = createArtifactPath(
+      this.config.artifactDirectory,
+      request.datasetId,
+      runId
+    );
+    const snapshot = {
+      runId,
+      datasetId: request.datasetId,
+      datasetName: dataset.dataset.getName(),
+      evaluatorIds: selectedEvaluators.map((item) => item.id),
+      queuedAt: Date.now(),
+      totalTestCases: totalEvaluations,
+      completedTestCases: 0,
+      passedTestCases: 0,
+      failedTestCases: 0,
+      status: "queued",
+      artifactPath
+    };
+    this.snapshots.set(runId, snapshot);
+    const queuedEvent = {
+      type: "RunQueued",
+      runId,
+      datasetId: request.datasetId,
+      datasetName: dataset.dataset.getName(),
+      evaluatorIds: selectedEvaluators.map((item) => item.id),
+      totalTestCases: totalEvaluations,
+      artifactPath
+    };
+    await Effect.runPromise(this.publishEvent(queuedEvent));
+    await Effect.runPromise(
+      Queue.offer(this.persistenceQueue, {
+        runId,
+        artifactPath,
+        payload: queuedEvent
+      })
+    );
+    const maxConcurrency = request.concurrency ?? this.config.maxConcurrency ?? 1;
+    await Effect.runPromise(
+      Queue.offer(this.runQueue, {
+        runId,
+        datasetId: request.datasetId,
+        dataset: dataset.dataset,
+        evaluators: selectedEvaluators,
+        testCases: selectedTestCases,
+        snapshot,
+        maxConcurrency
+      })
+    );
+    return snapshot;
   }
-  return matchers.some(
-    (matcher) => typeof matcher === "string" ? matcher === value : matcher.test(value)
-  );
-}
-function matchesPath(value, matchers) {
-  if (!matchers || matchers.length === 0) {
-    return true;
+  subscribeRunEvents(listener, options) {
+    const entry = { runId: options?.runId, listener };
+    this.listeners.add(entry);
+    return () => {
+      this.listeners.delete(entry);
+    };
   }
-  return matchers.some((matcher) => {
-    if (typeof matcher === "string") {
-      return value.includes(matcher);
-    }
-    return matcher.test(value);
-  });
-}
-function searchCollectedTestCases(all, query) {
-  if (!query) {
-    return all;
+  getRunSnapshot(runId) {
+    return this.snapshots.get(runId);
   }
-  return all.filter((item) => {
-    const tags = item.testCase.getTags();
-    if (query.excludedTags && tags.some((tag) => matchesAny(tag, query.excludedTags))) {
-      return false;
+  getAllRunSnapshots() {
+    return Array.from(this.snapshots.values()).sort(
+      (a, b) => b.queuedAt - a.queuedAt
+    );
+  }
+  async loadRunSnapshotsFromArtifacts() {
+    return loadRunSnapshotsFromArtifacts(this.config);
+  }
+  async shutdown() {
+    await Effect.runPromise(Fiber.interrupt(this.schedulerFiber));
+    await Effect.runPromise(Fiber.interrupt(this.persistenceFiber));
+    await Effect.runPromise(Queue.shutdown(this.runQueue));
+    await Effect.runPromise(Queue.shutdown(this.persistenceQueue));
+    await Effect.runPromise(PubSub.shutdown(this.eventBus));
+  }
+  createSchedulerEffect() {
+    const self = this;
+    return Effect.forever(
+      Effect.gen(function* () {
+        const task = yield* Queue.take(self.runQueue);
+        yield* Effect.fork(
+          executeRunTask(
+            task,
+            self.publishEvent.bind(self),
+            self.persistenceQueue,
+            self.updateSnapshot.bind(self)
+          )
+        );
+      })
+    );
+  }
+  updateSnapshot(runId, updater) {
+    const existing = this.snapshots.get(runId);
+    if (!existing) {
+      return;
     }
-    if (query.excludedPaths && matchesPath(item.filePath, query.excludedPaths)) {
-      return false;
+    this.snapshots.set(runId, updater(existing));
+  }
+  publishEvent(event) {
+    return Effect.sync(() => {
+      for (const entry of this.listeners) {
+        if (entry.runId && entry.runId !== event.runId) {
+          continue;
+        }
+        entry.listener(event);
+      }
+    }).pipe(
+      Effect.flatMap(() => PubSub.publish(this.eventBus, event)),
+      Effect.asVoid
+    );
+  }
+};
+var DETAILS_PAGE_SIZE = 20;
+function scoreColor(score) {
+  if (score >= 80)
+    return "green";
+  if (score >= 50)
+    return "yellow";
+  return "red";
+}
+function formatScorePart(item, scoreToColor) {
+  const def = getScoreById(item.id);
+  if (!def) {
+    const numeric = toNumericScore(item.data);
+    return numeric !== void 0 ? `${numeric.toFixed(2)}` : "n/a";
+  }
+  const formatted = def.format(item.data);
+  if (def.displayStrategy === "bar") {
+    const numeric = typeof item.data === "object" && item.data !== null && "value" in item.data ? item.data.value : toNumericScore(item.data);
+    if (typeof numeric === "number" && Number.isFinite(numeric)) {
+      const barWidth = 14;
+      const filled = Math.round(numeric / 100 * barWidth);
+      const bar = "\u2588".repeat(filled) + "\u2591".repeat(barWidth - filled);
+      return `${formatted} ${bar}`;
     }
-    const includedTagsMatch = !query.includedTags || query.includedTags.length === 0 || tags.some((tag) => matchesAny(tag, query.includedTags));
-    const includedPathsMatch = !query.includedPaths || query.includedPaths.length === 0 || matchesPath(item.filePath, query.includedPaths);
-    return includedTagsMatch && includedPathsMatch;
-  });
+  }
+  return formatted;
 }
-
-// src/runner/api.ts
-function parseRegexLiteral(pattern) {
-  if (!pattern.startsWith("/")) {
-    return void 0;
+function CheckRow({
+  name,
+  passed,
+  detail
+}) {
+  const status = passed ? "PASSED" : "FAILED";
+  const color = passed ? "green" : "red";
+  return /* @__PURE__ */ jsxs(Text, { children: [
+    /* @__PURE__ */ jsx(Text, { color: "gray", children: name.padEnd(14) }),
+    " ",
+    /* @__PURE__ */ jsx(Text, { color, bold: true, children: status }),
+    detail ? /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
+      "  (",
+      detail,
+      ")"
+    ] }) : null
+  ] });
+}
+function buildDetailRows(run, testCases, evaluatorNameById) {
+  const { performance, dimensions, checks, failures, meta } = run;
+  const latencyHistory = performance.latencyHistoryMs ?? [
+    performance.latencyAvgMs - 40,
+    performance.latencyAvgMs - 10,
+    performance.latencyAvgMs + 20,
+    performance.latencyP95Ms - 80,
+    performance.latencyP95Ms
+  ];
+  const rows = [
+    /* @__PURE__ */ jsx(SectionHeader, { children: "Meta" }, "meta-h"),
+    /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
+      "Model: ",
+      meta.model,
+      "   Provider: ",
+      meta.provider
+    ] }, "meta-1"),
+    /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
+      "Commit: ",
+      meta.commit,
+      "  Branch: ",
+      meta.branch,
+      "  Seed: ",
+      meta.seed
+    ] }, "meta-2"),
+    /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
+      "Duration: ",
+      meta.duration,
+      "   Concurrency: ",
+      meta.concurrency
+    ] }, "meta-3"),
+    /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
+      "Artifact: ",
+      meta.artifact
+    ] }, "meta-4"),
+    /* @__PURE__ */ jsx(Text, { children: " " }, "sp1"),
+    /* @__PURE__ */ jsx(SectionHeader, { children: "Scores (0\u2013100)" }, "scores-h"),
+    ...dimensions.map((d) => /* @__PURE__ */ jsx(TextBar, { label: d.name, value: d.score }, `dim-${d.name}`)),
+    /* @__PURE__ */ jsx(Text, { children: " " }, "sp2"),
+    /* @__PURE__ */ jsx(SectionHeader, { children: "Checks (boolean)" }, "checks-h"),
+    ...checks.map((c) => /* @__PURE__ */ jsx(CheckRow, { name: c.name, passed: c.passed, detail: c.detail }, `chk-${c.name}`)),
+    /* @__PURE__ */ jsx(Text, { children: " " }, "sp3"),
+    /* @__PURE__ */ jsx(SectionHeader, { children: "Performance" }, "perf-h"),
+    /* @__PURE__ */ jsx(
+      TextBar,
+      {
+        label: "pass rate",
+        value: performance.passRate,
+        format: (v) => `${v}%`
+      },
+      "perf-rate"
+    ),
+    /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
+      "latency avg     ",
+      performance.latencyAvgMs,
+      "ms   p95 ",
+      performance.latencyP95Ms,
+      "ms"
+    ] }, "perf-lat"),
+    /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
+      "tokens avg      ",
+      performance.tokensAvg,
+      "   p95 ",
+      performance.tokensP95
+    ] }, "perf-tok"),
+    /* @__PURE__ */ jsx(Text, { children: " " }, "sp4"),
+    /* @__PURE__ */ jsx(SectionHeader, { children: "Latency trend" }, "spark-h"),
+    /* @__PURE__ */ jsx(Sparkline, { data: latencyHistory, width: 20 }, "spark")
+  ];
+  if (failures.length > 0) {
+    rows.push(/* @__PURE__ */ jsx(Text, { children: " " }, "sp5"));
+    rows.push(/* @__PURE__ */ jsx(SectionHeader, { children: "Failures (top)" }, "fail-h"));
+    failures.forEach((f, i) => {
+      rows.push(
+        /* @__PURE__ */ jsxs(Text, { color: "red", children: [
+          i + 1,
+          ") ",
+          f.title
+        ] }, `fail-${i}`)
+      );
+    });
   }
-  const lastSlash = pattern.lastIndexOf("/");
-  if (lastSlash <= 0) {
-    return void 0;
+  if (testCases.length > 0) {
+    rows.push(/* @__PURE__ */ jsx(Text, { children: " " }, "sp6"));
+    rows.push(/* @__PURE__ */ jsx(SectionHeader, { children: "Test cases" }, "tc-h"));
+    for (const tc of testCases) {
+      const rerunPart = tc.rerunTotal != null && tc.rerunIndex != null ? ` (${tc.rerunIndex}/${tc.rerunTotal})` : "";
+      rows.push(
+        /* @__PURE__ */ jsxs(Text, { children: [
+          /* @__PURE__ */ jsxs(Text, { color: "cyan", children: [
+            "[",
+            tc.completedTestCases,
+            "/",
+            tc.totalTestCases,
+            "]"
+          ] }),
+          " ",
+          tc.testCaseName,
+          rerunPart ? /* @__PURE__ */ jsx(Text, { color: "cyan", children: rerunPart }) : null,
+          /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
+            " (",
+            tc.durationMs,
+            "ms)"
+          ] })
+        ] }, `tc-${tc.testCaseId}-${tc.rerunIndex ?? 0}`)
+      );
+      for (const item of tc.evaluatorScores) {
+        const name = evaluatorNameById.get(item.evaluatorId) ?? item.evaluatorId;
+        rows.push(
+          /* @__PURE__ */ jsxs(Text, { children: [
+            "   ",
+            name,
+            ":",
+            " ",
+            /* @__PURE__ */ jsx(Text, { color: item.passed ? "green" : "red", bold: true, children: item.passed ? "PASS" : "FAIL" }),
+            " ",
+            item.scores.map((s) => /* @__PURE__ */ jsxs(Text, { color: scoreColor(toNumericScore(s.data) ?? 0), children: [
+              formatScorePart(s),
+              " "
+            ] }, s.id)),
+            item.metrics?.map((m) => {
+              const def = getMetricById(m.id);
+              if (!def)
+                return null;
+              const formatted = def.format(m.data);
+              return /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
+                "[",
+                def.name ? `${def.name}: ` : "",
+                formatted,
+                "]",
+                " "
+              ] }, m.id);
+            })
+          ] }, `tc-${tc.testCaseId}-${item.evaluatorId}`)
+        );
+        if (!item.passed && item.logs && item.logs.length > 0) {
+          for (let logIdx = 0; logIdx < item.logs.length; logIdx++) {
+            const log = item.logs[logIdx];
+            if (log.type === "diff") {
+              const lines = getDiffLines(log);
+              for (let lineIdx = 0; lineIdx < lines.length; lineIdx++) {
+                const { type, line } = lines[lineIdx];
+                rows.push(
+                  /* @__PURE__ */ jsxs(
+                    Text,
+                    {
+                      color: type === "remove" ? "red" : type === "add" ? "green" : "gray",
+                      children: [
+                        "      ",
+                        line
+                      ]
+                    },
+                    `tc-${tc.testCaseId}-${item.evaluatorId}-${logIdx}-${lineIdx}`
+                  )
+                );
+              }
+            }
+          }
+        }
+      }
+    }
   }
-  return {
-    source: pattern.slice(1, lastSlash),
-    flags: pattern.slice(lastSlash + 1)
-  };
+  return rows;
 }
-function createNameMatcher(pattern) {
-  const normalizedPattern = pattern.trim();
-  const regexLiteral = parseRegexLiteral(normalizedPattern);
-  if (regexLiteral) {
-    const regex = new RegExp(regexLiteral.source, regexLiteral.flags);
-    return (value) => regex.test(value);
-  }
-  if (normalizedPattern.includes("*")) {
-    const escaped = normalizedPattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
-    const regex = new RegExp(`^${escaped}$`, "i");
-    return (value) => regex.test(value);
+function RunDetailsView({
+  state,
+  dataset,
+  selectedRun,
+  evaluators
+}) {
+  const runs = dataset?.runs ?? [];
+  const rightFocused = state.focus === "right";
+  const [testCases, setTestCases] = useState([]);
+  const evaluatorNameById = React.useMemo(
+    () => new Map(evaluators.map((e) => [e.id, e.name])),
+    [evaluators]
+  );
+  useEffect(() => {
+    if (!selectedRun?.meta?.artifact) {
+      setTestCases([]);
+      return;
+    }
+    const artifactPath = resolve(selectedRun.meta.artifact);
+    parseArtifactFile(artifactPath).then(setTestCases);
+  }, [selectedRun?.meta?.artifact]);
+  if (!selectedRun) {
+    return /* @__PURE__ */ jsxs(Fragment, { children: [
+      /* @__PURE__ */ jsx(RunsSidebar, { state, dataset, runs }),
+      /* @__PURE__ */ jsx(Pane, { flexGrow: 1, marginLeft: 1, focused: rightFocused, children: /* @__PURE__ */ jsx(Text, { color: "gray", children: "Select a run to inspect details." }) })
+    ] });
   }
-  return (value) => value.toLowerCase() === normalizedPattern.toLowerCase();
+  const rows = buildDetailRows(selectedRun, testCases, evaluatorNameById);
+  const offset = Math.max(0, state.detailsScrollOffset);
+  const visible = rows.slice(offset, offset + DETAILS_PAGE_SIZE);
+  return /* @__PURE__ */ jsxs(Fragment, { children: [
+    /* @__PURE__ */ jsx(RunsSidebar, { state, dataset, runs }),
+    /* @__PURE__ */ jsx(Pane, { flexGrow: 1, marginLeft: 1, focused: rightFocused, children: /* @__PURE__ */ jsx(Box, { flexDirection: "column", children: visible.map((row, i) => /* @__PURE__ */ jsx(React.Fragment, { children: row }, i)) }) })
+  ] });
 }
-function mergeRunnerOverrides(base, next) {
-  if (!base) {
-    return next;
-  }
-  if (!next) {
-    return base;
-  }
-  const discovery = base.discovery || next.discovery ? {
-    ...base.discovery ?? {},
-    ...next.discovery ?? {}
-  } : void 0;
+var LEFT_PANE_WIDTH3 = 44;
+function NewEvaluationView({
+  state,
+  data,
+  visibleEvaluators
+}) {
+  const selectedCount = state.selectedEvaluatorIds.length;
+  const focusedEvaluator = visibleEvaluators[state.evaluatorMenuIndex];
+  const leftFocused = state.focus === "left";
+  const rightFocused = state.focus === "right";
+  return /* @__PURE__ */ jsxs(Fragment, { children: [
+    /* @__PURE__ */ jsxs(Pane, { width: LEFT_PANE_WIDTH3, focused: leftFocused, children: [
+      /* @__PURE__ */ jsx(SectionHeader, { children: "Available Evaluators" }),
+      /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
+        "Search: ",
+        state.searchQuery || "(none)"
+      ] }),
+      visibleEvaluators.map((evaluator, index) => {
+        const selected = index === state.evaluatorMenuIndex;
+        const inSelection = state.selectedEvaluatorIds.includes(evaluator.id);
+        return /* @__PURE__ */ jsxs(
+          Text,
+          {
+            color: selected ? "cyan" : "gray",
+            bold: selected,
+            children: [
+              selected ? "\u25B8 " : "  ",
+              inSelection ? "[x] " : "[ ] ",
+              evaluator.name
+            ]
+          },
+          evaluator.id
+        );
+      })
+    ] }),
+    /* @__PURE__ */ jsxs(Pane, { flexGrow: 1, marginLeft: 1, focused: rightFocused, children: [
+      /* @__PURE__ */ jsxs(SectionHeader, { children: [
+        "Selected (",
+        selectedCount,
+        ")"
+      ] }),
+      state.selectedEvaluatorIds.map((id, index) => {
+        const evaluator = data.evaluators.find((item) => item.id === id);
+        if (!evaluator)
+          return null;
+        return /* @__PURE__ */ jsxs(Text, { children: [
+          index + 1,
+          ") ",
+          evaluator.name
+        ] }, id);
+      }),
+      /* @__PURE__ */ jsx(SectionHeader, { children: "Config preview" }),
+      /* @__PURE__ */ jsx(Text, { color: "gray", children: focusedEvaluator?.configPreview ?? "Select an evaluator to inspect config." })
+    ] })
+  ] });
+}
+function clampCursor(state, filteredDatasetsLength, selectedRunCount) {
+  const datasetMax = filteredDatasetsLength;
+  const runMax = selectedRunCount;
+  const evaluatorMax = 3;
   return {
-    ...base,
-    ...next,
-    discovery
+    ...state,
+    datasetMenuIndex: Math.max(0, Math.min(state.datasetMenuIndex, datasetMax)),
+    runMenuIndex: Math.max(0, Math.min(state.runMenuIndex, runMax)),
+    evaluatorMenuIndex: Math.max(
+      0,
+      Math.min(state.evaluatorMenuIndex, evaluatorMax)
+    )
   };
 }
-function createRunner(overrides) {
-  const fileOverrides = loadRunnerConfigFile();
-  const merged = mergeRunnerOverrides(fileOverrides, overrides);
-  return new EffectRunner(withRunnerConfig(merged));
-}
-var EffectRunner = class {
-  constructor(config) {
-    this.eventBus = Effect.runSync(PubSub.unbounded());
-    this.runQueue = Effect.runSync(Queue.unbounded());
-    this.persistenceQueue = Effect.runSync(
-      Queue.unbounded()
-    );
-    this.snapshots = /* @__PURE__ */ new Map();
-    this.listeners = /* @__PURE__ */ new Set();
-    this.datasetsById = /* @__PURE__ */ new Map();
-    this.evaluatorsById = /* @__PURE__ */ new Map();
-    this.schedulerFiber = Effect.runFork(
-      this.createSchedulerEffect()
-    );
-    this.persistenceFiber = Effect.runFork(
-      createPersistenceWorker(this.persistenceQueue)
-    );
-    this.config = config;
-  }
-  async collectDatasets() {
-    const datasets = await collectDatasetsFromFiles(this.config.discovery);
-    this.datasetsById.clear();
-    for (const dataset of datasets) {
-      this.datasetsById.set(dataset.id, dataset);
+function EvalsCliApp({
+  data,
+  args,
+  runner
+}) {
+  const { exit } = useApp();
+  const { width: stdoutWidth, height: stdoutHeight } = useScreenSize();
+  const [liveData, setLiveData] = useState(data);
+  const [runtimeMessage, setRuntimeMessage] = useState();
+  const [state, dispatch] = useReducer(
+    reduceCliState,
+    createInitialState(data, args)
+  );
+  useEffect(() => {
+    setLiveData(data);
+  }, [data]);
+  useEffect(() => {
+    if (!runner) {
+      return void 0;
     }
-    return datasets;
-  }
-  async collectEvaluators() {
-    const evaluators = await collectEvaluatorsFromFiles(this.config.discovery);
-    this.evaluatorsById.clear();
-    for (const evaluator of evaluators) {
-      this.evaluatorsById.set(evaluator.id, evaluator);
+    return runner.subscribeRunEvents((event) => {
+      setLiveData((current) => applyRunnerEvent(current, event, runner));
+      if (event.type === "RunQueued") {
+        setRuntimeMessage(`Queued ${event.runId} with ${event.totalTestCases} test cases.`);
+      }
+      if (event.type === "RunCompleted") {
+        setRuntimeMessage(
+          `Completed ${event.runId}: ${event.passedTestCases}/${event.totalTestCases} passed.`
+        );
+      }
+      if (event.type === "RunFailed") {
+        setRuntimeMessage(`Run failed: ${event.errorMessage}`);
+      }
+    });
+  }, [runner]);
+  const filteredDatasets = useMemo(
+    () => getFilteredDatasets(liveData, state.searchQuery),
+    [liveData, state.searchQuery]
+  );
+  const clampedState = clampCursor(
+    state,
+    filteredDatasets.length,
+    getDatasetByMenuIndex(filteredDatasets, state.datasetMenuIndex)?.runs.length ?? 0
+  );
+  const selectedDataset = getDatasetByMenuIndex(
+    filteredDatasets,
+    clampedState.datasetMenuIndex
+  );
+  const selectedRun = getRunByMenuIndex(
+    selectedDataset,
+    clampedState.runMenuIndex
+  );
+  const visibleEvaluators = liveData.evaluators.filter(
+    (evaluator) => evaluator.name.toLowerCase().includes(clampedState.searchQuery.toLowerCase())
+  );
+  useInput((input, key) => {
+    if (isQuitInput(input) || key.escape) {
+      exit();
+      return;
     }
-    return evaluators;
-  }
-  async resolveDatasetByName(name) {
-    if (this.datasetsById.size === 0) {
-      await this.collectDatasets();
+    if (key.tab) {
+      dispatch({ type: "TOGGLE_FOCUS" });
+      return;
     }
-    const normalized = name.trim().toLowerCase();
-    return Array.from(this.datasetsById.values()).find(
-      (item) => item.dataset.getName().toLowerCase() === normalized
-    );
-  }
-  async resolveEvaluatorsByNamePattern(pattern) {
-    if (this.evaluatorsById.size === 0) {
-      await this.collectEvaluators();
+    if (isSearchInput(input)) {
+      dispatch({ type: "START_SEARCH" });
+      return;
     }
-    const matcher = createNameMatcher(pattern);
-    return Array.from(this.evaluatorsById.values()).filter(
-      (item) => matcher(item.evaluator.getName() ?? "")
-    );
-  }
-  async searchTestCases(query) {
-    const testCases = await collectTestCasesFromFiles(this.config.discovery);
-    return searchCollectedTestCases(testCases, query);
-  }
-  async collectDatasetTestCases(datasetId) {
-    if (this.datasetsById.size === 0) {
-      await this.collectDatasets();
+    if (clampedState.searchMode) {
+      if (key.return) {
+        dispatch({ type: "END_SEARCH" });
+        return;
+      }
+      if (isBackKey(key)) {
+        dispatch({ type: "REMOVE_SEARCH_CHAR" });
+        return;
+      }
+      if (isPrintableCharacter(input)) {
+        dispatch({ type: "APPEND_SEARCH", value: input });
+      }
+      return;
     }
-    const dataset = this.datasetsById.get(datasetId);
-    if (!dataset) {
-      throw new Error(`Unknown dataset: ${datasetId}`);
+    if (key.upArrow) {
+      const max = clampedState.level === "details" ? 100 : clampedState.level === "new-evaluation" ? visibleEvaluators.length - 1 : 100;
+      dispatch({ type: "MOVE_UP", max });
+      return;
     }
-    const allTestCases = await collectTestCasesFromFiles(this.config.discovery);
-    return allTestCases.filter(
-      (testCase) => dataset.dataset.matchesTestCase(testCase.testCase, testCase.filePath)
-    );
-  }
-  async runDatasetWith(request) {
-    if (this.datasetsById.size === 0) {
-      await this.collectDatasets();
+    if (key.downArrow) {
+      const max = clampedState.level === "datasets" ? filteredDatasets.length : clampedState.level === "runs" ? selectedDataset?.runs.length ?? 0 : clampedState.level === "new-evaluation" ? Math.max(0, visibleEvaluators.length - 1) : 100;
+      dispatch({ type: "MOVE_DOWN", max });
+      return;
     }
-    if (this.evaluatorsById.size === 0) {
-      await this.collectEvaluators();
+    if (key.return) {
+      dispatch({
+        type: "ENTER",
+        hasDataset: Boolean(selectedDataset),
+        hasRun: Boolean(selectedRun)
+      });
+      if (clampedState.level === "new-evaluation") {
+        const evaluator = visibleEvaluators[clampedState.evaluatorMenuIndex];
+        if (evaluator) {
+          dispatch({ type: "TOGGLE_EVALUATOR", evaluatorId: evaluator.id });
+        }
+      }
+      return;
     }
-    const dataset = this.datasetsById.get(request.datasetId);
-    if (!dataset) {
-      throw new Error(`Unknown dataset: ${request.datasetId}`);
+    if (isBackKey(key)) {
+      dispatch({ type: "BACK" });
+      return;
     }
-    const selectedEvaluators = request.evaluatorIds.map((id) => this.evaluatorsById.get(id)).filter((value) => Boolean(value)).map((value) => ({ id: value.id, evaluator: value.evaluator }));
-    if (selectedEvaluators.length === 0) {
-      throw new Error("No evaluators selected for run");
+    if (input.toLowerCase() === "c") {
+      dispatch({ type: "CLEAR_WARNINGS" });
+      setRuntimeMessage(void 0);
+      return;
     }
-    const selectedTestCases = await this.collectDatasetTestCases(request.datasetId);
-    const runId = `run-${randomUUID()}`;
-    const artifactPath = createArtifactPath(
-      this.config.artifactDirectory,
-      request.datasetId,
-      runId
-    );
-    const snapshot = {
-      runId,
-      datasetId: request.datasetId,
-      datasetName: dataset.dataset.getName(),
-      evaluatorIds: selectedEvaluators.map((item) => item.id),
-      queuedAt: Date.now(),
-      totalTestCases: selectedTestCases.length,
-      completedTestCases: 0,
-      passedTestCases: 0,
-      failedTestCases: 0,
-      status: "queued",
-      artifactPath
-    };
-    this.snapshots.set(runId, snapshot);
-    const queuedEvent = {
-      type: "RunQueued",
-      runId,
-      datasetId: request.datasetId,
-      datasetName: dataset.dataset.getName(),
-      evaluatorIds: selectedEvaluators.map((item) => item.id),
-      totalTestCases: selectedTestCases.length,
-      artifactPath
-    };
-    await Effect.runPromise(this.publishEvent(queuedEvent));
-    await Effect.runPromise(
-      Queue.offer(this.persistenceQueue, {
-        runId,
-        artifactPath,
-        payload: queuedEvent
-      })
-    );
-    await Effect.runPromise(
-      Queue.offer(this.runQueue, {
-        runId,
-        datasetId: request.datasetId,
-        dataset: dataset.dataset,
-        evaluators: selectedEvaluators,
-        testCases: selectedTestCases,
-        snapshot
-      })
-    );
-    return snapshot;
-  }
-  subscribeRunEvents(listener, options) {
-    const entry = { runId: options?.runId, listener };
-    this.listeners.add(entry);
-    return () => {
-      this.listeners.delete(entry);
-    };
-  }
-  getRunSnapshot(runId) {
-    return this.snapshots.get(runId);
-  }
-  getAllRunSnapshots() {
-    return Array.from(this.snapshots.values()).sort(
-      (a, b) => b.queuedAt - a.queuedAt
-    );
-  }
-  async shutdown() {
-    await Effect.runPromise(Fiber.interrupt(this.schedulerFiber));
-    await Effect.runPromise(Fiber.interrupt(this.persistenceFiber));
-    await Effect.runPromise(Queue.shutdown(this.runQueue));
-    await Effect.runPromise(Queue.shutdown(this.persistenceQueue));
-    await Effect.runPromise(PubSub.shutdown(this.eventBus));
-  }
-  createSchedulerEffect() {
-    const self = this;
-    return Effect.forever(
-      Effect.gen(function* () {
-        const task = yield* Queue.take(self.runQueue);
-        yield* Effect.fork(
-          executeRunTask(
-            task,
-            self.publishEvent.bind(self),
-            self.persistenceQueue,
-            self.updateSnapshot.bind(self)
-          )
+    if (input.toLowerCase() === "s" && clampedState.level === "new-evaluation") {
+      if (!runner) {
+        setRuntimeMessage("Runner unavailable: cannot start evaluation.");
+        return;
+      }
+      if (!selectedDataset) {
+        setRuntimeMessage("Select a dataset before starting a new evaluation.");
+        return;
+      }
+      if (clampedState.selectedEvaluatorIds.length === 0) {
+        setRuntimeMessage("Select at least one evaluator before starting.");
+        return;
+      }
+      void runner.runDatasetWith({
+        datasetId: selectedDataset.id,
+        evaluatorIds: clampedState.selectedEvaluatorIds
+      }).then((snapshot) => {
+        setRuntimeMessage(
+          `Started ${snapshot.runId} on ${selectedDataset.name} (${snapshot.totalTestCases} cases).`
         );
-      })
-    );
-  }
-  updateSnapshot(runId, updater) {
-    const existing = this.snapshots.get(runId);
-    if (!existing) {
-      return;
+      }).catch((error) => {
+        setRuntimeMessage(
+          error instanceof Error ? error.message : "Failed to start evaluation."
+        );
+      });
     }
-    this.snapshots.set(runId, updater(existing));
-  }
-  publishEvent(event) {
-    return Effect.sync(() => {
-      for (const entry of this.listeners) {
-        if (entry.runId && entry.runId !== event.runId) {
-          continue;
+  });
+  const renderContent = () => {
+    if (clampedState.level === "new-evaluation") {
+      return /* @__PURE__ */ jsx(
+        NewEvaluationView,
+        {
+          state: clampedState,
+          data: liveData,
+          visibleEvaluators
         }
-        entry.listener(event);
+      );
+    }
+    if (clampedState.level === "datasets") {
+      return /* @__PURE__ */ jsx(
+        DatasetsView,
+        {
+          state: clampedState,
+          filteredDatasets,
+          selectedDataset
+        }
+      );
+    }
+    if (clampedState.level === "runs") {
+      return /* @__PURE__ */ jsx(
+        RunsView,
+        {
+          state: clampedState,
+          dataset: selectedDataset,
+          selectedRun
+        }
+      );
+    }
+    return /* @__PURE__ */ jsx(
+      RunDetailsView,
+      {
+        state: clampedState,
+        dataset: selectedDataset,
+        selectedRun,
+        evaluators: liveData.evaluators
       }
-    }).pipe(
-      Effect.flatMap(() => PubSub.publish(this.eventBus, event)),
-      Effect.asVoid
     );
-  }
-};
+  };
+  return /* @__PURE__ */ jsxs(
+    Box,
+    {
+      flexDirection: "column",
+      flexGrow: 1,
+      width: stdoutWidth,
+      height: stdoutHeight,
+      children: [
+        /* @__PURE__ */ jsx(
+          Box,
+          {
+            borderStyle: "round",
+            borderColor: "cyan",
+            paddingX: 1,
+            width: stdoutWidth,
+            children: /* @__PURE__ */ jsx(Text, { children: getBreadcrumbText(
+              clampedState,
+              selectedDataset?.name,
+              selectedRun?.label
+            ) })
+          }
+        ),
+        clampedState.startupWarnings.length > 0 && /* @__PURE__ */ jsxs(
+          Box,
+          {
+            marginTop: 1,
+            borderStyle: "round",
+            borderColor: "yellow",
+            paddingX: 1,
+            flexDirection: "column",
+            width: stdoutWidth,
+            children: [
+              /* @__PURE__ */ jsx(Text, { color: "yellow", children: "Startup warnings:" }),
+              clampedState.startupWarnings.map((warning, index) => /* @__PURE__ */ jsx(Text, { children: warning }, `${warning}-${index}`))
+            ]
+          }
+        ),
+        clampedState.searchMode && /* @__PURE__ */ jsxs(
+          Box,
+          {
+            marginTop: 1,
+            borderStyle: "round",
+            borderColor: "magenta",
+            paddingX: 1,
+            width: stdoutWidth,
+            children: [
+              /* @__PURE__ */ jsx(Text, { color: "magenta", bold: true, children: "Search: " }),
+              /* @__PURE__ */ jsx(Text, { color: "white", children: clampedState.searchQuery })
+            ]
+          }
+        ),
+        runtimeMessage && /* @__PURE__ */ jsx(
+          Box,
+          {
+            marginTop: 1,
+            borderStyle: "round",
+            borderColor: "blue",
+            paddingX: 1,
+            width: stdoutWidth,
+            children: /* @__PURE__ */ jsx(Text, { color: "blue", children: runtimeMessage })
+          }
+        ),
+        /* @__PURE__ */ jsx(
+          Box,
+          {
+            marginTop: 1,
+            flexGrow: 1,
+            width: stdoutWidth,
+            flexDirection: "row",
+            children: renderContent()
+          }
+        ),
+        /* @__PURE__ */ jsx(Box, { marginTop: 1, paddingX: 1, children: /* @__PURE__ */ jsx(Text, { color: "gray", children: getFooterText(clampedState) }) })
+      ]
+    }
+  );
+}
 async function main() {
   const args = parseStartupArgs(process.argv.slice(2));
   const runner = createRunner();