@m4trix/evals 0.1.0

package/dist/cli.js

@@ -0,0 +1,1974 @@
+#!/usr/bin/env node
+import { withFullScreen, useScreenSize } from 'fullscreen-ink';
+import React, { useState, useReducer, useEffect, useMemo } from 'react';
+import { useApp, useInput, Box, Text } from 'ink';
+import { jsx, jsxs, Fragment } from 'react/jsx-runtime';
+import { randomUUID } from 'crypto';
+import { Effect, PubSub, Queue, Fiber } from 'effect';
+import { mkdir, appendFile, readdir } from 'fs/promises';
+import { relative, join, dirname, resolve } from 'path';
+import { pathToFileURL } from 'url';
+
+var SEP = "  ";
+var ARROW = "\u203A";
+function getBreadcrumbText(state, datasetName, runLabel) {
+  const dim = (s, k) => /* @__PURE__ */ jsx(Text, { color: "gray", children: s }, k ?? s);
+  const accent = (s) => /* @__PURE__ */ jsx(Text, { color: "cyan", bold: true, children: s }, s);
+  if (state.level === "datasets") {
+    return /* @__PURE__ */ jsxs(Fragment, { children: [
+      dim("Evaluations"),
+      SEP,
+      dim(ARROW, "a1"),
+      SEP,
+      accent("Datasets")
+    ] });
+  }
+  if (state.level === "runs") {
+    return /* @__PURE__ */ jsxs(Fragment, { children: [
+      dim("Evaluations"),
+      SEP,
+      dim(ARROW, "a1"),
+      SEP,
+      dim("Dataset:"),
+      " ",
+      /* @__PURE__ */ jsx(Text, { color: "white", children: datasetName ?? "-" }, "ds"),
+      SEP,
+      dim(ARROW, "a2"),
+      SEP,
+      accent("Runs")
+    ] });
+  }
+  if (state.level === "details") {
+    return /* @__PURE__ */ jsxs(Fragment, { children: [
+      dim("Evaluations"),
+      SEP,
+      dim(ARROW, "a1"),
+      SEP,
+      dim("Dataset:"),
+      " ",
+      /* @__PURE__ */ jsx(Text, { color: "white", children: datasetName ?? "-" }, "ds"),
+      SEP,
+      dim(ARROW, "a2"),
+      SEP,
+      dim("Run:"),
+      " ",
+      /* @__PURE__ */ jsx(Text, { color: "white", children: runLabel ?? "-" }, "rl"),
+      SEP,
+      dim(ARROW, "a3"),
+      SEP,
+      accent("Details")
+    ] });
+  }
+  return /* @__PURE__ */ jsxs(Fragment, { children: [
+    dim("Evaluations"),
+    SEP,
+    dim(ARROW, "a1"),
+    SEP,
+    accent("New evaluation"),
+    SEP,
+    dim(ARROW, "a2"),
+    SEP,
+    dim("Select evaluators", "sel")
+  ] });
+}
+
+// src/cli/components/Footer.tsx
+function getFooterText(state) {
+  if (state.level === "datasets") {
+    return "\u2191\u2193 move  Enter open  / search  Tab focus  q quit";
+  }
+  if (state.level === "runs") {
+    return "\u2191\u2193 move  Enter details  Backspace datasets  Tab focus  q quit";
+  }
+  if (state.level === "details") {
+    return "\u2191\u2193 scroll  Backspace runs  Tab focus  q quit";
+  }
+  return "\u2191\u2193 move  Enter add/remove  S start run  / search  Esc cancel  q quit";
+}
+function ListItem({
+  selected,
+  label,
+  itemKey
+}) {
+  return /* @__PURE__ */ jsxs(Text, { color: selected ? "cyan" : "gray", bold: selected, children: [
+    selected ? "\u25B8 " : "  ",
+    label
+  ] }, itemKey);
+}
+function Pane({
+  children,
+  width,
+  flexGrow,
+  marginLeft,
+  focused = false
+}) {
+  return /* @__PURE__ */ jsx(
+    Box,
+    {
+      flexDirection: "column",
+      width,
+      flexGrow,
+      marginLeft,
+      borderStyle: focused ? "single" : "round",
+      borderColor: focused ? "cyan" : "gray",
+      paddingX: 1,
+      children
+    }
+  );
+}
+function SectionHeader({
+  children
+}) {
+  return /* @__PURE__ */ jsx(Text, { color: "cyan", bold: true, children });
+}
+function StatusText({ status }) {
+  const color = status === "PASS" ? "green" : status === "RUNNING" ? "yellow" : "red";
+  return /* @__PURE__ */ jsxs(Text, { color, children: [
+    "(",
+    status,
+    ")"
+  ] });
+}
+var LEFT_PANE_WIDTH = 44;
+function RunsSidebar({
+  state,
+  runs
+}) {
+  const focused = state.focus === "left";
+  return /* @__PURE__ */ jsxs(Pane, { width: LEFT_PANE_WIDTH, focused, children: [
+    /* @__PURE__ */ jsx(SectionHeader, { children: "Runs" }),
+    /* @__PURE__ */ jsx(
+      ListItem,
+      {
+        selected: state.runMenuIndex === 0,
+        label: "New evaluation",
+        itemKey: "runs-new-eval"
+      }
+    ),
+    runs.map((run, index) => /* @__PURE__ */ jsxs(
+      Text,
+      {
+        color: state.runMenuIndex === index + 1 ? "cyan" : "gray",
+        bold: state.runMenuIndex === index + 1,
+        children: [
+          state.runMenuIndex === index + 1 ? "\u25B8 " : "  ",
+          run.label,
+          " ",
+          /* @__PURE__ */ jsx(StatusText, { status: run.status })
+        ]
+      },
+      run.id
+    ))
+  ] });
+}
+var BLOCKS = ["\u2581", "\u2582", "\u2583", "\u2584", "\u2585", "\u2586", "\u2587", "\u2588"];
+function Sparkline({
+  data,
+  width,
+  label
+}) {
+  if (data.length === 0)
+    return null;
+  const max = Math.max(...data);
+  const min = Math.min(...data);
+  const range = max - min || 1;
+  const targetWidth = width ?? Math.min(data.length, 24);
+  let values;
+  if (data.length <= targetWidth) {
+    values = data;
+  } else {
+    const step = data.length / targetWidth;
+    values = Array.from({ length: targetWidth }, (_, i) => {
+      const start = Math.floor(i * step);
+      const end = Math.floor((i + 1) * step);
+      const slice = data.slice(start, end);
+      return slice.reduce((a, b) => a + b, 0) / slice.length;
+    });
+  }
+  const spark = values.map((v) => {
+    const normalized = (v - min) / range;
+    const idx = Math.min(7, Math.floor(normalized * 8));
+    return BLOCKS[idx];
+  }).join("");
+  return /* @__PURE__ */ jsxs(Text, { children: [
+    label !== void 0 && label !== "" ? /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
+      label.padEnd(14),
+      " "
+    ] }) : null,
+    /* @__PURE__ */ jsx(Text, { color: "cyan", children: spark })
+  ] });
+}
+function barColor(pct) {
+  if (pct >= 70)
+    return "green";
+  if (pct >= 40)
+    return "yellow";
+  return "red";
+}
+function TextBar({
+  label,
+  value,
+  max = 100,
+  labelWidth = 14,
+  barWidth = 20,
+  format = (v) => String(v),
+  colorByValue = true
+}) {
+  const clamped = Math.max(0, Math.min(max, value));
+  const pct = max > 0 ? clamped / max * 100 : 0;
+  const filled = Math.round(clamped / max * barWidth);
+  const filledBar = "\u2588".repeat(filled);
+  const emptyBar = "\u2591".repeat(Math.max(0, barWidth - filled));
+  const color = colorByValue ? barColor(pct) : void 0;
+  return /* @__PURE__ */ jsxs(Text, { children: [
+    /* @__PURE__ */ jsx(Text, { color: "gray", children: label.padEnd(labelWidth) }),
+    " [",
+    color ? /* @__PURE__ */ jsxs(Fragment, { children: [
+      /* @__PURE__ */ jsx(Text, { color, children: filledBar }),
+      /* @__PURE__ */ jsx(Text, { color: "gray", children: emptyBar })
+    ] }) : filledBar + emptyBar,
+    "] ",
+    /* @__PURE__ */ jsx(Text, { color: color ?? "white", bold: true, children: format(value) })
+  ] });
+}
+
+// src/cli/keys.ts
+function isQuitInput(input) {
+  return input.toLowerCase() === "q";
+}
+function isSearchInput(input) {
+  return input === "/";
+}
+function isPrintableCharacter(input) {
+  return input.length === 1 && input >= " " && input !== "\x7F";
+}
+function isBackKey(key) {
+  return key.backspace || key.delete;
+}
+
+// src/cli/data.mock.json
+var data_mock_default = {
+  datasets: [
+    {
+      id: "onboarding-flows",
+      name: "onboarding-flows",
+      overview: "Evaluate first-user journeys and schema compliance for generated onboarding payloads.",
+      runs: [
+        {
+          id: "run_2026-02-17_2044",
+          label: "2026-02-17 20:44",
+          status: "FAILED",
+          performance: {
+            passRate: 96,
+            avgScore: 0.91,
+            latencyP95Ms: 710,
+            latencyAvgMs: 502,
+            tokensAvg: 171,
+            tokensP95: 230,
+            costUsd: 24e-4,
+            latencyHistoryMs: [380, 420, 510, 480, 550, 620, 590, 710, 520, 480, 530, 600]
+          },
+          dimensions: [
+            { name: "correctness", score: 82 },
+            { name: "faithfulness", score: 79 },
+            { name: "brevity", score: 68 },
+            { name: "style", score: 90 }
+          ],
+          checks: [
+            { name: "json_schema", passed: false, detail: "3 violations" },
+            { name: "tool_calls", passed: true, detail: "0 unexpected" },
+            { name: "pii_leak", passed: true },
+            { name: "jailbreak", passed: true }
+          ],
+          failures: [
+            { title: "product_parser \u203A conforms to schema (price: string)" },
+            { title: "checkout \u203A tool-call count mismatch" }
+          ],
+          meta: {
+            model: "gpt-4o-mini",
+            provider: "OpenAI",
+            commit: "2f3c1a9",
+            branch: "main",
+            seed: 42,
+            concurrency: 4,
+            duration: "00:01:12",
+            artifact: "./eval-results/run_2026-02-17.jsonl"
+          }
+        },
+        {
+          id: "run_2026-02-16_1112",
+          label: "2026-02-16 11:12",
+          status: "PASS",
+          performance: {
+            passRate: 99,
+            avgScore: 0.95,
+            latencyP95Ms: 650,
+            latencyAvgMs: 488,
+            tokensAvg: 168,
+            tokensP95: 220,
+            costUsd: 2e-3,
+            latencyHistoryMs: [420, 450, 480, 460, 520, 490, 510, 650, 440, 470, 500, 480]
+          },
+          dimensions: [
+            { name: "correctness", score: 89 },
+            { name: "faithfulness", score: 88 },
+            { name: "brevity", score: 72 },
+            { name: "style", score: 93 }
+          ],
+          checks: [
+            { name: "json_schema", passed: true, detail: "0 violations" },
+            { name: "tool_calls", passed: true, detail: "0 unexpected" },
+            { name: "pii_leak", passed: true },
+            { name: "jailbreak", passed: true }
+          ],
+          failures: [],
+          meta: {
+            model: "gpt-4o-mini",
+            provider: "OpenAI",
+            commit: "0d24f8f",
+            branch: "main",
+            seed: 42,
+            concurrency: 4,
+            duration: "00:01:06",
+            artifact: "./eval-results/run_2026-02-16.jsonl"
+          }
+        },
+        {
+          id: "run_2026-02-15_0921",
+          label: "2026-02-15 09:21",
+          status: "PASS",
+          performance: {
+            passRate: 98,
+            avgScore: 0.93,
+            latencyP95Ms: 680,
+            latencyAvgMs: 495,
+            tokensAvg: 175,
+            tokensP95: 235,
+            costUsd: 22e-4,
+            latencyHistoryMs: [450, 480, 520, 490, 550, 580, 620, 680, 510, 470, 530, 560]
+          },
+          dimensions: [
+            { name: "correctness", score: 86 },
+            { name: "faithfulness", score: 84 },
+            { name: "brevity", score: 70 },
+            { name: "style", score: 91 }
+          ],
+          checks: [
+            { name: "json_schema", passed: true, detail: "0 violations" },
+            { name: "tool_calls", passed: true, detail: "0 unexpected" },
+            { name: "pii_leak", passed: true },
+            { name: "jailbreak", passed: true }
+          ],
+          failures: [],
+          meta: {
+            model: "gpt-4o-mini",
+            provider: "OpenAI",
+            commit: "a1b2c3d",
+            branch: "main",
+            seed: 42,
+            concurrency: 4,
+            duration: "00:01:08",
+            artifact: "./eval-results/run_2026-02-15.jsonl"
+          }
+        }
+      ]
+    },
+    {
+      id: "tool-calls",
+      name: "tool-calls",
+      overview: "Validate function-call conformance and unexpected tool invocation behavior.",
+      runs: [
+        {
+          id: "run_2026-02-14_1530",
+          label: "2026-02-14 15:30",
+          status: "PASS",
+          performance: {
+            passRate: 100,
+            avgScore: 1,
+            latencyP95Ms: 320,
+            latencyAvgMs: 280,
+            tokensAvg: 45,
+            tokensP95: 62,
+            costUsd: 8e-4,
+            latencyHistoryMs: [250, 270, 290, 280, 310, 320, 265, 290, 300, 275]
+          },
+          dimensions: [
+            { name: "contract_match", score: 100 },
+            { name: "arg_validity", score: 100 }
+          ],
+          checks: [
+            { name: "tool_calls", passed: true, detail: "0 unexpected" }
+          ],
+          failures: [],
+          meta: {
+            model: "gpt-4o-mini",
+            provider: "OpenAI",
+            commit: "e4f5g6h",
+            branch: "feat/tools",
+            seed: 42,
+            concurrency: 8,
+            duration: "00:00:45",
+            artifact: "./eval-results/tool-calls_2026-02-14.jsonl"
+          }
+        }
+      ]
+    },
+    {
+      id: "json-schema",
+      name: "json-schema",
+      overview: "Stress-test schema fidelity across generated extraction payloads.",
+      runs: []
+    }
+  ],
+  evaluators: [
+    { id: "json-schema-validator", name: "JSON Schema Validator", configPreview: "strict=true" },
+    { id: "tool-call-contract-checker", name: "Tool-call Contract Checker", configPreview: "unexpectedCalls=error" },
+    { id: "rubric-judge", name: "Rubric Judge (LLM)", configPreview: "model=gpt-4o-mini; scale=0-100" },
+    { id: "pii-leak-detector", name: "PII Leak Detector", configPreview: "redact=false" }
+  ]
+};
+
+// src/cli/state.ts
+function loadMockData() {
+  return data_mock_default;
+}
+function toSlug(input) {
+  return input.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
+}
+function toEvalRun(snapshot) {
+  const total = snapshot.totalTestCases === 0 ? 1 : snapshot.totalTestCases;
+  const passRate = Math.round(snapshot.passedTestCases / total * 100);
+  const avgScore = snapshot.passedTestCases / total;
+  const durationMs = snapshot.finishedAt ? snapshot.finishedAt - (snapshot.startedAt ?? snapshot.queuedAt) : Date.now() - (snapshot.startedAt ?? snapshot.queuedAt);
+  return {
+    id: snapshot.runId,
+    label: snapshot.runId.slice(0, 12),
+    status: snapshot.status === "completed" ? "PASS" : snapshot.status === "failed" ? "FAILED" : "RUNNING",
+    performance: {
+      passRate,
+      avgScore,
+      latencyP95Ms: Math.max(1, Math.floor(durationMs / Math.max(1, total))),
+      latencyAvgMs: Math.max(1, Math.floor(durationMs / Math.max(1, total))),
+      tokensAvg: 0,
+      tokensP95: 0,
+      costUsd: 0,
+      latencyHistoryMs: [durationMs]
+    },
+    dimensions: [
+      { name: "passed", score: Math.round(snapshot.passedTestCases / total * 100) },
+      { name: "failed", score: Math.round(snapshot.failedTestCases / total * 100) }
+    ],
+    checks: [
+      {
+        name: "run_status",
+        passed: snapshot.status === "completed",
+        detail: snapshot.status
+      }
+    ],
+    failures: snapshot.errorMessage && snapshot.errorMessage.length > 0 ? [{ title: snapshot.errorMessage }] : [],
+    meta: {
+      model: "n/a",
+      provider: "runner",
+      commit: "local",
+      branch: "local",
+      seed: 0,
+      concurrency: 1,
+      duration: `${durationMs}ms`,
+      artifact: snapshot.artifactPath
+    }
+  };
+}
+function toEvalDataset(item, snapshots) {
+  const runs = snapshots.filter((snapshot) => snapshot.datasetId === item.id).sort((a, b) => b.queuedAt - a.queuedAt).map(toEvalRun);
+  return {
+    id: item.id,
+    name: item.dataset.getName(),
+    overview: `Discovered from ${item.filePath}`,
+    runs
+  };
+}
+function toEvaluatorOption(item) {
+  return {
+    id: item.id,
+    name: item.evaluator.getName() ?? toSlug(item.id),
+    configPreview: `Source: ${item.filePath}`
+  };
+}
+async function loadRunnerData(runner) {
+  const [datasets, evaluators] = await Promise.all([
+    runner.collectDatasets(),
+    runner.collectEvaluators()
+  ]);
+  const snapshots = runner.getAllRunSnapshots();
+  if (datasets.length === 0 && evaluators.length === 0) {
+    return loadMockData();
+  }
+  return {
+    datasets: datasets.map((dataset) => toEvalDataset(dataset, snapshots)),
+    evaluators: evaluators.map(toEvaluatorOption)
+  };
+}
+function applyRunnerEvent(data, event, runner) {
+  const snapshot = runner.getRunSnapshot(event.runId);
+  if (!snapshot) {
+    return data;
+  }
+  const dataset = data.datasets.find((item) => item.id === snapshot.datasetId);
+  if (!dataset) {
+    return data;
+  }
+  const run = toEvalRun(snapshot);
+  const hasRun = dataset.runs.some((item) => item.id === run.id);
+  const nextRuns = hasRun ? dataset.runs.map((item) => item.id === run.id ? run : item) : [run, ...dataset.runs];
+  return {
+    ...data,
+    datasets: data.datasets.map(
+      (item) => item.id === dataset.id ? { ...item, runs: nextRuns } : item
+    )
+  };
+}
+function parseStartupArgs(argv) {
+  const args = { unknownArgs: [] };
+  for (let index = 0; index < argv.length; index += 1) {
+    const token = argv[index];
+    if (token === "--dataset" && argv[index + 1]) {
+      args.datasetId = argv[index + 1];
+      index += 1;
+      continue;
+    }
+    if (token === "--run" && argv[index + 1]) {
+      args.runId = argv[index + 1];
+      index += 1;
+      continue;
+    }
+    if (token === "--search" && argv[index + 1]) {
+      args.search = argv[index + 1];
+      index += 1;
+      continue;
+    }
+    args.unknownArgs.push(token);
+  }
+  return args;
+}
+function getFilteredDatasets(data, searchQuery) {
+  const query = searchQuery.trim().toLowerCase();
+  if (!query) {
+    return data.datasets;
+  }
+  return data.datasets.filter((dataset) => dataset.name.toLowerCase().includes(query));
+}
+function getDatasetByMenuIndex(datasets, menuIndex) {
+  if (menuIndex <= 0) {
+    return void 0;
+  }
+  return datasets[menuIndex - 1];
+}
+function getRunByMenuIndex(dataset, menuIndex) {
+  if (!dataset || menuIndex <= 0) {
+    return void 0;
+  }
+  return dataset.runs[menuIndex - 1];
+}
+function createInitialState(data, args) {
+  const warnings = [];
+  if (args.unknownArgs.length > 0) {
+    warnings.push(`Unknown args: ${args.unknownArgs.join(", ")}`);
+    warnings.push("Supported: --dataset <id>, --run <id>, --search <term>");
+  }
+  const searchQuery = args.search ?? "";
+  const filteredDatasets = getFilteredDatasets(data, searchQuery);
+  const datasetByArg = filteredDatasets.find((dataset) => dataset.id === args.datasetId);
+  const datasetMenuIndex = datasetByArg ? filteredDatasets.indexOf(datasetByArg) + 1 : 0;
+  let level = "datasets";
+  let runMenuIndex = 0;
+  if (datasetByArg) {
+    level = "runs";
+  } else if (args.datasetId) {
+    warnings.push(`Dataset "${args.datasetId}" not found.`);
+  }
+  if (datasetByArg && args.runId) {
+    const runIndex = datasetByArg.runs.findIndex((run) => run.id === args.runId);
+    if (runIndex >= 0) {
+      runMenuIndex = runIndex + 1;
+      level = "details";
+    } else {
+      warnings.push(`Run "${args.runId}" not found in dataset "${datasetByArg.id}".`);
+    }
+  }
+  return {
+    level,
+    focus: "left",
+    datasetMenuIndex,
+    runMenuIndex,
+    detailsScrollOffset: 0,
+    selectedEvaluatorIds: data.evaluators.slice(0, 2).map((item) => item.id),
+    evaluatorMenuIndex: 0,
+    searchQuery,
+    searchMode: false,
+    startupWarnings: warnings
+  };
+}
+function reduceCliState(state, action) {
+  if (action.type === "MOVE_UP") {
+    if (state.searchMode) {
+      return state;
+    }
+    if (state.level === "details" && state.focus === "right") {
+      return { ...state, detailsScrollOffset: Math.max(0, state.detailsScrollOffset - 1) };
+    }
+    if (state.level === "datasets") {
+      return { ...state, datasetMenuIndex: Math.max(0, state.datasetMenuIndex - 1) };
+    }
+    if (state.level === "runs") {
+      return { ...state, runMenuIndex: Math.max(0, state.runMenuIndex - 1) };
+    }
+    if (state.level === "new-evaluation") {
+      return { ...state, evaluatorMenuIndex: Math.max(0, state.evaluatorMenuIndex - 1) };
+    }
+    return state;
+  }
+  if (action.type === "MOVE_DOWN") {
+    if (state.searchMode) {
+      return state;
+    }
+    if (state.level === "details" && state.focus === "right") {
+      return { ...state, detailsScrollOffset: Math.min(action.max, state.detailsScrollOffset + 1) };
+    }
+    if (state.level === "datasets") {
+      return { ...state, datasetMenuIndex: Math.min(action.max, state.datasetMenuIndex + 1) };
+    }
+    if (state.level === "runs") {
+      return { ...state, runMenuIndex: Math.min(action.max, state.runMenuIndex + 1) };
+    }
+    if (state.level === "new-evaluation") {
+      return { ...state, evaluatorMenuIndex: Math.min(action.max, state.evaluatorMenuIndex + 1) };
+    }
+    return state;
+  }
+  if (action.type === "ENTER") {
+    if (state.searchMode) {
+      return { ...state, searchMode: false };
+    }
+    if (state.level === "datasets") {
+      if (state.datasetMenuIndex === 0) {
+        return { ...state, level: "new-evaluation" };
+      }
+      if (action.hasDataset) {
+        return { ...state, level: "runs", runMenuIndex: 0 };
+      }
+      return state;
+    }
+    if (state.level === "runs") {
+      if (state.runMenuIndex === 0) {
+        return { ...state, level: "new-evaluation" };
+      }
+      if (action.hasRun) {
+        return { ...state, level: "details", detailsScrollOffset: 0 };
+      }
+      return state;
+    }
+    if (state.level === "new-evaluation") {
+      return state;
+    }
+    return state;
+  }
+  if (action.type === "BACK") {
+    if (state.searchMode) {
+      return { ...state, searchMode: false };
+    }
+    if (state.level === "details") {
+      return { ...state, level: "runs" };
+    }
+    if (state.level === "runs" || state.level === "new-evaluation") {
+      return { ...state, level: "datasets" };
+    }
+    return state;
+  }
+  if (action.type === "TOGGLE_FOCUS") {
+    return { ...state, focus: state.focus === "left" ? "right" : "left" };
+  }
+  if (action.type === "START_SEARCH") {
+    return { ...state, searchMode: true };
+  }
+  if (action.type === "END_SEARCH") {
+    return { ...state, searchMode: false };
+  }
+  if (action.type === "APPEND_SEARCH") {
+    return { ...state, searchQuery: `${state.searchQuery}${action.value}` };
+  }
+  if (action.type === "REMOVE_SEARCH_CHAR") {
+    return { ...state, searchQuery: state.searchQuery.slice(0, -1) };
+  }
+  if (action.type === "TOGGLE_EVALUATOR") {
+    const exists = state.selectedEvaluatorIds.includes(action.evaluatorId);
+    return {
+      ...state,
+      selectedEvaluatorIds: exists ? state.selectedEvaluatorIds.filter((id) => id !== action.evaluatorId) : [...state.selectedEvaluatorIds, action.evaluatorId]
+    };
+  }
+  if (action.type === "CLEAR_WARNINGS") {
+    return { ...state, startupWarnings: [] };
+  }
+  return state;
+}
+var LEFT_PANE_WIDTH2 = 44;
+function DatasetsView({
+  state,
+  filteredDatasets,
+  selectedDataset
+}) {
+  const leftFocused = state.focus === "left";
+  const rightFocused = state.focus === "right";
+  return /* @__PURE__ */ jsxs(Fragment, { children: [
+    /* @__PURE__ */ jsxs(Pane, { width: LEFT_PANE_WIDTH2, focused: leftFocused, children: [
+      /* @__PURE__ */ jsx(SectionHeader, { children: "Datasets" }),
+      /* @__PURE__ */ jsx(
+        ListItem,
+        {
+          selected: state.datasetMenuIndex === 0,
+          label: "New evaluation",
+          itemKey: "datasets-new-eval"
+        }
+      ),
+      filteredDatasets.map((dataset, index) => /* @__PURE__ */ jsx(
+        ListItem,
+        {
+          selected: state.datasetMenuIndex === index + 1,
+          label: dataset.name,
+          itemKey: `dataset-${dataset.id}`
+        },
+        dataset.id
+      ))
+    ] }),
+    /* @__PURE__ */ jsxs(Pane, { flexGrow: 1, marginLeft: 1, focused: rightFocused, children: [
+      /* @__PURE__ */ jsx(SectionHeader, { children: "Overview" }),
+      /* @__PURE__ */ jsx(Text, { color: "gray", children: selectedDataset?.overview ?? "Select a dataset to inspect prior runs." })
+    ] })
+  ] });
+}
+function RunsView({
+  state,
+  dataset,
+  selectedRun
+}) {
+  const runs = dataset?.runs ?? [];
+  const rightFocused = state.focus === "right";
+  return /* @__PURE__ */ jsxs(Fragment, { children: [
+    /* @__PURE__ */ jsx(RunsSidebar, { state, dataset, runs }),
+    /* @__PURE__ */ jsx(Pane, { flexGrow: 1, marginLeft: 1, focused: rightFocused, children: !selectedRun ? /* @__PURE__ */ jsx(Text, { color: "gray", children: "Select a run to see summary metrics." }) : /* @__PURE__ */ jsxs(Box, { flexDirection: "column", children: [
+      /* @__PURE__ */ jsxs(Text, { children: [
+        /* @__PURE__ */ jsx(Text, { color: "gray", children: "Run:" }),
+        " ",
+        selectedRun.label,
+        " ",
+        /* @__PURE__ */ jsx(StatusText, { status: selectedRun.status })
+      ] }),
+      /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
+        "Commit: ",
+        selectedRun.meta.commit,
+        "  Branch: ",
+        selectedRun.meta.branch,
+        " ",
+        "Seed: ",
+        selectedRun.meta.seed
+      ] }),
+      /* @__PURE__ */ jsx(Text, { children: " " }),
+      /* @__PURE__ */ jsx(SectionHeader, { children: "Overall" }),
+      /* @__PURE__ */ jsx(
+        TextBar,
+        {
+          label: "pass rate",
+          value: selectedRun.performance.passRate,
+          format: (v) => `${v}%`
+        }
+      ),
+      /* @__PURE__ */ jsx(
+        TextBar,
+        {
+          label: "avg score",
+          value: Math.round(selectedRun.performance.avgScore * 100)
+        }
+      ),
+      /* @__PURE__ */ jsx(Text, { children: " " }),
+      /* @__PURE__ */ jsx(SectionHeader, { children: "Dimensions" }),
+      selectedRun.dimensions.map((dimension) => /* @__PURE__ */ jsx(
+        TextBar,
+        {
+          label: dimension.name,
+          value: dimension.score
+        },
+        dimension.name
+      )),
+      /* @__PURE__ */ jsx(Text, { children: " " }),
+      /* @__PURE__ */ jsx(SectionHeader, { children: "Latency trend" }),
+      /* @__PURE__ */ jsx(
+        Sparkline,
+        {
+          data: selectedRun.performance.latencyHistoryMs ?? [
+            selectedRun.performance.latencyAvgMs - 40,
+            selectedRun.performance.latencyAvgMs - 10,
+            selectedRun.performance.latencyAvgMs + 20,
+            selectedRun.performance.latencyP95Ms - 80,
+            selectedRun.performance.latencyP95Ms
+          ],
+          width: 24
+        }
+      )
+    ] }) })
+  ] });
+}
+var DETAILS_PAGE_SIZE = 20;
+function CheckRow({
+  name,
+  passed,
+  detail
+}) {
+  const status = passed ? "PASSED" : "FAILED";
+  const color = passed ? "green" : "red";
+  return /* @__PURE__ */ jsxs(Text, { children: [
+    /* @__PURE__ */ jsx(Text, { color: "gray", children: name.padEnd(14) }),
+    " ",
+    /* @__PURE__ */ jsx(Text, { color, bold: true, children: status }),
+    detail ? /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
+      "  (",
+      detail,
+      ")"
+    ] }) : null
+  ] });
+}
+function buildDetailRows(run) {
+  const { performance, dimensions, checks, failures, meta } = run;
+  const latencyHistory = performance.latencyHistoryMs ?? [
+    performance.latencyAvgMs - 40,
+    performance.latencyAvgMs - 10,
+    performance.latencyAvgMs + 20,
+    performance.latencyP95Ms - 80,
+    performance.latencyP95Ms
+  ];
+  const rows = [
+    /* @__PURE__ */ jsx(SectionHeader, { children: "Meta" }, "meta-h"),
+    /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
+      "Model: ",
+      meta.model,
+      "   Provider: ",
+      meta.provider
+    ] }, "meta-1"),
+    /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
+      "Commit: ",
+      meta.commit,
+      "  Branch: ",
+      meta.branch,
+      "  Seed: ",
+      meta.seed
+    ] }, "meta-2"),
+    /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
+      "Duration: ",
+      meta.duration,
+      "   Concurrency: ",
+      meta.concurrency
+    ] }, "meta-3"),
+    /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
+      "Artifact: ",
+      meta.artifact
+    ] }, "meta-4"),
+    /* @__PURE__ */ jsx(Text, { children: " " }, "sp1"),
+    /* @__PURE__ */ jsx(SectionHeader, { children: "Scores (0\u2013100)" }, "scores-h"),
+    ...dimensions.map((d) => /* @__PURE__ */ jsx(TextBar, { label: d.name, value: d.score }, `dim-${d.name}`)),
+    /* @__PURE__ */ jsx(Text, { children: " " }, "sp2"),
+    /* @__PURE__ */ jsx(SectionHeader, { children: "Checks (boolean)" }, "checks-h"),
+    ...checks.map((c) => /* @__PURE__ */ jsx(CheckRow, { name: c.name, passed: c.passed, detail: c.detail }, `chk-${c.name}`)),
+    /* @__PURE__ */ jsx(Text, { children: " " }, "sp3"),
+    /* @__PURE__ */ jsx(SectionHeader, { children: "Performance" }, "perf-h"),
+    /* @__PURE__ */ jsx(
+      TextBar,
+      {
+        label: "pass rate",
+        value: performance.passRate,
+        format: (v) => `${v}%`
+      },
+      "perf-rate"
+    ),
+    /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
+      "latency avg     ",
+      performance.latencyAvgMs,
+      "ms   p95 ",
+      performance.latencyP95Ms,
+      "ms"
+    ] }, "perf-lat"),
+    /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
+      "tokens avg      ",
+      performance.tokensAvg,
+      "   p95 ",
+      performance.tokensP95
+    ] }, "perf-tok"),
+    /* @__PURE__ */ jsx(Text, { children: " " }, "sp4"),
+    /* @__PURE__ */ jsx(SectionHeader, { children: "Latency trend" }, "spark-h"),
+    /* @__PURE__ */ jsx(Sparkline, { data: latencyHistory, width: 20 }, "spark")
+  ];
+  if (failures.length > 0) {
+    rows.push(/* @__PURE__ */ jsx(Text, { children: " " }, "sp5"));
+    rows.push(/* @__PURE__ */ jsx(SectionHeader, { children: "Failures (top)" }, "fail-h"));
+    failures.forEach((f, i) => {
+      rows.push(
+        /* @__PURE__ */ jsxs(Text, { color: "red", children: [
+          i + 1,
+          ") ",
+          f.title
+        ] }, `fail-${i}`)
+      );
+    });
+  }
+  return rows;
+}
+function RunDetailsView({
+  state,
+  dataset,
+  selectedRun
+}) {
+  const runs = dataset?.runs ?? [];
+  const rightFocused = state.focus === "right";
+  if (!selectedRun) {
+    return /* @__PURE__ */ jsxs(Fragment, { children: [
+      /* @__PURE__ */ jsx(RunsSidebar, { state, dataset, runs }),
+      /* @__PURE__ */ jsx(Pane, { flexGrow: 1, marginLeft: 1, focused: rightFocused, children: /* @__PURE__ */ jsx(Text, { color: "gray", children: "Select a run to inspect details." }) })
+    ] });
+  }
+  const rows = buildDetailRows(selectedRun);
+  const offset = Math.max(0, state.detailsScrollOffset);
+  const visible = rows.slice(offset, offset + DETAILS_PAGE_SIZE);
+  return /* @__PURE__ */ jsxs(Fragment, { children: [
+    /* @__PURE__ */ jsx(RunsSidebar, { state, dataset, runs }),
+    /* @__PURE__ */ jsx(Pane, { flexGrow: 1, marginLeft: 1, focused: rightFocused, children: /* @__PURE__ */ jsx(Box, { flexDirection: "column", children: visible.map((row, i) => /* @__PURE__ */ jsx(React.Fragment, { children: row }, i)) }) })
+  ] });
+}
+var LEFT_PANE_WIDTH3 = 44;
+function NewEvaluationView({
+  state,
+  data,
+  visibleEvaluators
+}) {
+  const selectedCount = state.selectedEvaluatorIds.length;
+  const focusedEvaluator = visibleEvaluators[state.evaluatorMenuIndex];
+  const leftFocused = state.focus === "left";
+  const rightFocused = state.focus === "right";
+  return /* @__PURE__ */ jsxs(Fragment, { children: [
+    /* @__PURE__ */ jsxs(Pane, { width: LEFT_PANE_WIDTH3, focused: leftFocused, children: [
+      /* @__PURE__ */ jsx(SectionHeader, { children: "Available Evaluators" }),
+      /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
+        "Search: ",
+        state.searchQuery || "(none)"
+      ] }),
+      visibleEvaluators.map((evaluator, index) => {
+        const selected = index === state.evaluatorMenuIndex;
+        const inSelection = state.selectedEvaluatorIds.includes(evaluator.id);
+        return /* @__PURE__ */ jsxs(
+          Text,
+          {
+            color: selected ? "cyan" : "gray",
+            bold: selected,
+            children: [
+              selected ? "\u25B8 " : "  ",
+              inSelection ? "[x] " : "[ ] ",
+              evaluator.name
+            ]
+          },
+          evaluator.id
+        );
+      })
+    ] }),
+    /* @__PURE__ */ jsxs(Pane, { flexGrow: 1, marginLeft: 1, focused: rightFocused, children: [
+      /* @__PURE__ */ jsxs(SectionHeader, { children: [
+        "Selected (",
+        selectedCount,
+        ")"
+      ] }),
+      state.selectedEvaluatorIds.map((id, index) => {
+        const evaluator = data.evaluators.find((item) => item.id === id);
+        if (!evaluator)
+          return null;
+        return /* @__PURE__ */ jsxs(Text, { children: [
+          index + 1,
+          ") ",
+          evaluator.name
+        ] }, id);
+      }),
+      /* @__PURE__ */ jsx(SectionHeader, { children: "Config preview" }),
+      /* @__PURE__ */ jsx(Text, { color: "gray", children: focusedEvaluator?.configPreview ?? "Select an evaluator to inspect config." })
+    ] })
+  ] });
+}
+function clampCursor(state, filteredDatasetsLength, selectedRunCount) {
+  const datasetMax = filteredDatasetsLength;
+  const runMax = selectedRunCount;
+  const evaluatorMax = 3;
+  return {
+    ...state,
+    datasetMenuIndex: Math.max(0, Math.min(state.datasetMenuIndex, datasetMax)),
+    runMenuIndex: Math.max(0, Math.min(state.runMenuIndex, runMax)),
+    evaluatorMenuIndex: Math.max(
+      0,
+      Math.min(state.evaluatorMenuIndex, evaluatorMax)
+    )
+  };
+}
+function EvalsCliApp({
+  data,
+  args,
+  runner
+}) {
+  const { exit } = useApp();
+  const { width: stdoutWidth, height: stdoutHeight } = useScreenSize();
+  const [liveData, setLiveData] = useState(data);
+  const [runtimeMessage, setRuntimeMessage] = useState();
+  const [state, dispatch] = useReducer(
+    reduceCliState,
+    createInitialState(data, args)
+  );
+  useEffect(() => {
+    setLiveData(data);
+  }, [data]);
+  useEffect(() => {
+    if (!runner) {
+      return void 0;
+    }
+    return runner.subscribeRunEvents((event) => {
+      setLiveData((current) => applyRunnerEvent(current, event, runner));
+      if (event.type === "RunQueued") {
+        setRuntimeMessage(`Queued ${event.runId} with ${event.totalTestCases} test cases.`);
+      }
+      if (event.type === "RunCompleted") {
+        setRuntimeMessage(
+          `Completed ${event.runId}: ${event.passedTestCases}/${event.totalTestCases} passed.`
+        );
+      }
+      if (event.type === "RunFailed") {
+        setRuntimeMessage(`Run failed: ${event.errorMessage}`);
+      }
+    });
+  }, [runner]);
+  const filteredDatasets = useMemo(
+    () => getFilteredDatasets(liveData, state.searchQuery),
+    [liveData, state.searchQuery]
+  );
+  const clampedState = clampCursor(
+    state,
+    filteredDatasets.length,
+    getDatasetByMenuIndex(filteredDatasets, state.datasetMenuIndex)?.runs.length ?? 0
+  );
+  const selectedDataset = getDatasetByMenuIndex(
+    filteredDatasets,
+    clampedState.datasetMenuIndex
+  );
+  const selectedRun = getRunByMenuIndex(
+    selectedDataset,
+    clampedState.runMenuIndex
+  );
+  const visibleEvaluators = liveData.evaluators.filter(
+    (evaluator) => evaluator.name.toLowerCase().includes(clampedState.searchQuery.toLowerCase())
+  );
+  useInput((input, key) => {
+    if (isQuitInput(input) || key.escape) {
+      exit();
+      return;
+    }
+    if (key.tab) {
+      dispatch({ type: "TOGGLE_FOCUS" });
+      return;
+    }
+    if (isSearchInput(input)) {
+      dispatch({ type: "START_SEARCH" });
+      return;
+    }
+    if (clampedState.searchMode) {
+      if (key.return) {
+        dispatch({ type: "END_SEARCH" });
+        return;
+      }
+      if (isBackKey(key)) {
+        dispatch({ type: "REMOVE_SEARCH_CHAR" });
+        return;
+      }
+      if (isPrintableCharacter(input)) {
+        dispatch({ type: "APPEND_SEARCH", value: input });
+      }
+      return;
+    }
+    if (key.upArrow) {
+      const max = clampedState.level === "details" ? 100 : clampedState.level === "new-evaluation" ? visibleEvaluators.length - 1 : 100;
+      dispatch({ type: "MOVE_UP", max });
+      return;
+    }
+    if (key.downArrow) {
+      const max = clampedState.level === "datasets" ? filteredDatasets.length : clampedState.level === "runs" ? selectedDataset?.runs.length ?? 0 : clampedState.level === "new-evaluation" ? Math.max(0, visibleEvaluators.length - 1) : 100;
+      dispatch({ type: "MOVE_DOWN", max });
+      return;
+    }
+    if (key.return) {
+      dispatch({
+        type: "ENTER",
+        hasDataset: Boolean(selectedDataset),
+        hasRun: Boolean(selectedRun)
+      });
+      if (clampedState.level === "new-evaluation") {
+        const evaluator = visibleEvaluators[clampedState.evaluatorMenuIndex];
+        if (evaluator) {
+          dispatch({ type: "TOGGLE_EVALUATOR", evaluatorId: evaluator.id });
+        }
+      }
+      return;
+    }
+    if (isBackKey(key)) {
+      dispatch({ type: "BACK" });
+      return;
+    }
+    if (input.toLowerCase() === "c") {
+      dispatch({ type: "CLEAR_WARNINGS" });
+      setRuntimeMessage(void 0);
+      return;
+    }
+    if (input.toLowerCase() === "s" && clampedState.level === "new-evaluation") {
+      if (!runner) {
+        setRuntimeMessage("Runner unavailable: cannot start evaluation.");
+        return;
+      }
+      if (!selectedDataset) {
+        setRuntimeMessage("Select a dataset before starting a new evaluation.");
+        return;
+      }
+      if (clampedState.selectedEvaluatorIds.length === 0) {
+        setRuntimeMessage("Select at least one evaluator before starting.");
+        return;
+      }
+      void runner.runDatasetWith({
+        datasetId: selectedDataset.id,
+        evaluatorIds: clampedState.selectedEvaluatorIds
+      }).then((snapshot) => {
+        setRuntimeMessage(
+          `Started ${snapshot.runId} on ${selectedDataset.name} (${snapshot.totalTestCases} cases).`
+        );
+      }).catch((error) => {
+        setRuntimeMessage(
+          error instanceof Error ? error.message : "Failed to start evaluation."
+        );
+      });
+    }
+  });
+  const renderContent = () => {
+    if (clampedState.level === "new-evaluation") {
+      return /* @__PURE__ */ jsx(
+        NewEvaluationView,
+        {
+          state: clampedState,
+          data: liveData,
+          visibleEvaluators
+        }
+      );
+    }
+    if (clampedState.level === "datasets") {
+      return /* @__PURE__ */ jsx(
+        DatasetsView,
+        {
+          state: clampedState,
+          filteredDatasets,
+          selectedDataset
+        }
+      );
+    }
+    if (clampedState.level === "runs") {
+      return /* @__PURE__ */ jsx(
+        RunsView,
+        {
+          state: clampedState,
+          dataset: selectedDataset,
+          selectedRun
+        }
+      );
+    }
+    return /* @__PURE__ */ jsx(
+      RunDetailsView,
+      {
+        state: clampedState,
+        dataset: selectedDataset,
+        selectedRun
+      }
+    );
+  };
+  return /* @__PURE__ */ jsxs(
+    Box,
+    {
+      flexDirection: "column",
+      flexGrow: 1,
+      width: stdoutWidth,
+      height: stdoutHeight,
+      children: [
+        /* @__PURE__ */ jsx(
+          Box,
+          {
+            borderStyle: "round",
+            borderColor: "cyan",
+            paddingX: 1,
+            width: stdoutWidth,
+            children: /* @__PURE__ */ jsx(Text, { children: getBreadcrumbText(
+              clampedState,
+              selectedDataset?.name,
+              selectedRun?.label
+            ) })
+          }
+        ),
+        clampedState.startupWarnings.length > 0 && /* @__PURE__ */ jsxs(
+          Box,
+          {
+            marginTop: 1,
+            borderStyle: "round",
+            borderColor: "yellow",
+            paddingX: 1,
+            flexDirection: "column",
+            width: stdoutWidth,
+            children: [
+              /* @__PURE__ */ jsx(Text, { color: "yellow", children: "Startup warnings:" }),
+              clampedState.startupWarnings.map((warning, index) => /* @__PURE__ */ jsx(Text, { children: warning }, `${warning}-${index}`))
+            ]
+          }
+        ),
+        clampedState.searchMode && /* @__PURE__ */ jsxs(
+          Box,
+          {
+            marginTop: 1,
+            borderStyle: "round",
+            borderColor: "magenta",
+            paddingX: 1,
+            width: stdoutWidth,
+            children: [
+              /* @__PURE__ */ jsx(Text, { color: "magenta", bold: true, children: "Search: " }),
+              /* @__PURE__ */ jsx(Text, { color: "white", children: clampedState.searchQuery })
+            ]
+          }
+        ),
+        runtimeMessage && /* @__PURE__ */ jsx(
+          Box,
+          {
+            marginTop: 1,
+            borderStyle: "round",
+            borderColor: "blue",
+            paddingX: 1,
+            width: stdoutWidth,
+            children: /* @__PURE__ */ jsx(Text, { color: "blue", children: runtimeMessage })
+          }
+        ),
+        /* @__PURE__ */ jsx(
+          Box,
+          {
+            marginTop: 1,
+            flexGrow: 1,
+            width: stdoutWidth,
+            flexDirection: "row",
+            children: renderContent()
+          }
+        ),
+        /* @__PURE__ */ jsx(Box, { marginTop: 1, paddingX: 1, children: /* @__PURE__ */ jsx(Text, { color: "gray", children: getFooterText(clampedState) }) })
+      ]
+    }
+  );
+}
+
+// src/runner/config.ts
+var defaultRunnerConfig = {
+  discovery: {
+    rootDir: process.cwd(),
+    datasetSuffixes: [".dataset.ts", ".dataset.tsx", ".dataset.js", ".dataset.mjs"],
+    evaluatorSuffixes: [
+      ".evaluator.ts",
+      ".evaluator.tsx",
+      ".evaluator.js",
+      ".evaluator.mjs"
+    ],
+    testCaseSuffixes: [
+      ".test-case.ts",
+      ".test-case.tsx",
+      ".test-case.js",
+      ".test-case.mjs"
+    ],
+    excludeDirectories: ["node_modules", "dist", ".next", ".git", ".pnpm-store"]
+  },
+  artifactDirectory: ".eval-results"
+};
+function withRunnerConfig(overrides) {
+  {
+    return defaultRunnerConfig;
+  }
+}
+var jitiLoader;
+function toId(prefix, filePath, name) {
+  const stable = name && name.trim().length > 0 ? name : filePath;
+  return `${prefix}:${stable}`.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
+}
+function hasMethod(value, methodName) {
+  return typeof value === "object" && value !== null && methodName in value && typeof value[methodName] === "function";
+}
+function isDatasetLike(value) {
+  return hasMethod(value, "getName") && hasMethod(value, "matchesTestCase");
+}
+function isEvaluatorLike(value) {
+  return hasMethod(value, "getName") && hasMethod(value, "resolveContext") && hasMethod(value, "getEvaluateFn");
+}
+function isTestCaseLike(value) {
+  return hasMethod(value, "getName") && hasMethod(value, "getTags") && hasMethod(value, "getInput");
+}
+async function walkDirectory(rootDir, excludeDirectories) {
+  const out = [];
+  async function walk(currentDir) {
+    let entries;
+    try {
+      entries = await readdir(currentDir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    await Promise.all(
+      entries.map(async (entry) => {
+        const absolute = resolve(currentDir, entry.name);
+        if (entry.isDirectory()) {
+          if (excludeDirectories.includes(entry.name)) {
+            return;
+          }
+          await walk(absolute);
+          return;
+        }
+        if (entry.isFile()) {
+          out.push(absolute);
+        }
+      })
+    );
+  }
+  await walk(rootDir);
+  return out;
+}
+function hasOneSuffix(filePath, suffixes) {
+  return suffixes.some((suffix) => filePath.endsWith(suffix));
+}
+async function loadModuleExports(filePath) {
+  if (filePath.endsWith(".ts") || filePath.endsWith(".tsx")) {
+    if (!jitiLoader) {
+      const jitiModule = await import('jiti');
+      const createJiti = jitiModule.createJiti ?? jitiModule.default;
+      if (!createJiti) {
+        throw new Error("Failed to initialize jiti TypeScript loader");
+      }
+      jitiLoader = createJiti(import.meta.url, {
+        interopDefault: true,
+        moduleCache: true
+      });
+    }
+    const loaded2 = jitiLoader.import ? await jitiLoader.import(filePath) : await Promise.resolve(jitiLoader(filePath));
+    return Object.values(loaded2);
+  }
+  const moduleUrl = pathToFileURL(filePath).href;
+  const loaded = await import(moduleUrl);
+  return Object.values(loaded);
+}
+async function collectDatasetsFromFiles(config) {
+  const files = await walkDirectory(config.rootDir, config.excludeDirectories);
+  const matched = files.filter(
+    (filePath) => hasOneSuffix(filePath, config.datasetSuffixes)
+  );
+  const found = await Promise.all(
+    matched.map(async (absolutePath) => {
+      const exports = await loadModuleExports(absolutePath);
+      const datasets = exports.filter(isDatasetLike);
+      const relPath = relative(config.rootDir, absolutePath);
+      return datasets.map((dataset) => ({
+        id: toId("dataset", relPath, dataset.getName()),
+        filePath: relPath,
+        dataset
+      }));
+    })
+  );
+  return found.flat();
+}
+async function collectEvaluatorsFromFiles(config) {
+  const files = await walkDirectory(config.rootDir, config.excludeDirectories);
+  const matched = files.filter(
+    (filePath) => hasOneSuffix(filePath, config.evaluatorSuffixes)
+  );
+  const found = await Promise.all(
+    matched.map(async (absolutePath) => {
+      const exports = await loadModuleExports(absolutePath);
+      const evaluators = exports.filter(isEvaluatorLike);
+      const relPath = relative(config.rootDir, absolutePath);
+      return evaluators.map((evaluator) => ({
+        id: toId("evaluator", relPath, evaluator.getName()),
+        filePath: relPath,
+        evaluator
+      }));
+    })
+  );
+  return found.flat();
+}
+async function collectTestCasesFromFiles(config) {
+  const files = await walkDirectory(config.rootDir, config.excludeDirectories);
+  const matched = files.filter(
+    (filePath) => hasOneSuffix(filePath, config.testCaseSuffixes)
+  );
+  const found = await Promise.all(
+    matched.map(async (absolutePath) => {
+      const exports = await loadModuleExports(absolutePath);
+      const testCases = exports.filter(isTestCaseLike);
+      const relPath = relative(config.rootDir, absolutePath);
+      return testCases.map((testCase) => ({
+        id: toId("test-case", relPath, testCase.getName()),
+        filePath: relPath,
+        testCase
+      }));
+    })
+  );
+  return found.flat();
+}
+
+// src/evals/metric.ts
+var registry = /* @__PURE__ */ new Map();
+var Metric = {
+  of(config) {
+    const def = {
+      id: config.id,
+      name: config.name,
+      format: config.format,
+      make: (data) => ({ id: config.id, data })
+    };
+    registry.set(config.id, def);
+    return def;
+  }
+};
+
+// src/evals/score.ts
+var registry2 = /* @__PURE__ */ new Map();
+var Score = {
+  of(config) {
+    const def = {
+      id: config.id,
+      name: config.name,
+      displayStrategy: config.displayStrategy,
+      format: config.format,
+      make: (data, options) => {
+        const passed = options?.definePassed !== void 0 ? options.definePassed(data) : void 0;
+        return {
+          id: config.id,
+          data,
+          ...passed !== void 0 && { passed }
+        };
+      }
+    };
+    registry2.set(config.id, def);
+    return def;
+  }
+};
+function getScoreById(id) {
+  return registry2.get(id);
+}
+
+// src/evals/metrics/standard.ts
+Metric.of({
+  id: "token-count",
+  name: "Tokens",
+  format: (data) => {
+    const input = data.input ?? 0;
+    const output = data.output ?? 0;
+    const inputCached = data.inputCached ?? 0;
+    const outputCached = data.outputCached ?? 0;
+    const cached = inputCached + outputCached;
+    return `in:${input} out:${output} cached:${cached}`;
+  }
+});
+Metric.of({
+  id: "latency",
+  name: "Latency",
+  format: (data) => `${data.ms}ms`
+});
+
+// src/evals/scores/standard.ts
+Score.of({
+  id: "percent",
+  name: "Score",
+  displayStrategy: "bar",
+  format: (data) => data.value.toFixed(2)
+});
+Score.of({
+  id: "binary",
+  name: "Result",
+  displayStrategy: "passFail",
+  format: (data) => data.passed ? "PASSED" : "NOT PASSED"
+});
+
+// src/runner/score-utils.ts
+function toNumericScoreFromScores(scores) {
+  for (const item of scores) {
+    const def = getScoreById(item.id);
+    if (def && def.displayStrategy === "bar" && typeof item.data === "object" && item.data !== null && "value" in item.data) {
+      const value = item.data.value;
+      if (typeof value === "number" && Number.isFinite(value)) {
+        return value;
+      }
+    }
+    const numeric = toNumericScore(item.data);
+    if (numeric !== void 0) {
+      return numeric;
+    }
+  }
+  return void 0;
+}
+function toNumericScore(value) {
+  if (typeof value === "number" && Number.isFinite(value)) {
+    return value;
+  }
+  if (typeof value !== "object" || value === null) {
+    return void 0;
+  }
+  const obj = value;
+  if ("score" in obj && typeof obj.score === "number" && Number.isFinite(obj.score)) {
+    return obj.score;
+  }
+  const numberValues = Object.values(value).filter(
+    (entry) => typeof entry === "number" && Number.isFinite(entry)
+  );
+  if (numberValues.length === 0) {
+    return void 0;
+  }
+  return numberValues.reduce((sum, entry) => sum + entry, 0) / numberValues.length;
+}
+
+// src/runner/execution.ts
+function computeEvaluatorPassed(evaluator, result, scores) {
+  const scoresWithPassed = scores.filter((s) => "passed" in s && s.passed !== void 0);
+  if (scoresWithPassed.length > 0) {
+    return scoresWithPassed.every((s) => s.passed === true);
+  }
+  const passCriterion = evaluator.getPassCriterion();
+  if (passCriterion) {
+    return passCriterion(result);
+  }
+  const passThreshold = evaluator.getPassThreshold();
+  if (passThreshold !== void 0) {
+    const numeric = toNumericScoreFromScores(scores);
+    return numeric !== void 0 && numeric >= passThreshold;
+  }
+  return true;
+}
+function normalizeResult(result) {
+  if (typeof result !== "object" || result === null) {
+    return { scores: [] };
+  }
+  const obj = result;
+  const scores = Array.isArray(obj.scores) ? obj.scores : [];
+  const metrics = Array.isArray(obj.metrics) ? obj.metrics : void 0;
+  return { scores, metrics };
+}
+function nowIsoForFile() {
+  return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
+}
+function createArtifactPath(artifactDirectory, datasetId, runId) {
+  return join(
+    artifactDirectory,
+    `${datasetId}_${runId}_${nowIsoForFile()}.jsonl`
+  );
+}
+var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => Effect.gen(function* () {
+  const startedAt = Date.now();
+  updateSnapshot(task.runId, (snapshot) => ({
+    ...snapshot,
+    status: "running",
+    startedAt
+  }));
+  yield* publishEvent({
+    type: "RunStarted",
+    runId: task.runId,
+    startedAt
+  });
+  let completedTestCases = 0;
+  let passedTestCases = 0;
+  let failedTestCases = 0;
+  for (const testCaseItem of task.testCases) {
+    const started = Date.now();
+    const evaluatorScores = [];
+    let testCaseError;
+    for (const { id: evaluatorId, evaluator } of task.evaluators) {
+      const evaluateFn = evaluator.getEvaluateFn();
+      if (!evaluateFn) {
+        continue;
+      }
+      try {
+        const ctx = yield* Effect.promise(
+          () => Promise.resolve(evaluator.resolveContext())
+        );
+        const result = yield* Effect.promise(
+          () => Promise.resolve(evaluateFn(testCaseItem.testCase.getInput(), ctx))
+        );
+        const { scores, metrics } = normalizeResult(result);
+        const passed = computeEvaluatorPassed(evaluator, result, scores);
+        evaluatorScores.push({ evaluatorId, scores, passed, metrics });
+      } catch (error) {
+        testCaseError = error instanceof Error ? error.message : "Evaluator execution failed";
+        evaluatorScores.push({
+          evaluatorId,
+          scores: [],
+          passed: false
+        });
+      }
+    }
+    const testCasePassed = evaluatorScores.every((s) => s.passed);
+    completedTestCases += 1;
+    if (testCasePassed) {
+      passedTestCases += 1;
+    } else {
+      failedTestCases += 1;
+    }
+    const progressEvent = {
+      type: "TestCaseProgress",
+      runId: task.runId,
+      testCaseId: testCaseItem.id,
+      testCaseName: testCaseItem.testCase.getName(),
+      completedTestCases,
+      totalTestCases: task.testCases.length,
+      passed: testCasePassed,
+      durationMs: Date.now() - started,
+      evaluatorScores,
+      errorMessage: testCaseError
+    };
+    updateSnapshot(task.runId, (snapshot) => ({
+      ...snapshot,
+      completedTestCases,
+      passedTestCases,
+      failedTestCases
+    }));
+    yield* publishEvent(progressEvent);
+    yield* Queue.offer(persistenceQueue, {
+      runId: task.runId,
+      artifactPath: task.snapshot.artifactPath,
+      payload: progressEvent
+    });
+  }
+  const finishedAt = Date.now();
+  const completedEvent = {
+    type: "RunCompleted",
+    runId: task.runId,
+    finishedAt,
+    passedTestCases,
+    failedTestCases,
+    totalTestCases: task.testCases.length,
+    artifactPath: task.snapshot.artifactPath
+  };
+  updateSnapshot(task.runId, (snapshot) => ({
+    ...snapshot,
+    status: "completed",
+    completedTestCases,
+    passedTestCases,
+    failedTestCases,
+    finishedAt
+  }));
+  yield* publishEvent(completedEvent);
+  yield* Queue.offer(persistenceQueue, {
+    runId: task.runId,
+    artifactPath: task.snapshot.artifactPath,
+    payload: completedEvent
+  });
+  yield* publishEvent({
+    type: "ArtifactFlushed",
+    runId: task.runId,
+    artifactPath: task.snapshot.artifactPath
+  });
+});
+async function appendJsonLine(artifactPath, payload) {
+  await mkdir(dirname(artifactPath), { recursive: true });
+  await appendFile(artifactPath, `${JSON.stringify(payload)}
+`, "utf8");
+}
+var createPersistenceWorker = (queue) => Effect.forever(
+  Effect.gen(function* () {
+    const message = yield* Queue.take(queue);
+    yield* Effect.promise(
+      () => appendJsonLine(message.artifactPath, {
+        runId: message.runId,
+        ts: Date.now(),
+        ...message.payload
+      })
+    );
+  })
+);
+
+// src/runner/search.ts
+function matchesAny(value, matchers) {
+  if (!matchers || matchers.length === 0) {
+    return true;
+  }
+  return matchers.some(
+    (matcher) => typeof matcher === "string" ? matcher === value : matcher.test(value)
+  );
+}
+function matchesPath(value, matchers) {
+  if (!matchers || matchers.length === 0) {
+    return true;
+  }
+  return matchers.some((matcher) => {
+    if (typeof matcher === "string") {
+      return value.includes(matcher);
+    }
+    return matcher.test(value);
+  });
+}
+function searchCollectedTestCases(all, query) {
+  if (!query) {
+    return all;
+  }
+  return all.filter((item) => {
+    const tags = item.testCase.getTags();
+    if (query.excludedTags && tags.some((tag) => matchesAny(tag, query.excludedTags))) {
+      return false;
+    }
+    if (query.excludedPaths && matchesPath(item.filePath, query.excludedPaths)) {
+      return false;
+    }
+    const includedTagsMatch = !query.includedTags || query.includedTags.length === 0 || tags.some((tag) => matchesAny(tag, query.includedTags));
+    const includedPathsMatch = !query.includedPaths || query.includedPaths.length === 0 || matchesPath(item.filePath, query.includedPaths);
+    return includedTagsMatch && includedPathsMatch;
+  });
+}
+
+// src/runner/api.ts
+function parseRegexLiteral(pattern) {
+  if (!pattern.startsWith("/")) {
+    return void 0;
+  }
+  const lastSlash = pattern.lastIndexOf("/");
+  if (lastSlash <= 0) {
+    return void 0;
+  }
+  return {
+    source: pattern.slice(1, lastSlash),
+    flags: pattern.slice(lastSlash + 1)
+  };
+}
+function createNameMatcher(pattern) {
+  const normalizedPattern = pattern.trim();
+  const regexLiteral = parseRegexLiteral(normalizedPattern);
+  if (regexLiteral) {
+    const regex = new RegExp(regexLiteral.source, regexLiteral.flags);
+    return (value) => regex.test(value);
+  }
+  if (normalizedPattern.includes("*")) {
+    const escaped = normalizedPattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
+    const regex = new RegExp(`^${escaped}$`, "i");
+    return (value) => regex.test(value);
+  }
+  return (value) => value.toLowerCase() === normalizedPattern.toLowerCase();
+}
+function createRunner(overrides) {
+  return new EffectRunner(withRunnerConfig());
+}
+var EffectRunner = class {
+  constructor(config) {
+    this.eventBus = Effect.runSync(PubSub.unbounded());
+    this.runQueue = Effect.runSync(Queue.unbounded());
+    this.persistenceQueue = Effect.runSync(
+      Queue.unbounded()
+    );
+    this.snapshots = /* @__PURE__ */ new Map();
+    this.listeners = /* @__PURE__ */ new Set();
+    this.datasetsById = /* @__PURE__ */ new Map();
+    this.evaluatorsById = /* @__PURE__ */ new Map();
+    this.schedulerFiber = Effect.runFork(
+      this.createSchedulerEffect()
+    );
+    this.persistenceFiber = Effect.runFork(
+      createPersistenceWorker(this.persistenceQueue)
+    );
+    this.config = config;
+  }
+  async collectDatasets() {
+    const datasets = await collectDatasetsFromFiles(this.config.discovery);
+    this.datasetsById.clear();
+    for (const dataset of datasets) {
+      this.datasetsById.set(dataset.id, dataset);
+    }
+    return datasets;
+  }
+  async collectEvaluators() {
+    const evaluators = await collectEvaluatorsFromFiles(this.config.discovery);
+    this.evaluatorsById.clear();
+    for (const evaluator of evaluators) {
+      this.evaluatorsById.set(evaluator.id, evaluator);
+    }
+    return evaluators;
+  }
+  async resolveDatasetByName(name) {
+    if (this.datasetsById.size === 0) {
+      await this.collectDatasets();
+    }
+    const normalized = name.trim().toLowerCase();
+    return Array.from(this.datasetsById.values()).find(
+      (item) => item.dataset.getName().toLowerCase() === normalized
+    );
+  }
+  async resolveEvaluatorsByNamePattern(pattern) {
+    if (this.evaluatorsById.size === 0) {
+      await this.collectEvaluators();
+    }
+    const matcher = createNameMatcher(pattern);
+    return Array.from(this.evaluatorsById.values()).filter(
+      (item) => matcher(item.evaluator.getName() ?? "")
+    );
+  }
+  async searchTestCases(query) {
+    const testCases = await collectTestCasesFromFiles(this.config.discovery);
+    return searchCollectedTestCases(testCases, query);
+  }
+  async collectDatasetTestCases(datasetId) {
+    if (this.datasetsById.size === 0) {
+      await this.collectDatasets();
+    }
+    const dataset = this.datasetsById.get(datasetId);
+    if (!dataset) {
+      throw new Error(`Unknown dataset: ${datasetId}`);
+    }
+    const allTestCases = await collectTestCasesFromFiles(this.config.discovery);
+    return allTestCases.filter(
+      (testCase) => dataset.dataset.matchesTestCase(testCase.testCase, testCase.filePath)
+    );
+  }
+  async runDatasetWith(request) {
+    if (this.datasetsById.size === 0) {
+      await this.collectDatasets();
+    }
+    if (this.evaluatorsById.size === 0) {
+      await this.collectEvaluators();
+    }
+    const dataset = this.datasetsById.get(request.datasetId);
+    if (!dataset) {
+      throw new Error(`Unknown dataset: ${request.datasetId}`);
+    }
+    const selectedEvaluators = request.evaluatorIds.map((id) => this.evaluatorsById.get(id)).filter((value) => Boolean(value)).map((value) => ({ id: value.id, evaluator: value.evaluator }));
+    if (selectedEvaluators.length === 0) {
+      throw new Error("No evaluators selected for run");
+    }
+    const selectedTestCases = await this.collectDatasetTestCases(request.datasetId);
+    const runId = `run-${randomUUID()}`;
+    const artifactPath = createArtifactPath(
+      this.config.artifactDirectory,
+      request.datasetId,
+      runId
+    );
+    const snapshot = {
+      runId,
+      datasetId: request.datasetId,
+      datasetName: dataset.dataset.getName(),
+      evaluatorIds: selectedEvaluators.map((item) => item.id),
+      queuedAt: Date.now(),
+      totalTestCases: selectedTestCases.length,
+      completedTestCases: 0,
+      passedTestCases: 0,
+      failedTestCases: 0,
+      status: "queued",
+      artifactPath
+    };
+    this.snapshots.set(runId, snapshot);
+    const queuedEvent = {
+      type: "RunQueued",
+      runId,
+      datasetId: request.datasetId,
+      datasetName: dataset.dataset.getName(),
+      evaluatorIds: selectedEvaluators.map((item) => item.id),
+      totalTestCases: selectedTestCases.length,
+      artifactPath
+    };
+    await Effect.runPromise(this.publishEvent(queuedEvent));
+    await Effect.runPromise(
+      Queue.offer(this.persistenceQueue, {
+        runId,
+        artifactPath,
+        payload: queuedEvent
+      })
+    );
+    await Effect.runPromise(
+      Queue.offer(this.runQueue, {
+        runId,
+        datasetId: request.datasetId,
+        dataset: dataset.dataset,
+        evaluators: selectedEvaluators,
+        testCases: selectedTestCases,
+        snapshot
+      })
+    );
+    return snapshot;
+  }
+  subscribeRunEvents(listener, options) {
+    const entry = { runId: options?.runId, listener };
+    this.listeners.add(entry);
+    return () => {
+      this.listeners.delete(entry);
+    };
+  }
+  getRunSnapshot(runId) {
+    return this.snapshots.get(runId);
+  }
+  getAllRunSnapshots() {
+    return Array.from(this.snapshots.values()).sort(
+      (a, b) => b.queuedAt - a.queuedAt
+    );
+  }
+  async shutdown() {
+    await Effect.runPromise(Fiber.interrupt(this.schedulerFiber));
+    await Effect.runPromise(Fiber.interrupt(this.persistenceFiber));
+    await Effect.runPromise(Queue.shutdown(this.runQueue));
+    await Effect.runPromise(Queue.shutdown(this.persistenceQueue));
+    await Effect.runPromise(PubSub.shutdown(this.eventBus));
+  }
+  createSchedulerEffect() {
+    const self = this;
+    return Effect.forever(
+      Effect.gen(function* () {
+        const task = yield* Queue.take(self.runQueue);
+        yield* Effect.fork(
+          executeRunTask(
+            task,
+            self.publishEvent.bind(self),
+            self.persistenceQueue,
+            self.updateSnapshot.bind(self)
+          )
+        );
+      })
+    );
+  }
+  updateSnapshot(runId, updater) {
+    const existing = this.snapshots.get(runId);
+    if (!existing) {
+      return;
+    }
+    this.snapshots.set(runId, updater(existing));
+  }
+  publishEvent(event) {
+    return Effect.sync(() => {
+      for (const entry of this.listeners) {
+        if (entry.runId && entry.runId !== event.runId) {
+          continue;
+        }
+        entry.listener(event);
+      }
+    }).pipe(
+      Effect.flatMap(() => PubSub.publish(this.eventBus, event)),
+      Effect.asVoid
+    );
+  }
+};
+async function main() {
+  const args = parseStartupArgs(process.argv.slice(2));
+  const runner = createRunner();
+  const data = await loadRunnerData(runner).catch(() => loadMockData());
+  process.on("SIGINT", () => {
+    void runner.shutdown().finally(() => process.exit(0));
+  });
+  process.on("SIGTERM", () => {
+    void runner.shutdown().finally(() => process.exit(0));
+  });
+  withFullScreen(
+    /* @__PURE__ */ jsx(EvalsCliApp, { data, args, runner })
+  ).start();
+}
+void main();
+//# sourceMappingURL=out.js.map
+//# sourceMappingURL=cli.js.map