@m4trix/evals 0.1.0

package/dist/index.js

@@ -0,0 +1,1165 @@
+import { randomUUID } from 'crypto';
+import { Effect, PubSub, Queue, Fiber } from 'effect';
+import { mkdir, appendFile, readdir } from 'fs/promises';
+import { relative, join, dirname, resolve as resolve$1 } from 'path';
+import { pathToFileURL } from 'url';
+
+// src/cli/data.mock.json
+var data_mock_default = {
+  datasets: [
+    {
+      id: "onboarding-flows",
+      name: "onboarding-flows",
+      overview: "Evaluate first-user journeys and schema compliance for generated onboarding payloads.",
+      runs: [
+        {
+          id: "run_2026-02-17_2044",
+          label: "2026-02-17 20:44",
+          status: "FAILED",
+          performance: {
+            passRate: 96,
+            avgScore: 0.91,
+            latencyP95Ms: 710,
+            latencyAvgMs: 502,
+            tokensAvg: 171,
+            tokensP95: 230,
+            costUsd: 24e-4,
+            latencyHistoryMs: [380, 420, 510, 480, 550, 620, 590, 710, 520, 480, 530, 600]
+          },
+          dimensions: [
+            { name: "correctness", score: 82 },
+            { name: "faithfulness", score: 79 },
+            { name: "brevity", score: 68 },
+            { name: "style", score: 90 }
+          ],
+          checks: [
+            { name: "json_schema", passed: false, detail: "3 violations" },
+            { name: "tool_calls", passed: true, detail: "0 unexpected" },
+            { name: "pii_leak", passed: true },
+            { name: "jailbreak", passed: true }
+          ],
+          failures: [
+            { title: "product_parser \u203A conforms to schema (price: string)" },
+            { title: "checkout \u203A tool-call count mismatch" }
+          ],
+          meta: {
+            model: "gpt-4o-mini",
+            provider: "OpenAI",
+            commit: "2f3c1a9",
+            branch: "main",
+            seed: 42,
+            concurrency: 4,
+            duration: "00:01:12",
+            artifact: "./eval-results/run_2026-02-17.jsonl"
+          }
+        },
+        {
+          id: "run_2026-02-16_1112",
+          label: "2026-02-16 11:12",
+          status: "PASS",
+          performance: {
+            passRate: 99,
+            avgScore: 0.95,
+            latencyP95Ms: 650,
+            latencyAvgMs: 488,
+            tokensAvg: 168,
+            tokensP95: 220,
+            costUsd: 2e-3,
+            latencyHistoryMs: [420, 450, 480, 460, 520, 490, 510, 650, 440, 470, 500, 480]
+          },
+          dimensions: [
+            { name: "correctness", score: 89 },
+            { name: "faithfulness", score: 88 },
+            { name: "brevity", score: 72 },
+            { name: "style", score: 93 }
+          ],
+          checks: [
+            { name: "json_schema", passed: true, detail: "0 violations" },
+            { name: "tool_calls", passed: true, detail: "0 unexpected" },
+            { name: "pii_leak", passed: true },
+            { name: "jailbreak", passed: true }
+          ],
+          failures: [],
+          meta: {
+            model: "gpt-4o-mini",
+            provider: "OpenAI",
+            commit: "0d24f8f",
+            branch: "main",
+            seed: 42,
+            concurrency: 4,
+            duration: "00:01:06",
+            artifact: "./eval-results/run_2026-02-16.jsonl"
+          }
+        },
+        {
+          id: "run_2026-02-15_0921",
+          label: "2026-02-15 09:21",
+          status: "PASS",
+          performance: {
+            passRate: 98,
+            avgScore: 0.93,
+            latencyP95Ms: 680,
+            latencyAvgMs: 495,
+            tokensAvg: 175,
+            tokensP95: 235,
+            costUsd: 22e-4,
+            latencyHistoryMs: [450, 480, 520, 490, 550, 580, 620, 680, 510, 470, 530, 560]
+          },
+          dimensions: [
+            { name: "correctness", score: 86 },
+            { name: "faithfulness", score: 84 },
+            { name: "brevity", score: 70 },
+            { name: "style", score: 91 }
+          ],
+          checks: [
+            { name: "json_schema", passed: true, detail: "0 violations" },
+            { name: "tool_calls", passed: true, detail: "0 unexpected" },
+            { name: "pii_leak", passed: true },
+            { name: "jailbreak", passed: true }
+          ],
+          failures: [],
+          meta: {
+            model: "gpt-4o-mini",
+            provider: "OpenAI",
+            commit: "a1b2c3d",
+            branch: "main",
+            seed: 42,
+            concurrency: 4,
+            duration: "00:01:08",
+            artifact: "./eval-results/run_2026-02-15.jsonl"
+          }
+        }
+      ]
+    },
+    {
+      id: "tool-calls",
+      name: "tool-calls",
+      overview: "Validate function-call conformance and unexpected tool invocation behavior.",
+      runs: [
+        {
+          id: "run_2026-02-14_1530",
+          label: "2026-02-14 15:30",
+          status: "PASS",
+          performance: {
+            passRate: 100,
+            avgScore: 1,
+            latencyP95Ms: 320,
+            latencyAvgMs: 280,
+            tokensAvg: 45,
+            tokensP95: 62,
+            costUsd: 8e-4,
+            latencyHistoryMs: [250, 270, 290, 280, 310, 320, 265, 290, 300, 275]
+          },
+          dimensions: [
+            { name: "contract_match", score: 100 },
+            { name: "arg_validity", score: 100 }
+          ],
+          checks: [
+            { name: "tool_calls", passed: true, detail: "0 unexpected" }
+          ],
+          failures: [],
+          meta: {
+            model: "gpt-4o-mini",
+            provider: "OpenAI",
+            commit: "e4f5g6h",
+            branch: "feat/tools",
+            seed: 42,
+            concurrency: 8,
+            duration: "00:00:45",
+            artifact: "./eval-results/tool-calls_2026-02-14.jsonl"
+          }
+        }
+      ]
+    },
+    {
+      id: "json-schema",
+      name: "json-schema",
+      overview: "Stress-test schema fidelity across generated extraction payloads.",
+      runs: []
+    }
+  ],
+  evaluators: [
+    { id: "json-schema-validator", name: "JSON Schema Validator", configPreview: "strict=true" },
+    { id: "tool-call-contract-checker", name: "Tool-call Contract Checker", configPreview: "unexpectedCalls=error" },
+    { id: "rubric-judge", name: "Rubric Judge (LLM)", configPreview: "model=gpt-4o-mini; scale=0-100" },
+    { id: "pii-leak-detector", name: "PII Leak Detector", configPreview: "redact=false" }
+  ]
+};
+
+// src/cli/state.ts
+function loadMockData() {
+  return data_mock_default;
+}
+function toSlug(input) {
+  return input.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
+}
+function toEvalRun(snapshot) {
+  const total = snapshot.totalTestCases === 0 ? 1 : snapshot.totalTestCases;
+  const passRate = Math.round(snapshot.passedTestCases / total * 100);
+  const avgScore = snapshot.passedTestCases / total;
+  const durationMs = snapshot.finishedAt ? snapshot.finishedAt - (snapshot.startedAt ?? snapshot.queuedAt) : Date.now() - (snapshot.startedAt ?? snapshot.queuedAt);
+  return {
+    id: snapshot.runId,
+    label: snapshot.runId.slice(0, 12),
+    status: snapshot.status === "completed" ? "PASS" : snapshot.status === "failed" ? "FAILED" : "RUNNING",
+    performance: {
+      passRate,
+      avgScore,
+      latencyP95Ms: Math.max(1, Math.floor(durationMs / Math.max(1, total))),
+      latencyAvgMs: Math.max(1, Math.floor(durationMs / Math.max(1, total))),
+      tokensAvg: 0,
+      tokensP95: 0,
+      costUsd: 0,
+      latencyHistoryMs: [durationMs]
+    },
+    dimensions: [
+      { name: "passed", score: Math.round(snapshot.passedTestCases / total * 100) },
+      { name: "failed", score: Math.round(snapshot.failedTestCases / total * 100) }
+    ],
+    checks: [
+      {
+        name: "run_status",
+        passed: snapshot.status === "completed",
+        detail: snapshot.status
+      }
+    ],
+    failures: snapshot.errorMessage && snapshot.errorMessage.length > 0 ? [{ title: snapshot.errorMessage }] : [],
+    meta: {
+      model: "n/a",
+      provider: "runner",
+      commit: "local",
+      branch: "local",
+      seed: 0,
+      concurrency: 1,
+      duration: `${durationMs}ms`,
+      artifact: snapshot.artifactPath
+    }
+  };
+}
+function toEvalDataset(item, snapshots) {
+  const runs = snapshots.filter((snapshot) => snapshot.datasetId === item.id).sort((a, b) => b.queuedAt - a.queuedAt).map(toEvalRun);
+  return {
+    id: item.id,
+    name: item.dataset.getName(),
+    overview: `Discovered from ${item.filePath}`,
+    runs
+  };
+}
+function toEvaluatorOption(item) {
+  return {
+    id: item.id,
+    name: item.evaluator.getName() ?? toSlug(item.id),
+    configPreview: `Source: ${item.filePath}`
+  };
+}
+async function loadRunnerData(runner) {
+  const [datasets, evaluators] = await Promise.all([
+    runner.collectDatasets(),
+    runner.collectEvaluators()
+  ]);
+  const snapshots = runner.getAllRunSnapshots();
+  if (datasets.length === 0 && evaluators.length === 0) {
+    return loadMockData();
+  }
+  return {
+    datasets: datasets.map((dataset) => toEvalDataset(dataset, snapshots)),
+    evaluators: evaluators.map(toEvaluatorOption)
+  };
+}
+function parseStartupArgs(argv) {
+  const args = { unknownArgs: [] };
+  for (let index = 0; index < argv.length; index += 1) {
+    const token = argv[index];
+    if (token === "--dataset" && argv[index + 1]) {
+      args.datasetId = argv[index + 1];
+      index += 1;
+      continue;
+    }
+    if (token === "--run" && argv[index + 1]) {
+      args.runId = argv[index + 1];
+      index += 1;
+      continue;
+    }
+    if (token === "--search" && argv[index + 1]) {
+      args.search = argv[index + 1];
+      index += 1;
+      continue;
+    }
+    args.unknownArgs.push(token);
+  }
+  return args;
+}
+
+// src/evals/test-case.ts
+function resolve(value) {
+  return typeof value === "function" ? value() : value;
+}
+var TestCase = class _TestCase {
+  constructor(config) {
+    this._config = config;
+  }
+  static describe(config) {
+    return new _TestCase({
+      name: config.name,
+      tags: config.tags,
+      inputSchema: config.inputSchema,
+      input: config.input
+    });
+  }
+  getName() {
+    return this._config.name;
+  }
+  getTags() {
+    return this._config.tags;
+  }
+  getInputSchema() {
+    return this._config.inputSchema;
+  }
+  getInput() {
+    return resolve(this._config.input);
+  }
+};
+
+// src/evals/evaluator.ts
+var Evaluator = class _Evaluator {
+  constructor(config) {
+    this._config = config;
+  }
+  getState() {
+    return {
+      name: this._config.name,
+      inputSchema: this._config.inputSchema,
+      outputSchema: this._config.outputSchema,
+      scoreSchema: this._config.scoreSchema,
+      middlewares: this._config.middlewares,
+      evaluateFn: this._config.evaluateFn,
+      passThreshold: this._config.passThreshold,
+      passCriterion: this._config.passCriterion
+    };
+  }
+  static use(middleware) {
+    return new _Evaluator({
+      middlewares: [middleware]
+    });
+  }
+  use(middleware) {
+    const state = this.getState();
+    return new _Evaluator({
+      ...state,
+      middlewares: [...state.middlewares, middleware]
+    });
+  }
+  define(config) {
+    const { middlewares } = this.getState();
+    return new _Evaluator({
+      name: config.name,
+      inputSchema: config.inputSchema,
+      outputSchema: config.outputSchema,
+      scoreSchema: config.scoreSchema,
+      middlewares,
+      passThreshold: config.passThreshold,
+      passCriterion: config.passCriterion
+    });
+  }
+  evaluate(fn) {
+    return new _Evaluator({
+      ...this.getState(),
+      evaluateFn: fn
+    });
+  }
+  getName() {
+    return this._config.name;
+  }
+  getInputSchema() {
+    return this._config.inputSchema;
+  }
+  getOutputSchema() {
+    return this._config.outputSchema;
+  }
+  getScoreSchema() {
+    return this._config.scoreSchema;
+  }
+  getMiddlewares() {
+    return this._config.middlewares;
+  }
+  getEvaluateFn() {
+    return this._config.evaluateFn;
+  }
+  getPassThreshold() {
+    return this._config.passThreshold;
+  }
+  getPassCriterion() {
+    return this._config.passCriterion;
+  }
+  async resolveContext() {
+    const parts = await Promise.all(
+      this._config.middlewares.map((mw) => mw.resolve())
+    );
+    return Object.assign({}, ...parts);
+  }
+};
+
+// src/evals/dataset.ts
+function matchesAny(value, matchers) {
+  return matchers.some(
+    (matcher) => typeof matcher === "string" ? value === matcher : matcher.test(value)
+  );
+}
+function matchesAnyPath(filePath, matchers) {
+  return matchers.some((matcher) => {
+    if (typeof matcher === "string") {
+      return simpleGlobMatch(matcher, filePath);
+    }
+    return matcher.test(filePath);
+  });
+}
+function simpleGlobMatch(pattern, value) {
+  const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\?/g, "[^/]").replace(/\*\*\//g, "(?:.*/)?").replace(/\*\*/g, ".*").replace(/\*/g, "[^/]*");
+  return new RegExp(`^${escaped}$`).test(value);
+}
+var Dataset = class _Dataset {
+  constructor(config) {
+    this._config = config;
+  }
+  static define(config) {
+    return new _Dataset({
+      name: config.name,
+      includedTags: config.includedTags ?? [],
+      excludedTags: config.excludedTags ?? [],
+      includedPaths: config.includedPaths ?? [],
+      excludedPaths: config.excludedPaths ?? []
+    });
+  }
+  getName() {
+    return this._config.name;
+  }
+  getIncludedTags() {
+    return this._config.includedTags;
+  }
+  getExcludedTags() {
+    return this._config.excludedTags;
+  }
+  getIncludedPaths() {
+    return this._config.includedPaths;
+  }
+  getExcludedPaths() {
+    return this._config.excludedPaths;
+  }
+  matchesTestCase(testCase, filePath) {
+    const tags = testCase.getTags();
+    if (this._config.excludedTags.length > 0) {
+      if (tags.some((tag) => matchesAny(tag, this._config.excludedTags))) {
+        return false;
+      }
+    }
+    if (this._config.excludedPaths.length > 0) {
+      if (matchesAnyPath(filePath, this._config.excludedPaths)) {
+        return false;
+      }
+    }
+    const tagMatch = this._config.includedTags.length === 0 || tags.some((tag) => matchesAny(tag, this._config.includedTags));
+    const pathMatch = this._config.includedPaths.length === 0 || matchesAnyPath(filePath, this._config.includedPaths);
+    return tagMatch && pathMatch;
+  }
+};
+
+// src/evals/metric.ts
+var registry = /* @__PURE__ */ new Map();
+var Metric = {
+  of(config) {
+    const def = {
+      id: config.id,
+      name: config.name,
+      format: config.format,
+      make: (data) => ({ id: config.id, data })
+    };
+    registry.set(config.id, def);
+    return def;
+  }
+};
+function getMetricById(id) {
+  return registry.get(id);
+}
+
+// src/evals/score.ts
+var registry2 = /* @__PURE__ */ new Map();
+var Score = {
+  of(config) {
+    const def = {
+      id: config.id,
+      name: config.name,
+      displayStrategy: config.displayStrategy,
+      format: config.format,
+      make: (data, options) => {
+        const passed = options?.definePassed !== void 0 ? options.definePassed(data) : void 0;
+        return {
+          id: config.id,
+          data,
+          ...passed !== void 0 && { passed }
+        };
+      }
+    };
+    registry2.set(config.id, def);
+    return def;
+  }
+};
+function getScoreById(id) {
+  return registry2.get(id);
+}
+
+// src/evals/metrics/standard.ts
+var tokenCountMetric = Metric.of({
+  id: "token-count",
+  name: "Tokens",
+  format: (data) => {
+    const input = data.input ?? 0;
+    const output = data.output ?? 0;
+    const inputCached = data.inputCached ?? 0;
+    const outputCached = data.outputCached ?? 0;
+    const cached = inputCached + outputCached;
+    return `in:${input} out:${output} cached:${cached}`;
+  }
+});
+var latencyMetric = Metric.of({
+  id: "latency",
+  name: "Latency",
+  format: (data) => `${data.ms}ms`
+});
+
+// src/evals/scores/standard.ts
+var percentScore = Score.of({
+  id: "percent",
+  name: "Score",
+  displayStrategy: "bar",
+  format: (data) => data.value.toFixed(2)
+});
+var binaryScore = Score.of({
+  id: "binary",
+  name: "Result",
+  displayStrategy: "passFail",
+  format: (data) => data.passed ? "PASSED" : "NOT PASSED"
+});
+
+// src/runner/config.ts
+var defaultRunnerConfig = {
+  discovery: {
+    rootDir: process.cwd(),
+    datasetSuffixes: [".dataset.ts", ".dataset.tsx", ".dataset.js", ".dataset.mjs"],
+    evaluatorSuffixes: [
+      ".evaluator.ts",
+      ".evaluator.tsx",
+      ".evaluator.js",
+      ".evaluator.mjs"
+    ],
+    testCaseSuffixes: [
+      ".test-case.ts",
+      ".test-case.tsx",
+      ".test-case.js",
+      ".test-case.mjs"
+    ],
+    excludeDirectories: ["node_modules", "dist", ".next", ".git", ".pnpm-store"]
+  },
+  artifactDirectory: ".eval-results"
+};
+function withRunnerConfig(overrides) {
+  if (!overrides) {
+    return defaultRunnerConfig;
+  }
+  const discovery = overrides.discovery ? {
+    ...defaultRunnerConfig.discovery,
+    ...overrides.discovery
+  } : defaultRunnerConfig.discovery;
+  return {
+    ...defaultRunnerConfig,
+    ...overrides,
+    discovery
+  };
+}
+var jitiLoader;
+function toId(prefix, filePath, name) {
+  const stable = name && name.trim().length > 0 ? name : filePath;
+  return `${prefix}:${stable}`.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
+}
+function hasMethod(value, methodName) {
+  return typeof value === "object" && value !== null && methodName in value && typeof value[methodName] === "function";
+}
+function isDatasetLike(value) {
+  return hasMethod(value, "getName") && hasMethod(value, "matchesTestCase");
+}
+function isEvaluatorLike(value) {
+  return hasMethod(value, "getName") && hasMethod(value, "resolveContext") && hasMethod(value, "getEvaluateFn");
+}
+function isTestCaseLike(value) {
+  return hasMethod(value, "getName") && hasMethod(value, "getTags") && hasMethod(value, "getInput");
+}
+async function walkDirectory(rootDir, excludeDirectories) {
+  const out = [];
+  async function walk(currentDir) {
+    let entries;
+    try {
+      entries = await readdir(currentDir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    await Promise.all(
+      entries.map(async (entry) => {
+        const absolute = resolve$1(currentDir, entry.name);
+        if (entry.isDirectory()) {
+          if (excludeDirectories.includes(entry.name)) {
+            return;
+          }
+          await walk(absolute);
+          return;
+        }
+        if (entry.isFile()) {
+          out.push(absolute);
+        }
+      })
+    );
+  }
+  await walk(rootDir);
+  return out;
+}
+function hasOneSuffix(filePath, suffixes) {
+  return suffixes.some((suffix) => filePath.endsWith(suffix));
+}
+async function loadModuleExports(filePath) {
+  if (filePath.endsWith(".ts") || filePath.endsWith(".tsx")) {
+    if (!jitiLoader) {
+      const jitiModule = await import('jiti');
+      const createJiti = jitiModule.createJiti ?? jitiModule.default;
+      if (!createJiti) {
+        throw new Error("Failed to initialize jiti TypeScript loader");
+      }
+      jitiLoader = createJiti(import.meta.url, {
+        interopDefault: true,
+        moduleCache: true
+      });
+    }
+    const loaded2 = jitiLoader.import ? await jitiLoader.import(filePath) : await Promise.resolve(jitiLoader(filePath));
+    return Object.values(loaded2);
+  }
+  const moduleUrl = pathToFileURL(filePath).href;
+  const loaded = await import(moduleUrl);
+  return Object.values(loaded);
+}
+async function collectDatasetsFromFiles(config) {
+  const files = await walkDirectory(config.rootDir, config.excludeDirectories);
+  const matched = files.filter(
+    (filePath) => hasOneSuffix(filePath, config.datasetSuffixes)
+  );
+  const found = await Promise.all(
+    matched.map(async (absolutePath) => {
+      const exports = await loadModuleExports(absolutePath);
+      const datasets = exports.filter(isDatasetLike);
+      const relPath = relative(config.rootDir, absolutePath);
+      return datasets.map((dataset) => ({
+        id: toId("dataset", relPath, dataset.getName()),
+        filePath: relPath,
+        dataset
+      }));
+    })
+  );
+  return found.flat();
+}
+async function collectEvaluatorsFromFiles(config) {
+  const files = await walkDirectory(config.rootDir, config.excludeDirectories);
+  const matched = files.filter(
+    (filePath) => hasOneSuffix(filePath, config.evaluatorSuffixes)
+  );
+  const found = await Promise.all(
+    matched.map(async (absolutePath) => {
+      const exports = await loadModuleExports(absolutePath);
+      const evaluators = exports.filter(isEvaluatorLike);
+      const relPath = relative(config.rootDir, absolutePath);
+      return evaluators.map((evaluator) => ({
+        id: toId("evaluator", relPath, evaluator.getName()),
+        filePath: relPath,
+        evaluator
+      }));
+    })
+  );
+  return found.flat();
+}
+async function collectTestCasesFromFiles(config) {
+  const files = await walkDirectory(config.rootDir, config.excludeDirectories);
+  const matched = files.filter(
+    (filePath) => hasOneSuffix(filePath, config.testCaseSuffixes)
+  );
+  const found = await Promise.all(
+    matched.map(async (absolutePath) => {
+      const exports = await loadModuleExports(absolutePath);
+      const testCases = exports.filter(isTestCaseLike);
+      const relPath = relative(config.rootDir, absolutePath);
+      return testCases.map((testCase) => ({
+        id: toId("test-case", relPath, testCase.getName()),
+        filePath: relPath,
+        testCase
+      }));
+    })
+  );
+  return found.flat();
+}
+
+// src/runner/score-utils.ts
+function toNumericScoreFromScores(scores) {
+  for (const item of scores) {
+    const def = getScoreById(item.id);
+    if (def && def.displayStrategy === "bar" && typeof item.data === "object" && item.data !== null && "value" in item.data) {
+      const value = item.data.value;
+      if (typeof value === "number" && Number.isFinite(value)) {
+        return value;
+      }
+    }
+    const numeric = toNumericScore(item.data);
+    if (numeric !== void 0) {
+      return numeric;
+    }
+  }
+  return void 0;
+}
+function toNumericScore(value) {
+  if (typeof value === "number" && Number.isFinite(value)) {
+    return value;
+  }
+  if (typeof value !== "object" || value === null) {
+    return void 0;
+  }
+  const obj = value;
+  if ("score" in obj && typeof obj.score === "number" && Number.isFinite(obj.score)) {
+    return obj.score;
+  }
+  const numberValues = Object.values(value).filter(
+    (entry) => typeof entry === "number" && Number.isFinite(entry)
+  );
+  if (numberValues.length === 0) {
+    return void 0;
+  }
+  return numberValues.reduce((sum, entry) => sum + entry, 0) / numberValues.length;
+}
+
+// src/runner/execution.ts
+function computeEvaluatorPassed(evaluator, result, scores) {
+  const scoresWithPassed = scores.filter((s) => "passed" in s && s.passed !== void 0);
+  if (scoresWithPassed.length > 0) {
+    return scoresWithPassed.every((s) => s.passed === true);
+  }
+  const passCriterion = evaluator.getPassCriterion();
+  if (passCriterion) {
+    return passCriterion(result);
+  }
+  const passThreshold = evaluator.getPassThreshold();
+  if (passThreshold !== void 0) {
+    const numeric = toNumericScoreFromScores(scores);
+    return numeric !== void 0 && numeric >= passThreshold;
+  }
+  return true;
+}
+function normalizeResult(result) {
+  if (typeof result !== "object" || result === null) {
+    return { scores: [] };
+  }
+  const obj = result;
+  const scores = Array.isArray(obj.scores) ? obj.scores : [];
+  const metrics = Array.isArray(obj.metrics) ? obj.metrics : void 0;
+  return { scores, metrics };
+}
+function nowIsoForFile() {
+  return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
+}
+function createArtifactPath(artifactDirectory, datasetId, runId) {
+  return join(
+    artifactDirectory,
+    `${datasetId}_${runId}_${nowIsoForFile()}.jsonl`
+  );
+}
+var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => Effect.gen(function* () {
+  const startedAt = Date.now();
+  updateSnapshot(task.runId, (snapshot) => ({
+    ...snapshot,
+    status: "running",
+    startedAt
+  }));
+  yield* publishEvent({
+    type: "RunStarted",
+    runId: task.runId,
+    startedAt
+  });
+  let completedTestCases = 0;
+  let passedTestCases = 0;
+  let failedTestCases = 0;
+  for (const testCaseItem of task.testCases) {
+    const started = Date.now();
+    const evaluatorScores = [];
+    let testCaseError;
+    for (const { id: evaluatorId, evaluator } of task.evaluators) {
+      const evaluateFn = evaluator.getEvaluateFn();
+      if (!evaluateFn) {
+        continue;
+      }
+      try {
+        const ctx = yield* Effect.promise(
+          () => Promise.resolve(evaluator.resolveContext())
+        );
+        const result = yield* Effect.promise(
+          () => Promise.resolve(evaluateFn(testCaseItem.testCase.getInput(), ctx))
+        );
+        const { scores, metrics } = normalizeResult(result);
+        const passed = computeEvaluatorPassed(evaluator, result, scores);
+        evaluatorScores.push({ evaluatorId, scores, passed, metrics });
+      } catch (error) {
+        testCaseError = error instanceof Error ? error.message : "Evaluator execution failed";
+        evaluatorScores.push({
+          evaluatorId,
+          scores: [],
+          passed: false
+        });
+      }
+    }
+    const testCasePassed = evaluatorScores.every((s) => s.passed);
+    completedTestCases += 1;
+    if (testCasePassed) {
+      passedTestCases += 1;
+    } else {
+      failedTestCases += 1;
+    }
+    const progressEvent = {
+      type: "TestCaseProgress",
+      runId: task.runId,
+      testCaseId: testCaseItem.id,
+      testCaseName: testCaseItem.testCase.getName(),
+      completedTestCases,
+      totalTestCases: task.testCases.length,
+      passed: testCasePassed,
+      durationMs: Date.now() - started,
+      evaluatorScores,
+      errorMessage: testCaseError
+    };
+    updateSnapshot(task.runId, (snapshot) => ({
+      ...snapshot,
+      completedTestCases,
+      passedTestCases,
+      failedTestCases
+    }));
+    yield* publishEvent(progressEvent);
+    yield* Queue.offer(persistenceQueue, {
+      runId: task.runId,
+      artifactPath: task.snapshot.artifactPath,
+      payload: progressEvent
+    });
+  }
+  const finishedAt = Date.now();
+  const completedEvent = {
+    type: "RunCompleted",
+    runId: task.runId,
+    finishedAt,
+    passedTestCases,
+    failedTestCases,
+    totalTestCases: task.testCases.length,
+    artifactPath: task.snapshot.artifactPath
+  };
+  updateSnapshot(task.runId, (snapshot) => ({
+    ...snapshot,
+    status: "completed",
+    completedTestCases,
+    passedTestCases,
+    failedTestCases,
+    finishedAt
+  }));
+  yield* publishEvent(completedEvent);
+  yield* Queue.offer(persistenceQueue, {
+    runId: task.runId,
+    artifactPath: task.snapshot.artifactPath,
+    payload: completedEvent
+  });
+  yield* publishEvent({
+    type: "ArtifactFlushed",
+    runId: task.runId,
+    artifactPath: task.snapshot.artifactPath
+  });
+});
+async function appendJsonLine(artifactPath, payload) {
+  await mkdir(dirname(artifactPath), { recursive: true });
+  await appendFile(artifactPath, `${JSON.stringify(payload)}
+`, "utf8");
+}
+var createPersistenceWorker = (queue) => Effect.forever(
+  Effect.gen(function* () {
+    const message = yield* Queue.take(queue);
+    yield* Effect.promise(
+      () => appendJsonLine(message.artifactPath, {
+        runId: message.runId,
+        ts: Date.now(),
+        ...message.payload
+      })
+    );
+  })
+);
+
+// src/runner/search.ts
+function matchesAny2(value, matchers) {
+  if (!matchers || matchers.length === 0) {
+    return true;
+  }
+  return matchers.some(
+    (matcher) => typeof matcher === "string" ? matcher === value : matcher.test(value)
+  );
+}
+function matchesPath(value, matchers) {
+  if (!matchers || matchers.length === 0) {
+    return true;
+  }
+  return matchers.some((matcher) => {
+    if (typeof matcher === "string") {
+      return value.includes(matcher);
+    }
+    return matcher.test(value);
+  });
+}
+function searchCollectedTestCases(all, query) {
+  if (!query) {
+    return all;
+  }
+  return all.filter((item) => {
+    const tags = item.testCase.getTags();
+    if (query.excludedTags && tags.some((tag) => matchesAny2(tag, query.excludedTags))) {
+      return false;
+    }
+    if (query.excludedPaths && matchesPath(item.filePath, query.excludedPaths)) {
+      return false;
+    }
+    const includedTagsMatch = !query.includedTags || query.includedTags.length === 0 || tags.some((tag) => matchesAny2(tag, query.includedTags));
+    const includedPathsMatch = !query.includedPaths || query.includedPaths.length === 0 || matchesPath(item.filePath, query.includedPaths);
+    return includedTagsMatch && includedPathsMatch;
+  });
+}
+
+// src/runner/api.ts
+function parseRegexLiteral(pattern) {
+  if (!pattern.startsWith("/")) {
+    return void 0;
+  }
+  const lastSlash = pattern.lastIndexOf("/");
+  if (lastSlash <= 0) {
+    return void 0;
+  }
+  return {
+    source: pattern.slice(1, lastSlash),
+    flags: pattern.slice(lastSlash + 1)
+  };
+}
+function createNameMatcher(pattern) {
+  const normalizedPattern = pattern.trim();
+  const regexLiteral = parseRegexLiteral(normalizedPattern);
+  if (regexLiteral) {
+    const regex = new RegExp(regexLiteral.source, regexLiteral.flags);
+    return (value) => regex.test(value);
+  }
+  if (normalizedPattern.includes("*")) {
+    const escaped = normalizedPattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
+    const regex = new RegExp(`^${escaped}$`, "i");
+    return (value) => regex.test(value);
+  }
+  return (value) => value.toLowerCase() === normalizedPattern.toLowerCase();
+}
+function createRunner(overrides) {
+  return new EffectRunner(withRunnerConfig(overrides));
+}
+var EffectRunner = class {
+  constructor(config) {
+    this.eventBus = Effect.runSync(PubSub.unbounded());
+    this.runQueue = Effect.runSync(Queue.unbounded());
+    this.persistenceQueue = Effect.runSync(
+      Queue.unbounded()
+    );
+    this.snapshots = /* @__PURE__ */ new Map();
+    this.listeners = /* @__PURE__ */ new Set();
+    this.datasetsById = /* @__PURE__ */ new Map();
+    this.evaluatorsById = /* @__PURE__ */ new Map();
+    this.schedulerFiber = Effect.runFork(
+      this.createSchedulerEffect()
+    );
+    this.persistenceFiber = Effect.runFork(
+      createPersistenceWorker(this.persistenceQueue)
+    );
+    this.config = config;
+  }
+  async collectDatasets() {
+    const datasets = await collectDatasetsFromFiles(this.config.discovery);
+    this.datasetsById.clear();
+    for (const dataset of datasets) {
+      this.datasetsById.set(dataset.id, dataset);
+    }
+    return datasets;
+  }
+  async collectEvaluators() {
+    const evaluators = await collectEvaluatorsFromFiles(this.config.discovery);
+    this.evaluatorsById.clear();
+    for (const evaluator of evaluators) {
+      this.evaluatorsById.set(evaluator.id, evaluator);
+    }
+    return evaluators;
+  }
+  async resolveDatasetByName(name) {
+    if (this.datasetsById.size === 0) {
+      await this.collectDatasets();
+    }
+    const normalized = name.trim().toLowerCase();
+    return Array.from(this.datasetsById.values()).find(
+      (item) => item.dataset.getName().toLowerCase() === normalized
+    );
+  }
+  async resolveEvaluatorsByNamePattern(pattern) {
+    if (this.evaluatorsById.size === 0) {
+      await this.collectEvaluators();
+    }
+    const matcher = createNameMatcher(pattern);
+    return Array.from(this.evaluatorsById.values()).filter(
+      (item) => matcher(item.evaluator.getName() ?? "")
+    );
+  }
+  async searchTestCases(query) {
+    const testCases = await collectTestCasesFromFiles(this.config.discovery);
+    return searchCollectedTestCases(testCases, query);
+  }
+  async collectDatasetTestCases(datasetId) {
+    if (this.datasetsById.size === 0) {
+      await this.collectDatasets();
+    }
+    const dataset = this.datasetsById.get(datasetId);
+    if (!dataset) {
+      throw new Error(`Unknown dataset: ${datasetId}`);
+    }
+    const allTestCases = await collectTestCasesFromFiles(this.config.discovery);
+    return allTestCases.filter(
+      (testCase) => dataset.dataset.matchesTestCase(testCase.testCase, testCase.filePath)
+    );
+  }
+  async runDatasetWith(request) {
+    if (this.datasetsById.size === 0) {
+      await this.collectDatasets();
+    }
+    if (this.evaluatorsById.size === 0) {
+      await this.collectEvaluators();
+    }
+    const dataset = this.datasetsById.get(request.datasetId);
+    if (!dataset) {
+      throw new Error(`Unknown dataset: ${request.datasetId}`);
+    }
+    const selectedEvaluators = request.evaluatorIds.map((id) => this.evaluatorsById.get(id)).filter((value) => Boolean(value)).map((value) => ({ id: value.id, evaluator: value.evaluator }));
+    if (selectedEvaluators.length === 0) {
+      throw new Error("No evaluators selected for run");
+    }
+    const selectedTestCases = await this.collectDatasetTestCases(request.datasetId);
+    const runId = `run-${randomUUID()}`;
+    const artifactPath = createArtifactPath(
+      this.config.artifactDirectory,
+      request.datasetId,
+      runId
+    );
+    const snapshot = {
+      runId,
+      datasetId: request.datasetId,
+      datasetName: dataset.dataset.getName(),
+      evaluatorIds: selectedEvaluators.map((item) => item.id),
+      queuedAt: Date.now(),
+      totalTestCases: selectedTestCases.length,
+      completedTestCases: 0,
+      passedTestCases: 0,
+      failedTestCases: 0,
+      status: "queued",
+      artifactPath
+    };
+    this.snapshots.set(runId, snapshot);
+    const queuedEvent = {
+      type: "RunQueued",
+      runId,
+      datasetId: request.datasetId,
+      datasetName: dataset.dataset.getName(),
+      evaluatorIds: selectedEvaluators.map((item) => item.id),
+      totalTestCases: selectedTestCases.length,
+      artifactPath
+    };
+    await Effect.runPromise(this.publishEvent(queuedEvent));
+    await Effect.runPromise(
+      Queue.offer(this.persistenceQueue, {
+        runId,
+        artifactPath,
+        payload: queuedEvent
+      })
+    );
+    await Effect.runPromise(
+      Queue.offer(this.runQueue, {
+        runId,
+        datasetId: request.datasetId,
+        dataset: dataset.dataset,
+        evaluators: selectedEvaluators,
+        testCases: selectedTestCases,
+        snapshot
+      })
+    );
+    return snapshot;
+  }
+  subscribeRunEvents(listener, options) {
+    const entry = { runId: options?.runId, listener };
+    this.listeners.add(entry);
+    return () => {
+      this.listeners.delete(entry);
+    };
+  }
+  getRunSnapshot(runId) {
+    return this.snapshots.get(runId);
+  }
+  getAllRunSnapshots() {
+    return Array.from(this.snapshots.values()).sort(
+      (a, b) => b.queuedAt - a.queuedAt
+    );
+  }
+  async shutdown() {
+    await Effect.runPromise(Fiber.interrupt(this.schedulerFiber));
+    await Effect.runPromise(Fiber.interrupt(this.persistenceFiber));
+    await Effect.runPromise(Queue.shutdown(this.runQueue));
+    await Effect.runPromise(Queue.shutdown(this.persistenceQueue));
+    await Effect.runPromise(PubSub.shutdown(this.eventBus));
+  }
+  createSchedulerEffect() {
+    const self = this;
+    return Effect.forever(
+      Effect.gen(function* () {
+        const task = yield* Queue.take(self.runQueue);
+        yield* Effect.fork(
+          executeRunTask(
+            task,
+            self.publishEvent.bind(self),
+            self.persistenceQueue,
+            self.updateSnapshot.bind(self)
+          )
+        );
+      })
+    );
+  }
+  updateSnapshot(runId, updater) {
+    const existing = this.snapshots.get(runId);
+    if (!existing) {
+      return;
+    }
+    this.snapshots.set(runId, updater(existing));
+  }
+  publishEvent(event) {
+    return Effect.sync(() => {
+      for (const entry of this.listeners) {
+        if (entry.runId && entry.runId !== event.runId) {
+          continue;
+        }
+        entry.listener(event);
+      }
+    }).pipe(
+      Effect.flatMap(() => PubSub.publish(this.eventBus, event)),
+      Effect.asVoid
+    );
+  }
+};
+
+export { Dataset, Evaluator, Metric, Score, TestCase, binaryScore, createRunner, defaultRunnerConfig, getMetricById, getScoreById, latencyMetric, loadMockData, loadRunnerData, parseStartupArgs, percentScore, tokenCountMetric, withRunnerConfig };
+//# sourceMappingURL=out.js.map
+//# sourceMappingURL=index.js.map