@m4trix/evals 0.1.0

package/dist/cli-simple.cjs

@@ -0,0 +1,1075 @@
+#!/usr/bin/env node
+'use strict';
+
+var crypto = require('crypto');
+var effect = require('effect');
+var promises = require('fs/promises');
+var path = require('path');
+var url = require('url');
+
+var _documentCurrentScript = typeof document !== 'undefined' ? document.currentScript : null;
+// src/runner/config.ts
+var defaultRunnerConfig = {
+  discovery: {
+    rootDir: process.cwd(),
+    datasetSuffixes: [".dataset.ts", ".dataset.tsx", ".dataset.js", ".dataset.mjs"],
+    evaluatorSuffixes: [
+      ".evaluator.ts",
+      ".evaluator.tsx",
+      ".evaluator.js",
+      ".evaluator.mjs"
+    ],
+    testCaseSuffixes: [
+      ".test-case.ts",
+      ".test-case.tsx",
+      ".test-case.js",
+      ".test-case.mjs"
+    ],
+    excludeDirectories: ["node_modules", "dist", ".next", ".git", ".pnpm-store"]
+  },
+  artifactDirectory: ".eval-results"
+};
+function withRunnerConfig(overrides) {
+  {
+    return defaultRunnerConfig;
+  }
+}
+var jitiLoader;
+function toId(prefix, filePath, name) {
+  const stable = name && name.trim().length > 0 ? name : filePath;
+  return `${prefix}:${stable}`.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
+}
+function hasMethod(value, methodName) {
+  return typeof value === "object" && value !== null && methodName in value && typeof value[methodName] === "function";
+}
+function isDatasetLike(value) {
+  return hasMethod(value, "getName") && hasMethod(value, "matchesTestCase");
+}
+function isEvaluatorLike(value) {
+  return hasMethod(value, "getName") && hasMethod(value, "resolveContext") && hasMethod(value, "getEvaluateFn");
+}
+function isTestCaseLike(value) {
+  return hasMethod(value, "getName") && hasMethod(value, "getTags") && hasMethod(value, "getInput");
+}
+async function walkDirectory(rootDir, excludeDirectories) {
+  const out = [];
+  async function walk(currentDir) {
+    let entries;
+    try {
+      entries = await promises.readdir(currentDir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    await Promise.all(
+      entries.map(async (entry) => {
+        const absolute = path.resolve(currentDir, entry.name);
+        if (entry.isDirectory()) {
+          if (excludeDirectories.includes(entry.name)) {
+            return;
+          }
+          await walk(absolute);
+          return;
+        }
+        if (entry.isFile()) {
+          out.push(absolute);
+        }
+      })
+    );
+  }
+  await walk(rootDir);
+  return out;
+}
+function hasOneSuffix(filePath, suffixes) {
+  return suffixes.some((suffix) => filePath.endsWith(suffix));
+}
+async function loadModuleExports(filePath) {
+  if (filePath.endsWith(".ts") || filePath.endsWith(".tsx")) {
+    if (!jitiLoader) {
+      const jitiModule = await import('jiti');
+      const createJiti = jitiModule.createJiti ?? jitiModule.default;
+      if (!createJiti) {
+        throw new Error("Failed to initialize jiti TypeScript loader");
+      }
+      jitiLoader = createJiti((typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('out.js', document.baseURI).href)), {
+        interopDefault: true,
+        moduleCache: true
+      });
+    }
+    const loaded2 = jitiLoader.import ? await jitiLoader.import(filePath) : await Promise.resolve(jitiLoader(filePath));
+    return Object.values(loaded2);
+  }
+  const moduleUrl = url.pathToFileURL(filePath).href;
+  const loaded = await import(moduleUrl);
+  return Object.values(loaded);
+}
+async function collectDatasetsFromFiles(config) {
+  const files = await walkDirectory(config.rootDir, config.excludeDirectories);
+  const matched = files.filter(
+    (filePath) => hasOneSuffix(filePath, config.datasetSuffixes)
+  );
+  const found = await Promise.all(
+    matched.map(async (absolutePath) => {
+      const exports = await loadModuleExports(absolutePath);
+      const datasets = exports.filter(isDatasetLike);
+      const relPath = path.relative(config.rootDir, absolutePath);
+      return datasets.map((dataset) => ({
+        id: toId("dataset", relPath, dataset.getName()),
+        filePath: relPath,
+        dataset
+      }));
+    })
+  );
+  return found.flat();
+}
+async function collectEvaluatorsFromFiles(config) {
+  const files = await walkDirectory(config.rootDir, config.excludeDirectories);
+  const matched = files.filter(
+    (filePath) => hasOneSuffix(filePath, config.evaluatorSuffixes)
+  );
+  const found = await Promise.all(
+    matched.map(async (absolutePath) => {
+      const exports = await loadModuleExports(absolutePath);
+      const evaluators = exports.filter(isEvaluatorLike);
+      const relPath = path.relative(config.rootDir, absolutePath);
+      return evaluators.map((evaluator) => ({
+        id: toId("evaluator", relPath, evaluator.getName()),
+        filePath: relPath,
+        evaluator
+      }));
+    })
+  );
+  return found.flat();
+}
+async function collectTestCasesFromFiles(config) {
+  const files = await walkDirectory(config.rootDir, config.excludeDirectories);
+  const matched = files.filter(
+    (filePath) => hasOneSuffix(filePath, config.testCaseSuffixes)
+  );
+  const found = await Promise.all(
+    matched.map(async (absolutePath) => {
+      const exports = await loadModuleExports(absolutePath);
+      const testCases = exports.filter(isTestCaseLike);
+      const relPath = path.relative(config.rootDir, absolutePath);
+      return testCases.map((testCase) => ({
+        id: toId("test-case", relPath, testCase.getName()),
+        filePath: relPath,
+        testCase
+      }));
+    })
+  );
+  return found.flat();
+}
+
+// src/evals/metric.ts
+var registry = /* @__PURE__ */ new Map();
+var Metric = {
+  of(config) {
+    const def = {
+      id: config.id,
+      name: config.name,
+      format: config.format,
+      make: (data) => ({ id: config.id, data })
+    };
+    registry.set(config.id, def);
+    return def;
+  }
+};
+function getMetricById(id) {
+  return registry.get(id);
+}
+
+// src/evals/score.ts
+var registry2 = /* @__PURE__ */ new Map();
+var Score = {
+  of(config) {
+    const def = {
+      id: config.id,
+      name: config.name,
+      displayStrategy: config.displayStrategy,
+      format: config.format,
+      make: (data, options) => {
+        const passed = options?.definePassed !== void 0 ? options.definePassed(data) : void 0;
+        return {
+          id: config.id,
+          data,
+          ...passed !== void 0 && { passed }
+        };
+      }
+    };
+    registry2.set(config.id, def);
+    return def;
+  }
+};
+function getScoreById(id) {
+  return registry2.get(id);
+}
+
+// src/evals/metrics/standard.ts
+Metric.of({
+  id: "token-count",
+  name: "Tokens",
+  format: (data) => {
+    const input = data.input ?? 0;
+    const output = data.output ?? 0;
+    const inputCached = data.inputCached ?? 0;
+    const outputCached = data.outputCached ?? 0;
+    const cached = inputCached + outputCached;
+    return `in:${input} out:${output} cached:${cached}`;
+  }
+});
+Metric.of({
+  id: "latency",
+  name: "Latency",
+  format: (data) => `${data.ms}ms`
+});
+
+// src/evals/scores/standard.ts
+Score.of({
+  id: "percent",
+  name: "Score",
+  displayStrategy: "bar",
+  format: (data) => data.value.toFixed(2)
+});
+Score.of({
+  id: "binary",
+  name: "Result",
+  displayStrategy: "passFail",
+  format: (data) => data.passed ? "PASSED" : "NOT PASSED"
+});
+
+// src/runner/score-utils.ts
+function toNumericScoreFromScores(scores) {
+  for (const item of scores) {
+    const def = getScoreById(item.id);
+    if (def && def.displayStrategy === "bar" && typeof item.data === "object" && item.data !== null && "value" in item.data) {
+      const value = item.data.value;
+      if (typeof value === "number" && Number.isFinite(value)) {
+        return value;
+      }
+    }
+    const numeric = toNumericScore(item.data);
+    if (numeric !== void 0) {
+      return numeric;
+    }
+  }
+  return void 0;
+}
+function toNumericScore(value) {
+  if (typeof value === "number" && Number.isFinite(value)) {
+    return value;
+  }
+  if (typeof value !== "object" || value === null) {
+    return void 0;
+  }
+  const obj = value;
+  if ("score" in obj && typeof obj.score === "number" && Number.isFinite(obj.score)) {
+    return obj.score;
+  }
+  const numberValues = Object.values(value).filter(
+    (entry) => typeof entry === "number" && Number.isFinite(entry)
+  );
+  if (numberValues.length === 0) {
+    return void 0;
+  }
+  return numberValues.reduce((sum, entry) => sum + entry, 0) / numberValues.length;
+}
+
+// src/runner/execution.ts
+function computeEvaluatorPassed(evaluator, result, scores) {
+  const scoresWithPassed = scores.filter((s) => "passed" in s && s.passed !== void 0);
+  if (scoresWithPassed.length > 0) {
+    return scoresWithPassed.every((s) => s.passed === true);
+  }
+  const passCriterion = evaluator.getPassCriterion();
+  if (passCriterion) {
+    return passCriterion(result);
+  }
+  const passThreshold = evaluator.getPassThreshold();
+  if (passThreshold !== void 0) {
+    const numeric = toNumericScoreFromScores(scores);
+    return numeric !== void 0 && numeric >= passThreshold;
+  }
+  return true;
+}
+function normalizeResult(result) {
+  if (typeof result !== "object" || result === null) {
+    return { scores: [] };
+  }
+  const obj = result;
+  const scores = Array.isArray(obj.scores) ? obj.scores : [];
+  const metrics = Array.isArray(obj.metrics) ? obj.metrics : void 0;
+  return { scores, metrics };
+}
+function nowIsoForFile() {
+  return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
+}
+function createArtifactPath(artifactDirectory, datasetId, runId) {
+  return path.join(
+    artifactDirectory,
+    `${datasetId}_${runId}_${nowIsoForFile()}.jsonl`
+  );
+}
+var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => effect.Effect.gen(function* () {
+  const startedAt = Date.now();
+  updateSnapshot(task.runId, (snapshot) => ({
+    ...snapshot,
+    status: "running",
+    startedAt
+  }));
+  yield* publishEvent({
+    type: "RunStarted",
+    runId: task.runId,
+    startedAt
+  });
+  let completedTestCases = 0;
+  let passedTestCases = 0;
+  let failedTestCases = 0;
+  for (const testCaseItem of task.testCases) {
+    const started = Date.now();
+    const evaluatorScores = [];
+    let testCaseError;
+    for (const { id: evaluatorId, evaluator } of task.evaluators) {
+      const evaluateFn = evaluator.getEvaluateFn();
+      if (!evaluateFn) {
+        continue;
+      }
+      try {
+        const ctx = yield* effect.Effect.promise(
+          () => Promise.resolve(evaluator.resolveContext())
+        );
+        const result = yield* effect.Effect.promise(
+          () => Promise.resolve(evaluateFn(testCaseItem.testCase.getInput(), ctx))
+        );
+        const { scores, metrics } = normalizeResult(result);
+        const passed = computeEvaluatorPassed(evaluator, result, scores);
+        evaluatorScores.push({ evaluatorId, scores, passed, metrics });
+      } catch (error) {
+        testCaseError = error instanceof Error ? error.message : "Evaluator execution failed";
+        evaluatorScores.push({
+          evaluatorId,
+          scores: [],
+          passed: false
+        });
+      }
+    }
+    const testCasePassed = evaluatorScores.every((s) => s.passed);
+    completedTestCases += 1;
+    if (testCasePassed) {
+      passedTestCases += 1;
+    } else {
+      failedTestCases += 1;
+    }
+    const progressEvent = {
+      type: "TestCaseProgress",
+      runId: task.runId,
+      testCaseId: testCaseItem.id,
+      testCaseName: testCaseItem.testCase.getName(),
+      completedTestCases,
+      totalTestCases: task.testCases.length,
+      passed: testCasePassed,
+      durationMs: Date.now() - started,
+      evaluatorScores,
+      errorMessage: testCaseError
+    };
+    updateSnapshot(task.runId, (snapshot) => ({
+      ...snapshot,
+      completedTestCases,
+      passedTestCases,
+      failedTestCases
+    }));
+    yield* publishEvent(progressEvent);
+    yield* effect.Queue.offer(persistenceQueue, {
+      runId: task.runId,
+      artifactPath: task.snapshot.artifactPath,
+      payload: progressEvent
+    });
+  }
+  const finishedAt = Date.now();
+  const completedEvent = {
+    type: "RunCompleted",
+    runId: task.runId,
+    finishedAt,
+    passedTestCases,
+    failedTestCases,
+    totalTestCases: task.testCases.length,
+    artifactPath: task.snapshot.artifactPath
+  };
+  updateSnapshot(task.runId, (snapshot) => ({
+    ...snapshot,
+    status: "completed",
+    completedTestCases,
+    passedTestCases,
+    failedTestCases,
+    finishedAt
+  }));
+  yield* publishEvent(completedEvent);
+  yield* effect.Queue.offer(persistenceQueue, {
+    runId: task.runId,
+    artifactPath: task.snapshot.artifactPath,
+    payload: completedEvent
+  });
+  yield* publishEvent({
+    type: "ArtifactFlushed",
+    runId: task.runId,
+    artifactPath: task.snapshot.artifactPath
+  });
+});
+async function appendJsonLine(artifactPath, payload) {
+  await promises.mkdir(path.dirname(artifactPath), { recursive: true });
+  await promises.appendFile(artifactPath, `${JSON.stringify(payload)}
+`, "utf8");
+}
+var createPersistenceWorker = (queue) => effect.Effect.forever(
+  effect.Effect.gen(function* () {
+    const message = yield* effect.Queue.take(queue);
+    yield* effect.Effect.promise(
+      () => appendJsonLine(message.artifactPath, {
+        runId: message.runId,
+        ts: Date.now(),
+        ...message.payload
+      })
+    );
+  })
+);
+
+// src/runner/search.ts
+function matchesAny(value, matchers) {
+  if (!matchers || matchers.length === 0) {
+    return true;
+  }
+  return matchers.some(
+    (matcher) => typeof matcher === "string" ? matcher === value : matcher.test(value)
+  );
+}
+function matchesPath(value, matchers) {
+  if (!matchers || matchers.length === 0) {
+    return true;
+  }
+  return matchers.some((matcher) => {
+    if (typeof matcher === "string") {
+      return value.includes(matcher);
+    }
+    return matcher.test(value);
+  });
+}
+function searchCollectedTestCases(all, query) {
+  if (!query) {
+    return all;
+  }
+  return all.filter((item) => {
+    const tags = item.testCase.getTags();
+    if (query.excludedTags && tags.some((tag) => matchesAny(tag, query.excludedTags))) {
+      return false;
+    }
+    if (query.excludedPaths && matchesPath(item.filePath, query.excludedPaths)) {
+      return false;
+    }
+    const includedTagsMatch = !query.includedTags || query.includedTags.length === 0 || tags.some((tag) => matchesAny(tag, query.includedTags));
+    const includedPathsMatch = !query.includedPaths || query.includedPaths.length === 0 || matchesPath(item.filePath, query.includedPaths);
+    return includedTagsMatch && includedPathsMatch;
+  });
+}
+
+// src/runner/api.ts
+function parseRegexLiteral(pattern) {
+  if (!pattern.startsWith("/")) {
+    return void 0;
+  }
+  const lastSlash = pattern.lastIndexOf("/");
+  if (lastSlash <= 0) {
+    return void 0;
+  }
+  return {
+    source: pattern.slice(1, lastSlash),
+    flags: pattern.slice(lastSlash + 1)
+  };
+}
+function createNameMatcher(pattern) {
+  const normalizedPattern = pattern.trim();
+  const regexLiteral = parseRegexLiteral(normalizedPattern);
+  if (regexLiteral) {
+    const regex = new RegExp(regexLiteral.source, regexLiteral.flags);
+    return (value) => regex.test(value);
+  }
+  if (normalizedPattern.includes("*")) {
+    const escaped = normalizedPattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
+    const regex = new RegExp(`^${escaped}$`, "i");
+    return (value) => regex.test(value);
+  }
+  return (value) => value.toLowerCase() === normalizedPattern.toLowerCase();
+}
+function createRunner(overrides) {
+  return new EffectRunner(withRunnerConfig());
+}
+var EffectRunner = class {
+  constructor(config) {
+    this.eventBus = effect.Effect.runSync(effect.PubSub.unbounded());
+    this.runQueue = effect.Effect.runSync(effect.Queue.unbounded());
+    this.persistenceQueue = effect.Effect.runSync(
+      effect.Queue.unbounded()
+    );
+    this.snapshots = /* @__PURE__ */ new Map();
+    this.listeners = /* @__PURE__ */ new Set();
+    this.datasetsById = /* @__PURE__ */ new Map();
+    this.evaluatorsById = /* @__PURE__ */ new Map();
+    this.schedulerFiber = effect.Effect.runFork(
+      this.createSchedulerEffect()
+    );
+    this.persistenceFiber = effect.Effect.runFork(
+      createPersistenceWorker(this.persistenceQueue)
+    );
+    this.config = config;
+  }
+  async collectDatasets() {
+    const datasets = await collectDatasetsFromFiles(this.config.discovery);
+    this.datasetsById.clear();
+    for (const dataset of datasets) {
+      this.datasetsById.set(dataset.id, dataset);
+    }
+    return datasets;
+  }
+  async collectEvaluators() {
+    const evaluators = await collectEvaluatorsFromFiles(this.config.discovery);
+    this.evaluatorsById.clear();
+    for (const evaluator of evaluators) {
+      this.evaluatorsById.set(evaluator.id, evaluator);
+    }
+    return evaluators;
+  }
+  async resolveDatasetByName(name) {
+    if (this.datasetsById.size === 0) {
+      await this.collectDatasets();
+    }
+    const normalized = name.trim().toLowerCase();
+    return Array.from(this.datasetsById.values()).find(
+      (item) => item.dataset.getName().toLowerCase() === normalized
+    );
+  }
+  async resolveEvaluatorsByNamePattern(pattern) {
+    if (this.evaluatorsById.size === 0) {
+      await this.collectEvaluators();
+    }
+    const matcher = createNameMatcher(pattern);
+    return Array.from(this.evaluatorsById.values()).filter(
+      (item) => matcher(item.evaluator.getName() ?? "")
+    );
+  }
+  async searchTestCases(query) {
+    const testCases = await collectTestCasesFromFiles(this.config.discovery);
+    return searchCollectedTestCases(testCases, query);
+  }
+  async collectDatasetTestCases(datasetId) {
+    if (this.datasetsById.size === 0) {
+      await this.collectDatasets();
+    }
+    const dataset = this.datasetsById.get(datasetId);
+    if (!dataset) {
+      throw new Error(`Unknown dataset: ${datasetId}`);
+    }
+    const allTestCases = await collectTestCasesFromFiles(this.config.discovery);
+    return allTestCases.filter(
+      (testCase) => dataset.dataset.matchesTestCase(testCase.testCase, testCase.filePath)
+    );
+  }
+  async runDatasetWith(request) {
+    if (this.datasetsById.size === 0) {
+      await this.collectDatasets();
+    }
+    if (this.evaluatorsById.size === 0) {
+      await this.collectEvaluators();
+    }
+    const dataset = this.datasetsById.get(request.datasetId);
+    if (!dataset) {
+      throw new Error(`Unknown dataset: ${request.datasetId}`);
+    }
+    const selectedEvaluators = request.evaluatorIds.map((id) => this.evaluatorsById.get(id)).filter((value) => Boolean(value)).map((value) => ({ id: value.id, evaluator: value.evaluator }));
+    if (selectedEvaluators.length === 0) {
+      throw new Error("No evaluators selected for run");
+    }
+    const selectedTestCases = await this.collectDatasetTestCases(request.datasetId);
+    const runId = `run-${crypto.randomUUID()}`;
+    const artifactPath = createArtifactPath(
+      this.config.artifactDirectory,
+      request.datasetId,
+      runId
+    );
+    const snapshot = {
+      runId,
+      datasetId: request.datasetId,
+      datasetName: dataset.dataset.getName(),
+      evaluatorIds: selectedEvaluators.map((item) => item.id),
+      queuedAt: Date.now(),
+      totalTestCases: selectedTestCases.length,
+      completedTestCases: 0,
+      passedTestCases: 0,
+      failedTestCases: 0,
+      status: "queued",
+      artifactPath
+    };
+    this.snapshots.set(runId, snapshot);
+    const queuedEvent = {
+      type: "RunQueued",
+      runId,
+      datasetId: request.datasetId,
+      datasetName: dataset.dataset.getName(),
+      evaluatorIds: selectedEvaluators.map((item) => item.id),
+      totalTestCases: selectedTestCases.length,
+      artifactPath
+    };
+    await effect.Effect.runPromise(this.publishEvent(queuedEvent));
+    await effect.Effect.runPromise(
+      effect.Queue.offer(this.persistenceQueue, {
+        runId,
+        artifactPath,
+        payload: queuedEvent
+      })
+    );
+    await effect.Effect.runPromise(
+      effect.Queue.offer(this.runQueue, {
+        runId,
+        datasetId: request.datasetId,
+        dataset: dataset.dataset,
+        evaluators: selectedEvaluators,
+        testCases: selectedTestCases,
+        snapshot
+      })
+    );
+    return snapshot;
+  }
+  subscribeRunEvents(listener, options) {
+    const entry = { runId: options?.runId, listener };
+    this.listeners.add(entry);
+    return () => {
+      this.listeners.delete(entry);
+    };
+  }
+  getRunSnapshot(runId) {
+    return this.snapshots.get(runId);
+  }
+  getAllRunSnapshots() {
+    return Array.from(this.snapshots.values()).sort(
+      (a, b) => b.queuedAt - a.queuedAt
+    );
+  }
+  async shutdown() {
+    await effect.Effect.runPromise(effect.Fiber.interrupt(this.schedulerFiber));
+    await effect.Effect.runPromise(effect.Fiber.interrupt(this.persistenceFiber));
+    await effect.Effect.runPromise(effect.Queue.shutdown(this.runQueue));
+    await effect.Effect.runPromise(effect.Queue.shutdown(this.persistenceQueue));
+    await effect.Effect.runPromise(effect.PubSub.shutdown(this.eventBus));
+  }
+  createSchedulerEffect() {
+    const self = this;
+    return effect.Effect.forever(
+      effect.Effect.gen(function* () {
+        const task = yield* effect.Queue.take(self.runQueue);
+        yield* effect.Effect.fork(
+          executeRunTask(
+            task,
+            self.publishEvent.bind(self),
+            self.persistenceQueue,
+            self.updateSnapshot.bind(self)
+          )
+        );
+      })
+    );
+  }
+  updateSnapshot(runId, updater) {
+    const existing = this.snapshots.get(runId);
+    if (!existing) {
+      return;
+    }
+    this.snapshots.set(runId, updater(existing));
+  }
+  publishEvent(event) {
+    return effect.Effect.sync(() => {
+      for (const entry of this.listeners) {
+        if (entry.runId && entry.runId !== event.runId) {
+          continue;
+        }
+        entry.listener(event);
+      }
+    }).pipe(
+      effect.Effect.flatMap(() => effect.PubSub.publish(this.eventBus, event)),
+      effect.Effect.asVoid
+    );
+  }
+};
+
+// src/cli-simple/args.ts
+function parseSimpleCliArgs(argv) {
+  const args = {
+    help: false,
+    unknownArgs: []
+  };
+  let index = 0;
+  if (argv[0] === "run" || argv[0] === "generate") {
+    args.command = argv[0];
+    index = 1;
+  }
+  for (; index < argv.length; index += 1) {
+    const token = argv[index];
+    if (token === "--help" || token === "-h") {
+      args.help = true;
+      continue;
+    }
+    if ((token === "--dataset" || token === "--datasetName") && argv[index + 1]) {
+      args.datasetName = argv[index + 1];
+      index += 1;
+      continue;
+    }
+    if ((token === "--evaluator" || token === "--name") && argv[index + 1]) {
+      args.evaluatorPattern = argv[index + 1];
+      index += 1;
+      continue;
+    }
+    args.unknownArgs.push(token);
+  }
+  return args;
+}
+function getSimpleCliUsage() {
+  return [
+    "Usage:",
+    "  eval-agents-simple run --dataset <datasetName> --evaluator <name-or-pattern>",
+    "  eval-agents-simple generate --dataset <datasetName>",
+    "",
+    "Pattern examples for --evaluator:",
+    "  score-evaluator       exact name (case-insensitive)",
+    '  "*score*"             wildcard pattern',
+    '  "/score/i"            regex literal'
+  ].join("\n");
+}
+function createOutputPath(datasetFilePath) {
+  const parsed = path.parse(datasetFilePath);
+  return path.join(parsed.dir, `${parsed.name}.cases.json`);
+}
+async function generateDatasetJsonCommand(runner, datasetName) {
+  const dataset = await runner.resolveDatasetByName(datasetName);
+  if (!dataset) {
+    throw new Error(`Dataset "${datasetName}" not found.`);
+  }
+  const testCases = await runner.collectDatasetTestCases(dataset.id);
+  const payload = testCases.map((item) => ({
+    name: item.testCase.getName(),
+    input: item.testCase.getInput()
+  }));
+  const absoluteDatasetPath = path.resolve(process.cwd(), dataset.filePath);
+  const outputPath = createOutputPath(absoluteDatasetPath);
+  await promises.writeFile(outputPath, `${JSON.stringify(payload, null, 2)}
+`, "utf8");
+  console.log(`Generated ${payload.length} test cases for dataset "${dataset.dataset.getName()}".`);
+  console.log(`Wrote ${outputPath}`);
+}
+
+// src/cli-simple/run.ts
+var ansi = {
+  reset: "\x1B[0m",
+  bold: "\x1B[1m",
+  dim: "\x1B[2m",
+  green: "\x1B[32m",
+  yellow: "\x1B[33m",
+  red: "\x1B[31m",
+  cyan: "\x1B[36m",
+  magenta: "\x1B[35m"
+};
+function colorize(text, color) {
+  return `${color}${text}${ansi.reset}`;
+}
+function scoreToColor(score) {
+  if (score >= 80) {
+    return ansi.green;
+  }
+  if (score >= 50) {
+    return ansi.yellow;
+  }
+  return ansi.red;
+}
+function getEvaluatorSummaryLine(evaluatorName, aggregate) {
+  if (!aggregate || aggregate.count === 0) {
+    return `- ${evaluatorName.padEnd(28)} no numeric scores`;
+  }
+  const mean = aggregate.total / aggregate.count;
+  return `- ${evaluatorName.padEnd(28)} avg=${colorize(mean.toFixed(2), scoreToColor(mean))} passed=${aggregate.passed} failed=${aggregate.failed}`;
+}
+function createBar(value, max = 100, width = 20) {
+  const safe = Math.max(0, Math.min(max, value));
+  const filled = Math.round(safe / max * width);
+  return `${"\u2588".repeat(filled)}${"\u2591".repeat(width - filled)}`;
+}
+function formatEvaluatorScoreLine(name, scores, passed, metrics) {
+  const passLabel = passed ? colorize("PASS", `${ansi.bold}${ansi.green}`) : colorize("FAIL", `${ansi.bold}${ansi.red}`);
+  const scoreParts = [];
+  for (const item of scores) {
+    const def = getScoreById(item.id);
+    if (!def) {
+      const numeric = toNumericScore(item.data);
+      scoreParts.push(
+        numeric !== void 0 ? colorize(numeric.toFixed(2), scoreToColor(numeric)) : "n/a"
+      );
+      continue;
+    }
+    const formatted = def.format(item.data);
+    switch (def.displayStrategy) {
+      case "bar": {
+        const numeric = typeof item.data === "object" && item.data !== null && "value" in item.data ? item.data.value : toNumericScore(item.data);
+        if (typeof numeric === "number" && Number.isFinite(numeric)) {
+          scoreParts.push(
+            `${colorize(formatted, scoreToColor(numeric))} ${colorize(createBar(numeric), ansi.dim)}`
+          );
+        } else {
+          scoreParts.push(formatted);
+        }
+        break;
+      }
+      case "number":
+        scoreParts.push(formatted);
+        break;
+      case "passFail":
+        scoreParts.push(
+          colorize(
+            formatted,
+            item.passed === true ? `${ansi.bold}${ansi.green}` : item.passed === false ? `${ansi.bold}${ansi.red}` : ansi.dim
+          )
+        );
+        break;
+    }
+  }
+  const scoreStr = scoreParts.length > 0 ? scoreParts.join(" ") : "n/a";
+  let line = `   ${name}: ${passLabel} ${scoreStr}`;
+  if (metrics && metrics.length > 0) {
+    const metricParts = [];
+    for (const { id, data } of metrics) {
+      const def = getMetricById(id);
+      if (def) {
+        const formatted = def.format(data);
+        metricParts.push(
+          def.name ? `[${def.name}: ${formatted}]` : `[${formatted}]`
+        );
+      }
+    }
+    if (metricParts.length > 0) {
+      line += ` ${metricParts.join(" ")}`;
+    }
+  }
+  return line;
+}
+async function runSimpleEvalCommand(runner, datasetName, evaluatorPattern) {
+  const dataset = await runner.resolveDatasetByName(datasetName);
+  if (!dataset) {
+    const known = await runner.collectDatasets();
+    const available = known.map((item) => item.dataset.getName()).sort();
+    throw new Error(
+      available.length > 0 ? `Dataset "${datasetName}" not found. Available datasets: ${available.join(", ")}` : `Dataset "${datasetName}" not found and no datasets were discovered.`
+    );
+  }
+  const evaluators = await runner.resolveEvaluatorsByNamePattern(evaluatorPattern);
+  if (evaluators.length === 0) {
+    const known = await runner.collectEvaluators();
+    const available = known.map((item) => item.evaluator.getName()).filter((name) => typeof name === "string").sort();
+    throw new Error(
+      available.length > 0 ? `No evaluator matched "${evaluatorPattern}". Available evaluators: ${available.join(", ")}` : `No evaluator matched "${evaluatorPattern}" and no evaluators were discovered.`
+    );
+  }
+  const evaluatorNameById = new Map(
+    evaluators.map((item) => [item.id, item.evaluator.getName() ?? item.id])
+  );
+  const aggregates = /* @__PURE__ */ new Map();
+  const testCaseSummaries = [];
+  let overallScoreTotal = 0;
+  let overallScoreCount = 0;
+  let completedCount = 0;
+  let totalCount = 0;
+  let runFinished = false;
+  const spinnerFrames = ["\u280B", "\u2819", "\u2838", "\u2834", "\u2826", "\u2807"];
+  let spinnerIndex = 0;
+  function clearLine() {
+    if (!process.stdout.isTTY) {
+      return;
+    }
+    process.stdout.write("\r\x1B[2K");
+  }
+  function drawSpinner() {
+    if (!process.stdout.isTTY || runFinished) {
+      return;
+    }
+    const frame = spinnerFrames[spinnerIndex % spinnerFrames.length];
+    spinnerIndex += 1;
+    process.stdout.write(
+      `\r${colorize(frame, ansi.cyan)} Running evaluations ${colorize(
+        `${completedCount}/${totalCount}`,
+        ansi.bold
+      )} ${colorize("(live)", ansi.dim)}`
+    );
+  }
+  let spinnerTimer;
+  const done = new Promise((resolve3) => {
+    const unsubscribe = runner.subscribeRunEvents((event) => {
+      if (event.type === "TestCaseProgress") {
+        completedCount = event.completedTestCases;
+        const numericScores = event.evaluatorScores.map((item) => toNumericScoreFromScores(item.scores)).filter((item) => item !== void 0);
+        const averageScore = numericScores.length > 0 ? numericScores.reduce((sum, value) => sum + value, 0) / numericScores.length : void 0;
+        clearLine();
+        console.log(
+          `${colorize(`[${event.completedTestCases}/${event.totalTestCases}]`, ansi.cyan)} ${event.testCaseName} ${colorize(`(${event.durationMs}ms)`, ansi.dim)}`
+        );
+        for (const item of event.evaluatorScores) {
+          const name = evaluatorNameById.get(item.evaluatorId) ?? item.evaluatorId;
+          console.log(
+            formatEvaluatorScoreLine(
+              name,
+              item.scores,
+              item.passed,
+              item.metrics
+            )
+          );
+          const numeric = toNumericScoreFromScores(item.scores);
+          if (numeric !== void 0) {
+            const current = aggregates.get(item.evaluatorId) ?? {
+              total: 0,
+              count: 0,
+              passed: 0,
+              failed: 0
+            };
+            aggregates.set(item.evaluatorId, {
+              total: current.total + numeric,
+              count: current.count + 1,
+              passed: current.passed + (item.passed ? 1 : 0),
+              failed: current.failed + (item.passed ? 0 : 1)
+            });
+            overallScoreTotal += numeric;
+            overallScoreCount += 1;
+          }
+        }
+        testCaseSummaries.push({
+          name: event.testCaseName,
+          averageScore,
+          durationMs: event.durationMs,
+          passed: event.passed
+        });
+        drawSpinner();
+      }
+      if (event.type === "RunCompleted" || event.type === "RunFailed") {
+        runFinished = true;
+        clearLine();
+        unsubscribe();
+        resolve3(event);
+      }
+    });
+  });
+  const snapshot = await runner.runDatasetWith({
+    datasetId: dataset.id,
+    evaluatorIds: evaluators.map((item) => item.id)
+  });
+  totalCount = snapshot.totalTestCases;
+  console.log(colorize("=== Eval Run Started ===", `${ansi.bold}${ansi.cyan}`));
+  console.log(`Run: ${colorize(snapshot.runId, ansi.cyan)}`);
+  console.log(`Dataset: ${colorize(snapshot.datasetName, ansi.bold)}`);
+  console.log(
+    `Evaluators: ${evaluators.map((item) => item.evaluator.getName() ?? item.id).join(", ")}`
+  );
+  console.log(
+    `Total test cases: ${colorize(String(snapshot.totalTestCases), ansi.bold)}`
+  );
+  console.log("");
+  drawSpinner();
+  spinnerTimer = setInterval(drawSpinner, 100);
+  const finalEvent = await done;
+  if (spinnerTimer) {
+    clearInterval(spinnerTimer);
+  }
+  if (finalEvent.type === "RunFailed") {
+    throw new Error(`Run failed: ${finalEvent.errorMessage}`);
+  }
+  console.log("");
+  console.log(colorize("=== Run Summary ===", `${ansi.bold}${ansi.cyan}`));
+  console.log(
+    `- passed: ${colorize(
+      `${finalEvent.passedTestCases}/${finalEvent.totalTestCases}`,
+      ansi.green
+    )}`
+  );
+  console.log(
+    `- failed: ${colorize(
+      `${finalEvent.failedTestCases}/${finalEvent.totalTestCases}`,
+      finalEvent.failedTestCases > 0 ? ansi.red : ansi.dim
+    )}`
+  );
+  if (overallScoreCount > 0) {
+    const overallAverage = overallScoreTotal / overallScoreCount;
+    console.log(
+      `- overall avg score: ${colorize(
+        overallAverage.toFixed(2),
+        scoreToColor(overallAverage)
+      )} ${colorize(createBar(overallAverage), ansi.dim)}`
+    );
+  }
+  console.log(colorize("- evaluator averages:", ansi.magenta));
+  for (const [evaluatorId, evaluatorName] of evaluatorNameById.entries()) {
+    console.log(
+      getEvaluatorSummaryLine(evaluatorName, aggregates.get(evaluatorId))
+    );
+  }
+  if (testCaseSummaries.length > 0) {
+    console.log(colorize("- test case scores:", ansi.magenta));
+    for (const summary of testCaseSummaries) {
+      const status = summary.passed ? colorize("PASS", ansi.green) : colorize("FAIL", ansi.red);
+      if (summary.averageScore === void 0) {
+        console.log(
+          `  ${status} ${summary.name.padEnd(24)} score=n/a ${colorize(`(${summary.durationMs}ms)`, ansi.dim)}`
+        );
+        continue;
+      }
+      console.log(
+        `  ${status} ${summary.name.padEnd(24)} score=${colorize(
+          summary.averageScore.toFixed(2),
+          scoreToColor(summary.averageScore)
+        )} ${colorize(createBar(summary.averageScore, 100, 14), ansi.dim)} ${colorize(`(${summary.durationMs}ms)`, ansi.dim)}`
+      );
+    }
+  }
+  console.log(`- artifact: ${colorize(finalEvent.artifactPath, ansi.dim)}`);
+}
+
+// src/cli-simple/index.ts
+function printUsageAndExit(exitCode) {
+  const printer = exitCode === 0 ? console.log : console.error;
+  printer(getSimpleCliUsage());
+  process.exit(exitCode);
+}
+async function main() {
+  const args = parseSimpleCliArgs(process.argv.slice(2));
+  if (args.help) {
+    printUsageAndExit(0);
+  }
+  if (args.unknownArgs.length > 0) {
+    console.error(`Unknown arguments: ${args.unknownArgs.join(", ")}`);
+    printUsageAndExit(1);
+  }
+  if (!args.command) {
+    printUsageAndExit(1);
+  }
+  if (!args.datasetName) {
+    console.error("Missing required --dataset <datasetName> argument.");
+    printUsageAndExit(1);
+  }
+  const runner = createRunner();
+  try {
+    if (args.command === "run") {
+      if (!args.evaluatorPattern) {
+        console.error("Missing required --evaluator <name-or-pattern> argument.");
+        printUsageAndExit(1);
+      }
+      await runSimpleEvalCommand(runner, args.datasetName, args.evaluatorPattern);
+      return;
+    }
+    await generateDatasetJsonCommand(runner, args.datasetName);
+  } finally {
+    await runner.shutdown();
+  }
+}
+void main().catch((error) => {
+  console.error(error instanceof Error ? error.message : "Command failed");
+  process.exit(1);
+});
+//# sourceMappingURL=out.js.map
+//# sourceMappingURL=cli-simple.cjs.map