axiom 0.13.0 → 0.14.0

package/dist/bin.cjs

@@ -6,6 +6,11 @@ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
 var __getOwnPropNames = Object.getOwnPropertyNames;
 var __getProtoOf = Object.getPrototypeOf;
 var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
 var __copyProps = (to, from, except, desc) => {
   if (from && typeof from === "object" || typeof from === "function") {
     for (let key of __getOwnPropNames(from))
@@ -22,12 +27,19 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
   isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
   mod
 ));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var __publicField = (obj, key, value) => __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
 
 // src/bin.ts
-var import_env = __toESM(require("@next/env"), 1);
+var bin_exports = {};
+__export(bin_exports, {
+  evalcmd: () => evalcmd,
+  program: () => program
+});
+module.exports = __toCommonJS(bin_exports);
 var import_commander4 = require("commander");
 
-// src/commands/push.command.ts
+// src/cli/commands/push.command.ts
 var import_commander = require("commander");
 
 // src/transpiler.ts
@@ -236,7 +248,7 @@ ${options ? Object.entries(options).map(([key, value]) => `    ${key}: ${value}`
 `;
 }
 
-// src/commands/push.command.ts
+// src/cli/commands/push.command.ts
 var import_promises2 = __toESM(require("fs/promises"), 1);
 var import_node_readline = __toESM(require("readline"), 1);
 async function askConfirmation(message) {
@@ -327,7 +339,7 @@ var loadPushCommand = (program2) => {
   program2.addCommand(push);
 };
 
-// src/commands/pull.command.ts
+// src/cli/commands/pull.command.ts
 var import_commander2 = require("commander");
 var fs3 = __toESM(require("fs/promises"), 1);
 var path2 = __toESM(require("path"), 1);
@@ -338,8 +350,8 @@ var loadPullCommand = (program2) => {
   ).option("--version <version>", "The version to pull, default: latest", "latest").option("--output <path>", "Output file path (optional, defaults to <slug>.prompt.ts)").action(async (slug, options) => {
     try {
       console.log(`Pulling prompt: ${slug} (version: ${options.version})`);
-      const url = `${process.env.AXIOM_URL}/v1/prompts/${slug}`;
-      const response = await fetch(url, {
+      const url2 = `${process.env.AXIOM_URL}/v1/prompts/${slug}`;
+      const response = await fetch(url2, {
         method: "GET",
         headers: {
           Authorization: `Bearer ${process.env.AXIOM_TOKEN}`,
@@ -377,50 +389,502 @@ var loadPullCommand = (program2) => {
   program2.addCommand(pull);
 };
 
-// src/commands/eval.command.ts
+// src/cli/commands/eval.command.ts
 var import_commander3 = require("commander");
 
 // src/evals/run-vitest.ts
 var import_node = require("vitest/node");
 
+// ../../node_modules/.pnpm/tinyrainbow@2.0.0/node_modules/tinyrainbow/dist/chunk-BVHSVHOK.js
+var f = {
+  reset: [0, 0],
+  bold: [1, 22, "\x1B[22m\x1B[1m"],
+  dim: [2, 22, "\x1B[22m\x1B[2m"],
+  italic: [3, 23],
+  underline: [4, 24],
+  inverse: [7, 27],
+  hidden: [8, 28],
+  strikethrough: [9, 29],
+  black: [30, 39],
+  red: [31, 39],
+  green: [32, 39],
+  yellow: [33, 39],
+  blue: [34, 39],
+  magenta: [35, 39],
+  cyan: [36, 39],
+  white: [37, 39],
+  gray: [90, 39],
+  bgBlack: [40, 49],
+  bgRed: [41, 49],
+  bgGreen: [42, 49],
+  bgYellow: [43, 49],
+  bgBlue: [44, 49],
+  bgMagenta: [45, 49],
+  bgCyan: [46, 49],
+  bgWhite: [47, 49],
+  blackBright: [90, 39],
+  redBright: [91, 39],
+  greenBright: [92, 39],
+  yellowBright: [93, 39],
+  blueBright: [94, 39],
+  magentaBright: [95, 39],
+  cyanBright: [96, 39],
+  whiteBright: [97, 39],
+  bgBlackBright: [100, 49],
+  bgRedBright: [101, 49],
+  bgGreenBright: [102, 49],
+  bgYellowBright: [103, 49],
+  bgBlueBright: [104, 49],
+  bgMagentaBright: [105, 49],
+  bgCyanBright: [106, 49],
+  bgWhiteBright: [107, 49]
+};
+var h = Object.entries(f);
+function a(n) {
+  return String(n);
+}
+a.open = "";
+a.close = "";
+function C(n = false) {
+  let e = typeof process != "undefined" ? process : void 0, i = (e == null ? void 0 : e.env) || {}, g = (e == null ? void 0 : e.argv) || [];
+  return !("NO_COLOR" in i || g.includes("--no-color")) && ("FORCE_COLOR" in i || g.includes("--color") || (e == null ? void 0 : e.platform) === "win32" || n && i.TERM !== "dumb" || "CI" in i) || typeof window != "undefined" && !!window.chrome;
+}
+function p(n = false) {
+  let e = C(n), i = (r2, t, c, o) => {
+    let l = "", s2 = 0;
+    do
+      l += r2.substring(s2, o) + c, s2 = o + t.length, o = r2.indexOf(t, s2);
+    while (~o);
+    return l + r2.substring(s2);
+  }, g = (r2, t, c = r2) => {
+    let o = (l) => {
+      let s2 = String(l), b = s2.indexOf(t, r2.length);
+      return ~b ? r2 + i(s2, t, c, b) + t : r2 + s2 + t;
+    };
+    return o.open = r2, o.close = t, o;
+  }, u2 = {
+    isColorSupported: e
+  }, d = (r2) => `\x1B[${r2}m`;
+  for (let [r2, t] of h)
+    u2[r2] = e ? g(
+      d(t[0]),
+      d(t[1]),
+      t[2]
+    ) : a;
+  return u2;
+}
+
+// ../../node_modules/.pnpm/tinyrainbow@2.0.0/node_modules/tinyrainbow/dist/node.js
+var import_tty = require("tty");
+var r = process.env.FORCE_TTY !== void 0 || (0, import_tty.isatty)(1);
+var u = p(r);
+
+// src/evals/eval.service.ts
+var datasetName = process.env.AXIOM_DATASET ?? "";
+var url = process.env.AXIOM_URL ?? "https://api.axiom.co";
+var token = process.env.AXIOM_TOKEN;
+var findEvaluationCases = async (evalId) => {
+  try {
+    const apl = `['${datasetName}'] | where trace_id == "${evalId}" | order by _time`;
+    const headers = new Headers({
+      Authorization: `Bearer ${token}`,
+      "Content-Type": "application/json"
+    });
+    const resp = await fetch(`${url}/v1/datasets/_apl?format=legacy`, {
+      headers,
+      method: "POST",
+      body: JSON.stringify({ apl })
+    });
+    const payload = await resp.json();
+    if (!resp.ok) {
+      console.log(payload);
+      return void 0;
+    }
+    if (payload.matches.length) {
+      return buildSpanTree(payload.matches);
+    }
+  } catch (err) {
+    console.log(err);
+    return void 0;
+  }
+};
+var mapSpanToEval = (span) => {
+  return {
+    id: span.data.attributes.custom["eval.id"],
+    name: span.data.attributes.custom["eval.name"],
+    type: span.data.attributes.custom["eval.type"],
+    version: span.data.attributes.custom["eval.version"],
+    collection: {
+      name: span.data.attributes.custom["eval.collection.name"],
+      size: span.data.attributes.custom["eval.collection.size"]
+    },
+    baseline: {
+      id: span.data.attributes.custom["eval.baseline.id"],
+      name: span.data.attributes.custom["eval.baseline.name"]
+    },
+    prompt: {
+      model: span.data.attributes.custom["eval.prompt.model"],
+      params: span.data.attributes.custom["eval.prompt.params"]
+    },
+    duration: span.data.duration,
+    status: span.data.status.code,
+    traceId: span.data.trace_id,
+    runAt: span._time,
+    tags: span.data.attributes.custom["eval.tags"].length ? JSON.parse(span.data.attributes.custom["eval.tags"]) : [],
+    user: {
+      name: span.data.attributes.custom["eval.user.name"],
+      email: span.data.attributes.custom["eval.user.email"]
+    },
+    cases: []
+  };
+};
+var mapSpanToCase = (item) => {
+  const data = item.data;
+  const d = data.duration;
+  let duration = "-";
+  if (d.endsWith("s")) {
+    duration = `${Number(d.replace("s", "")).toFixed(2)}s`;
+  } else {
+    duration = d;
+  }
+  return {
+    index: data.attributes.custom["eval.case.index"],
+    input: data.attributes.custom["eval.case.input"],
+    output: data.attributes.custom["eval.case.output"],
+    expected: data.attributes.custom["eval.case.expected"],
+    duration,
+    status: data.status.code,
+    scores: data.attributes.custom["eval.case.scores"] ? JSON.parse(data.attributes.custom["eval.case.scores"]) : {},
+    runAt: item._time,
+    spanId: data.span_id,
+    traceId: data.trace_id
+  };
+};
+var buildSpanTree = (spans) => {
+  if (!spans.length) {
+    return null;
+  }
+  const evalSpan = spans.find((span) => span.data.attributes.gen_ai.operation.name === "eval");
+  if (!evalSpan) {
+    return null;
+  }
+  const rootSpan = mapSpanToEval(evalSpan);
+  const caseSpans = spans.filter((span) => span.data.name.startsWith("case"));
+  for (const caseSpan of caseSpans) {
+    const caseData = mapSpanToCase(caseSpan);
+    const taskSpans = spans.filter(
+      (span) => span.data.name.startsWith("task") && span.data.parent_span_id === caseSpan.data.span_id
+    );
+    if (taskSpans.length > 0) {
+      const taskSpan = taskSpans[0];
+      const chatSpans = spans.filter(
+        (span) => span.data.name.startsWith("chat") && span.data.parent_span_id === taskSpan.data.span_id
+      );
+      const chatData = chatSpans.map((chatSpan) => ({
+        operation: chatSpan.data.attributes.custom?.operation || "",
+        capability: chatSpan.data.attributes.custom?.capability || "",
+        step: chatSpan.data.attributes.custom?.step || "",
+        request: {
+          max_token: chatSpan.data.attributes.custom?.["request.max_token"] || "",
+          model: chatSpan.data.attributes.custom?.["request.model"] || "",
+          temperature: chatSpan.data.attributes.custom?.["request.temperature"] || 0
+        },
+        response: {
+          finish_reasons: chatSpan.data.attributes.custom?.["response.finish_reasons"] || ""
+        },
+        usage: {
+          input_tokens: chatSpan.data.attributes.gen_ai?.usage?.input_tokens || 0,
+          output_tokens: chatSpan.data.attributes.gen_ai?.usage?.output_tokens || 0
+        }
+      }));
+      const taskData = {
+        name: taskSpan.data.name,
+        output: taskSpan.data.attributes.custom?.output || "",
+        trial: taskSpan.data.attributes.custom?.trial || 0,
+        type: taskSpan.data.attributes.custom?.type || "",
+        error: taskSpan.data.attributes.custom?.error,
+        chat: chatData[0] || {
+          operation: "",
+          capability: "",
+          step: "",
+          request: { max_token: "", model: "", temperature: 0 },
+          response: { finish_reasons: "" },
+          usage: { input_tokens: 0, output_tokens: 0 }
+        }
+      };
+      caseData.task = taskData;
+    }
+    const scoreSpans = spans.filter(
+      (span) => span.data.attributes.gen_ai.operation.name === "eval.score" && span.data.parent_span_id === caseSpan.data.span_id
+    );
+    caseData.scores = {};
+    scoreSpans.forEach((score) => {
+      const name = score.data.attributes.custom["eval.score.name"];
+      caseData.scores[name] = {
+        name,
+        value: score.data.attributes.custom["eval.score.value"],
+        metadata: {
+          error: score.data.attributes.error
+        }
+      };
+    });
+    rootSpan.cases.push(caseData);
+  }
+  rootSpan.cases.sort((a2, b) => a2.index - b.index);
+  return rootSpan;
+};
+
 // src/evals/reporter.ts
-var import_console_table_printer = require("console-table-printer");
-var prRed = (s) => `\x1B[91m ${s}\x1B[00m`;
 var AxiomReporter = class {
-  onTestSuiteReady(_testSuite) {
+  constructor() {
+    __publicField(this, "baseline");
+    __publicField(this, "startTime", 0);
+    __publicField(this, "start", 0);
+  }
+  onTestRunStart() {
+    this.start = performance.now();
+    this.startTime = (/* @__PURE__ */ new Date()).getTime();
+  }
+  async onTestSuiteReady(_testSuite) {
+    const meta = _testSuite.meta();
+    const baseline = meta.evaluation.baseline;
+    if (baseline) {
+      this.baseline = await findEvaluationCases(baseline.id);
+    }
+    const cwd = process.cwd();
+    console.log(
+      " ",
+      u.bgCyan(u.black(` ${_testSuite.project.name} `)),
+      u.bgBlue(u.black(` ${meta.evaluation.name}-${meta.evaluation.version} `)),
+      u.dim(`(${_testSuite.children.size} cases)`)
+    );
+    console.log(" ", u.dim(_testSuite.module.moduleId.replace(cwd, "")));
+    if (meta.evaluation.baseline) {
+      console.log(
+        " ",
+        " baseline ",
+        u.bgMagenta(
+          u.black(` ${meta.evaluation.baseline.name}-${meta.evaluation.baseline.version} `)
+        )
+      );
+    } else {
+      console.log(" ", u.bgWhite(u.blackBright(" baseline: ")), "none");
+    }
+    console.log("");
+  }
+  onTestCaseReady(test) {
+    const meta = test.meta();
+    console.log(u.blue(` \u2713 evaluating case ${meta.case.index}`));
   }
   onTestSuiteResult(testSuite) {
-    const scoreboard = new import_console_table_printer.Table({
-      title: testSuite.name
-    });
-    for (const test of testSuite.children.array()) {
-      if (test.type !== "test") continue;
-      const testMeta = test.meta();
-      if (!testMeta.eval) {
-        return;
-      }
-      const scores = Object.keys(testMeta.eval.scores).map((k) => {
-        const v = testMeta.eval.scores[k].score ? testMeta.eval.scores[k].score : 0;
-        const scoreValue = Number(v * 100).toFixed(2);
-        const score = testMeta.eval.threshold && v < testMeta.eval.threshold ? prRed(scoreValue + "%") : scoreValue + "%";
-        return [k, score];
-      });
-      scoreboard.addRow({
-        case: testMeta.eval.index.toString(),
-        ...Object.fromEntries(scores)
-      });
+    const duration = Number((performance.now() - this.start) / 1e3).toFixed(2);
+    console.log(" ");
+    console.log(" ", u.dim("Cases"), testSuite.children.size);
+    console.log(" ", u.dim("Start at"), new Date(this.startTime).toTimeString());
+    console.log(" ", u.dim("Duration"), `${duration}s`);
+    const meta = testSuite.meta();
+    const url2 = `https://app.axiom.co/evaluations/${meta.evaluation.name}/${meta.evaluation.id}`;
+    for (const test of testSuite.children) {
+      if (test.type !== "test") return;
+      this.printCaseResult(test);
     }
-    scoreboard.printTable();
+    console.log("");
+    console.log(
+      " ",
+      `see results for ${meta.evaluation.name}-${meta.evaluation.version} at ${url2}`
+    );
+    console.log(
+      " ",
+      u.cyanBright("=== === === === === === === === === === === === === === === ===")
+    );
+    console.log("");
   }
   async onTestRunEnd(_testModules, _errors, _reason) {
   }
+  printCaseResult(test) {
+    const ok = test.ok();
+    const testMeta = test.meta();
+    if (!testMeta || !testMeta.case) {
+      return;
+    }
+    const index = testMeta.case.index;
+    if (ok) {
+      console.log(" ", u.yellow(` \u2714 case ${index}:`));
+    } else {
+      console.log(" ", u.red(` \u2716 case ${index}: failed`));
+      for (const e of testMeta.case.errors ?? []) {
+        console.log("", e.message);
+      }
+    }
+    Object.keys(testMeta.case.scores).forEach((k) => {
+      const v = testMeta.case.scores[k].score ? testMeta.case.scores[k].score : 0;
+      const scoreValue = Number(v * 100).toFixed(2) + "%";
+      if (this.baseline && this.baseline.cases[index] && this.baseline.cases[index].scores[k]) {
+        const baselineScoreValue = this.baseline.cases[index].scores[k].value;
+        const diff = v - baselineScoreValue;
+        const diffText = Number(diff * 100).toFixed(2) + "%";
+        const blScoreText = Number(baselineScoreValue * 100).toFixed(2) + "%";
+        console.log(
+          "   ",
+          k,
+          u.magentaBright(blScoreText),
+          "->",
+          u.blueBright(scoreValue),
+          diff > 0 ? u.green("+" + diffText) : u.red(diffText)
+        );
+      } else {
+        console.log("   ", k, u.blueBright(scoreValue));
+      }
+      return [k, scoreValue];
+    });
+  }
 };
 
 // src/evals/instrument.ts
 var import_sdk_trace_node = require("@opentelemetry/sdk-trace-node");
 var import_resources = require("@opentelemetry/resources");
 var import_exporter_trace_otlp_http = require("@opentelemetry/exporter-trace-otlp-http");
+var import_api2 = require("@opentelemetry/api");
+
+// src/otel/initAxiomAI.ts
 var import_api = require("@opentelemetry/api");
+
+// package.json
+var package_default = {
+  name: "axiom",
+  version: "0.14.0",
+  type: "module",
+  author: "Axiom, Inc.",
+  contributors: [
+    "Islam Shehata <islam@axiom.co>",
+    "Chris Ehrlich <chris@axiom.co>",
+    "Gabriel de Andrade <gabriel@axiom.co>"
+  ],
+  scripts: {
+    dev: "tsup --watch",
+    build: "tsup && chmod +x dist/bin.js",
+    format: "prettier --write .",
+    "format:check": "prettier --check .",
+    lint: "eslint './**/*.{js,ts}'",
+    typecheck: "tsc --noEmit",
+    test: "vitest run",
+    "test:watch": "vitest --watch",
+    publint: "npx publint"
+  },
+  types: "./dist/index.d.ts",
+  main: "./dist/index.cjs",
+  module: "./dist/index.js",
+  bin: {
+    axiom: "./dist/bin.js"
+  },
+  exports: {
+    "./ai": {
+      import: {
+        types: "./dist/index.d.ts",
+        default: "./dist/index.js"
+      },
+      require: {
+        types: "./dist/index.d.cts",
+        default: "./dist/index.cjs"
+      }
+    },
+    "./ai/evals": {
+      import: {
+        types: "./dist/evals.d.ts",
+        default: "./dist/evals.js"
+      },
+      require: {
+        types: "./dist/evals.d.cts",
+        default: "./dist/evals.cjs"
+      }
+    }
+  },
+  keywords: [
+    "axiom",
+    "logging",
+    "ai",
+    "otel",
+    "opentelemetry"
+  ],
+  repository: {
+    type: "git",
+    url: "git+https://github.com/axiomhq/ai.git",
+    directory: "packages/ai"
+  },
+  license: "MIT",
+  dependencies: {
+    "@next/env": "^15.4.2",
+    "@opentelemetry/auto-instrumentations-node": "^0.60.1",
+    "@opentelemetry/context-async-hooks": "^2.0.1",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.202.0",
+    "@opentelemetry/resources": "^2.0.1",
+    "@opentelemetry/sdk-node": "^0.202.0",
+    "@opentelemetry/sdk-trace-node": "^2.0.1",
+    "@opentelemetry/semantic-conventions": "^1.36.0",
+    "@sinclair/typebox": "^0.34.37",
+    commander: "^14.0.0",
+    "console-table-printer": "^2.14.6",
+    esbuild: "^0.25.8",
+    handlebars: "^4.7.8",
+    nanoid: "^5.1.5",
+    vitest: "catalog:",
+    zod: "catalog:"
+  },
+  peerDependencies: {
+    "@opentelemetry/api": "^1.9.0"
+  },
+  devDependencies: {
+    "@ai-sdk/anthropicv1": "npm:@ai-sdk/anthropic@^1.2.12",
+    "@ai-sdk/anthropicv2": "npm:@ai-sdk/anthropic@2.0.0-beta.9",
+    "@ai-sdk/openaiv1": "npm:@ai-sdk/openai@^1.3.23",
+    "@ai-sdk/openaiv2": "npm:@ai-sdk/openai@2.0.0-beta.12",
+    "@ai-sdk/providerv1": "npm:@ai-sdk/provider@^1.1.3",
+    "@ai-sdk/providerv2": "npm:@ai-sdk/provider@2.0.0-beta.1",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/core": "^2.0.1",
+    "@opentelemetry/sdk-trace-base": "^2.0.1",
+    "@opentelemetry/sdk-trace-node": "^2.0.1",
+    "@repo/eslint-config": "workspace:*",
+    "@types/node": "^22.15.29",
+    "@vitest/coverage-v8": "^3.2.4",
+    aiv4: "npm:ai@^4.3.19",
+    aiv5: "npm:ai@^5.0.0",
+    eslint: "catalog:",
+    prettier: "catalog:",
+    tinyrainbow: "^2.0.0",
+    tsup: "catalog:",
+    typescript: "catalog:",
+    vitest: "catalog:"
+  },
+  files: [
+    "dist"
+  ],
+  packageManager: "pnpm@10.11.1"
+};
+
+// src/otel/initAxiomAI.ts
+var AXIOM_AI_SCOPE_KEY = Symbol.for("__axiom_ai_scope__");
+function extractTracerScope(tracer2) {
+  const tracerAny = tracer2;
+  const name = tracerAny._instrumentationScope?.name || tracerAny.instrumentationLibrary?.name || package_default.name;
+  const version = tracerAny._instrumentationScope?.version || tracerAny.instrumentationLibrary?.version || package_default.version;
+  return { name, version };
+}
+function initAxiomAI(config) {
+  const newScope = extractTracerScope(config.tracer);
+  const existingScope = globalThis[AXIOM_AI_SCOPE_KEY];
+  if (existingScope && existingScope.name === newScope.name && existingScope.version === newScope.version) {
+    return;
+  }
+  if (existingScope) {
+    console.warn(
+      `[AxiomAI] initAxiomAI() called multiple times with different scopes. Previous: ${existingScope.name}@${existingScope.version}, New: ${newScope.name}@${newScope.version}`
+    );
+  }
+  globalThis[AXIOM_AI_SCOPE_KEY] = newScope;
+}
+
+// src/evals/instrument.ts
 var collectorOptions = {
   url: process.env.AXIOM_URL ? `${process.env.AXIOM_URL}/v1/traces` : "https://api.axiom.co/v1/traces",
   // Axiom API endpoint for trace data
@@ -442,26 +906,38 @@ var processor = new import_sdk_trace_node.BatchSpanProcessor(exporter, {
 });
 var provider = new import_sdk_trace_node.NodeTracerProvider({
   resource: (0, import_resources.resourceFromAttributes)({
-    ["service.name"]: "axiom-ai",
-    ["service.version"]: "0.13.0"
+    ["service.name"]: "axiom",
+    ["service.version"]: "0.14.0"
   }),
   spanProcessors: [processor]
 });
 provider.register();
-var tracer = import_api.trace.getTracer("axiom-ai", "0.13.0");
+var tracer = import_api2.trace.getTracer("axiom", "0.14.0");
 var flush = async () => {
   await provider.forceFlush();
 };
+initAxiomAI({ tracer });
 
 // src/evals/run-vitest.ts
-var runVitest = async (file) => {
+var DEFAULT_TIMEOUT = 1e4;
+var runVitest = async (dir, opts) => {
   const vi = await (0, import_node.createVitest)("test", {
-    // root: process.cwd(),
+    root: dir ? dir : process.cwd(),
     mode: "test",
-    include: [file ? file : "**/*.eval.ts"],
-    reporters: ["verbose", new AxiomReporter()],
+    include: ["**/*.eval.ts"],
+    reporters: [new AxiomReporter()],
     environment: "node",
-    browser: void 0
+    browser: void 0,
+    watch: opts.watch,
+    name: "axiom:eval",
+    printConsoleTrace: true,
+    silent: false,
+    disableConsoleIntercept: true,
+    testTimeout: DEFAULT_TIMEOUT,
+    globals: true,
+    provide: {
+      baseline: opts.baseline
+    }
   });
   await vi.start();
   const dispose = (0, import_node.registerConsoleShortcuts)(vi, process.stdin, process.stdout);
@@ -474,25 +950,39 @@ var runVitest = async (file) => {
   await flush();
 };
 
-// src/commands/eval.command.ts
-var loadRunCommand = (program2) => {
+// src/cli/commands/eval.command.ts
+var loadEvalCommand = (program2) => {
   return program2.addCommand(
-    new import_commander3.Command("eval").description("run evals locally").argument("<path>", "Path to an eval test file, should be in the form of *.eval.ts").action(async (file) => {
-      if (!process.env.AXIOM_URL || !process.env.AXIOM_TOKEN || !process.env.AXIOM_DATASET) {
-        throw new Error("AXIOM_URL, AXIOM_TOKEN, and AXIOM_DATASET must be set");
+    new import_commander3.Command("eval").description("run evals locally").addArgument(
+      new import_commander3.Argument("[dir]", "path of base directory to scan for *.eval.ts files").default(
+        ".",
+        "any eval file in current directory"
+      )
+    ).option("-w, --watch true", "keep server running and watch for changes", false).option("-t, --token <TOKEN>", "axiom token", process.env.AXIOM_TOKEN).option("-d, --dataset <DATASET>", "axiom dataset name", process.env.AXIOM_DATASET).option("-u, --url <AXIOM URL>", "axiom url", process.env.AXIOM_URL ?? "https://api.axiom.co").option("-b, --baseline <BASELINE ID>", "id of baseline evaluation to compare against").action(async (dir, options) => {
+      if (!options.token || !options.dataset) {
+        throw new Error("AXIOM_TOKEN, and AXIOM_DATASET must be set");
       }
-      await runVitest(file);
+      await runVitest(dir, {
+        watch: options.watch,
+        baseline: options.baseline
+      });
     })
   );
 };
 
 // src/bin.ts
+var import_env = __toESM(require("@next/env"), 1);
 var { loadEnvConfig } = import_env.default;
 loadEnvConfig(process.cwd());
 var program = new import_commander4.Command();
-program.name("axiom").description("Axiom's CLI to manage your objects and run evals").version("0.13.0");
+program.name("axiom").description("Axiom's CLI to manage your objects and run evals").version("0.14.0");
 loadPushCommand(program);
 loadPullCommand(program);
-loadRunCommand(program);
+var evalcmd = loadEvalCommand(program);
 program.parse();
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  evalcmd,
+  program
+});
 //# sourceMappingURL=bin.cjs.map