axiom 0.25.0 → 0.26.0

package/dist/bin.cjs

@@ -255,10 +255,10 @@ async function askConfirmation(message) {
     input: process.stdin,
     output: process.stdout
   });
-  return new Promise((resolve3) => {
+  return new Promise((resolve4) => {
     rl.question(`${message} (y/N): `, (answer) => {
       rl.close();
-      resolve3(answer.toLowerCase() === "y" || answer.toLowerCase() === "yes");
+      resolve4(answer.toLowerCase() === "y" || answer.toLowerCase() === "yes");
     });
   });
 }
@@ -477,7 +477,11 @@ var r = process.env.FORCE_TTY !== void 0 || (0, import_tty.isatty)(1);
 var u = p(r);
 
 // src/evals/run-vitest.ts
-var import_node_path3 = __toESM(require("path"), 1);
+var import_node_url = require("url");
+var import_node_path3 = require("path");
+var import_node_fs = require("fs");
+var import_node_os2 = require("os");
+var import_node_path4 = __toESM(require("path"), 1);
 var import_node = require("vitest/node");
 
 // src/evals/context/storage.ts
@@ -879,7 +883,7 @@ var import_api4 = require("@opentelemetry/api");
 // package.json
 var package_default = {
   name: "axiom",
-  version: "0.25.0",
+  version: "0.26.0",
   type: "module",
   author: "Axiom, Inc.",
   contributors: [
@@ -1998,11 +2002,11 @@ function setupEvalProvider(connection) {
   axiomProvider = new import_sdk_trace_node.NodeTracerProvider({
     resource: (0, import_resources.resourceFromAttributes)({
       ["service.name"]: "axiom",
-      ["service.version"]: "0.25.0"
+      ["service.version"]: "0.26.0"
     }),
     spanProcessors: [processor]
   });
-  axiomTracer = axiomProvider.getTracer("axiom", "0.25.0");
+  axiomTracer = axiomProvider.getTracer("axiom", "0.26.0");
 }
 async function initInstrumentation(config) {
   if (initialized) {
@@ -2014,7 +2018,7 @@ async function initInstrumentation(config) {
   }
   initializationPromise = (async () => {
     if (!config.enabled) {
-      axiomTracer = import_api10.trace.getTracer("axiom", "0.25.0");
+      axiomTracer = import_api10.trace.getTracer("axiom", "0.26.0");
       initialized = true;
       return;
     }
@@ -2077,6 +2081,9 @@ var flush = async () => {
 };
 
 // src/evals/run-vitest.ts
+var import_meta2 = {};
+var __filename = (0, import_node_url.fileURLToPath)(import_meta2.url);
+var __dirname = (0, import_node_path3.dirname)(__filename);
 var printCollectedEvals = (result, rootDir) => {
   if (!result.testModules || result.testModules.length === 0) {
     console.log(u.yellow("\nNo evaluations found\n"));
@@ -2086,7 +2093,7 @@ var printCollectedEvals = (result, rootDir) => {
   let totalEvals = 0;
   let totalCases = 0;
   for (const module2 of result.testModules) {
-    const relativePath = import_node_path3.default.relative(rootDir, module2.moduleId);
+    const relativePath = import_node_path4.default.relative(rootDir, module2.moduleId);
     for (const suite of module2.children.suites()) {
       totalEvals++;
       const caseCount = suite.children.size;
@@ -2116,6 +2123,16 @@ var runVitest = async (dir, opts) => {
   if (opts.debug) {
     console.log(u.bgWhite(u.blackBright(" Debug mode enabled ")));
   }
+  const tmpDir = (0, import_node_path3.join)((0, import_node_os2.tmpdir)(), "axiom-eval", opts.runId);
+  (0, import_node_fs.mkdirSync)(tmpDir, { recursive: true });
+  const nameRegistryFile = (0, import_node_path3.join)(tmpDir, "names.jsonl");
+  const abortFile = (0, import_node_path3.join)(tmpDir, "abort.txt");
+  (0, import_node_fs.writeFileSync)(nameRegistryFile, "", "utf8");
+  if ((0, import_node_fs.existsSync)(abortFile)) {
+    (0, import_node_fs.unlinkSync)(abortFile);
+  }
+  process.env.AXIOM_NAME_REGISTRY_FILE = nameRegistryFile;
+  process.env.AXIOM_ABORT_FILE = abortFile;
   if (opts.list) {
     console.log(u.bgWhite(u.blackBright(" List mode ")));
   }
@@ -2135,6 +2152,7 @@ var runVitest = async (dir, opts) => {
     disableConsoleIntercept: true,
     testTimeout: opts.config?.eval?.timeoutMs || 6e4,
     globals: true,
+    runner: (0, import_node_path3.resolve)(__dirname, "evals", "custom-runner.js"),
     provide: {
       baseline: opts.baseline,
       debug: opts.debug,
@@ -2151,6 +2169,12 @@ var runVitest = async (dir, opts) => {
     process.exit(0);
   }
   await vi.start();
+  if ((0, import_node_fs.existsSync)(abortFile)) {
+    const message = (0, import_node_fs.readFileSync)(abortFile, "utf8");
+    console.error("\n" + message);
+    await vi.close();
+    process.exit(1);
+  }
   const dispose = (0, import_node.registerConsoleShortcuts)(vi, process.stdin, process.stdout);
   if (!vi.shouldKeepServer()) {
     dispose();
@@ -2162,7 +2186,7 @@ var runVitest = async (dir, opts) => {
 };
 
 // src/cli/commands/eval.command.ts
-var import_node_fs = require("fs");
+var import_node_fs2 = require("fs");
 
 // src/context.ts
 function overrideFlags(partial) {
@@ -2228,7 +2252,7 @@ var loadEvalCommand = (program2, flagOverrides = {}) => {
           include = [target];
         } else {
           try {
-            const stat = (0, import_node_fs.lstatSync)(target);
+            const stat = (0, import_node_fs2.lstatSync)(target);
             if (stat.isDirectory()) {
               include = config?.eval?.include || [];
             } else {
@@ -2278,8 +2302,8 @@ var loadEvalCommand = (program2, flagOverrides = {}) => {
 
 // src/cli/utils/parse-flag-overrides.ts
 var import_zod5 = require("zod");
-var import_node_fs2 = require("fs");
-var import_node_path4 = require("path");
+var import_node_fs3 = require("fs");
+var import_node_path5 = require("path");
 var FLAG_RE = /^--flag\.([^=]+)(?:=(.*))?$/;
 var CONFIG_RE = /^--flags-config(?:=(.*))?$/;
 function ensureNoSpaceSeparatedSyntax(flagName, value, nextToken, flagType) {
@@ -2309,9 +2333,9 @@ function coerceValue(raw) {
   }
 }
 function loadConfigFile(path4) {
-  const abs = (0, import_node_path4.resolve)(process.cwd(), path4);
+  const abs = (0, import_node_path5.resolve)(process.cwd(), path4);
   try {
-    const contents = (0, import_node_fs2.readFileSync)(abs, "utf8");
+    const contents = (0, import_node_fs3.readFileSync)(abs, "utf8");
     const parsed = JSON.parse(contents);
     if (typeof parsed !== "object" || Array.isArray(parsed) || parsed === null) {
       console.error(
@@ -2384,7 +2408,7 @@ var import_commander4 = require("commander");
 var loadVersionCommand = (program2) => {
   return program2.addCommand(
     new import_commander4.Command("version").description("cli version").action(() => {
-      console.log("0.25.0");
+      console.log("0.26.0");
     })
   );
 };
@@ -2394,7 +2418,7 @@ var { loadEnvConfig } = import_env.default;
 loadEnvConfig(process.cwd());
 var { cleanedArgv, overrides } = extractOverrides(process.argv.slice(2));
 var program = new import_commander5.Command();
-program.name("axiom").description("Axiom's CLI to manage your objects and run evals").version("0.25.0");
+program.name("axiom").description("Axiom's CLI to manage your objects and run evals").version("0.26.0");
 loadPushCommand(program);
 loadPullCommand(program);
 loadEvalCommand(program, overrides);