axiom 0.23.0 → 0.24.0

package/dist/bin.cjs

@@ -390,6 +390,7 @@ var loadPullCommand = (program2) => {
 
 // src/cli/commands/eval.command.ts
 var import_commander3 = require("commander");
+var import_nanoid = require("nanoid");
 
 // ../../node_modules/.pnpm/tinyrainbow@2.0.0/node_modules/tinyrainbow/dist/chunk-BVHSVHOK.js
 var f = {
@@ -604,7 +605,7 @@ var import_api4 = require("@opentelemetry/api");
 // package.json
 var package_default = {
   name: "axiom",
-  version: "0.23.0",
+  version: "0.24.0",
   type: "module",
   author: "Axiom, Inc.",
   contributors: [
@@ -826,30 +827,22 @@ function resolveAxiomConnection(config) {
 
 // src/evals/eval.service.ts
 var findEvaluationCases = async (evalId, config) => {
-  try {
-    const { dataset, url, token } = resolveAxiomConnection(config);
-    const apl = `['${dataset}'] | where trace_id == "${evalId}" | order by _time`;
-    const headers = new Headers({
-      Authorization: `Bearer ${token}`,
-      "Content-Type": "application/json"
-    });
-    const resp = await fetch(`${url}/v1/datasets/_apl?format=legacy`, {
-      headers,
-      method: "POST",
-      body: JSON.stringify({ apl })
-    });
-    const payload = await resp.json();
-    if (!resp.ok) {
-      console.log(payload);
-      return void 0;
-    }
-    if (payload.matches.length) {
-      return buildSpanTree(payload.matches);
-    }
-  } catch (err) {
-    console.log(err);
-    return void 0;
+  const { dataset, url, token } = resolveAxiomConnection(config);
+  const apl = `['${dataset}'] | where trace_id == "${evalId}" | order by _time`;
+  const headers = new Headers({
+    Authorization: `Bearer ${token}`,
+    "Content-Type": "application/json"
+  });
+  const resp = await fetch(`${url}/v1/datasets/_apl?format=legacy`, {
+    headers,
+    method: "POST",
+    body: JSON.stringify({ apl })
+  });
+  const payload = await resp.json();
+  if (!resp.ok) {
+    throw new Error(`Failed to query evaluation cases: ${payload.message || resp.statusText}`);
   }
+  return payload.matches.length ? buildSpanTree(payload.matches) : null;
 };
 var mapSpanToEval = (span) => {
   const flagConfigRaw = span.data.attributes["eval.config.flags"] ?? span.data.attributes.custom["eval.config.flags"];
@@ -1394,7 +1387,11 @@ function calculateFlagDiff(suite) {
   }
   return diffs;
 }
-function printFinalReport({ suiteData }) {
+function printFinalReport({
+  suiteData,
+  config,
+  registrationStatus
+}) {
   console.log("");
   console.log(u.bgBlue(u.white(" FINAL EVALUATION REPORT ")));
   console.log("");
@@ -1404,8 +1401,28 @@ function printFinalReport({ suiteData }) {
     printSuiteBox({ suite, scorerAverages, calculateBaselineScorerAverage, flagDiff });
     console.log("");
   }
-  console.log("View full report:");
-  console.log("https://app.axiom.co/evaluations/run/<run-id>");
+  const runId = suiteData[0]?.runId;
+  const orgId = suiteData[0]?.orgId;
+  const anyRegistered = registrationStatus.some((s2) => s2.registered);
+  const anyFailed = registrationStatus.some((s2) => !s2.registered);
+  if (anyRegistered && orgId && config?.consoleEndpointUrl) {
+    console.log("View full report:");
+    console.log(`${config.consoleEndpointUrl}/${orgId}/ai-engineering/evaluations?runId=${runId}`);
+  } else {
+    console.log("Results not available in Axiom UI (registration failed)");
+  }
+  if (anyFailed) {
+    console.log("");
+    for (const status of registrationStatus) {
+      if (!status.registered) {
+        console.log(u.yellow(`\u26A0\uFE0F  Warning: Failed to register "${status.name}" with Axiom`));
+        if (status.error) {
+          console.log(u.dim(`   Error: ${status.error}`));
+        }
+        console.log(u.dim(`   Results for this evaluation will not be available in the Axiom UI.`));
+      }
+    }
+  }
 }
 
 // src/cli/errors.ts
@@ -1434,14 +1451,19 @@ var AxiomReporter = class {
     __publicField(this, "_suiteData", []);
     __publicField(this, "_baselines", /* @__PURE__ */ new Map());
     __publicField(this, "_printedFlagOverrides", false);
+    __publicField(this, "_config");
   }
   onTestRunStart() {
     this.start = performance.now();
     this.startTime = (/* @__PURE__ */ new Date()).getTime();
+    const config = getAxiomConfig();
+    if (config) {
+      this._config = resolveAxiomConnection(config);
+    }
   }
   async onTestSuiteReady(_testSuite) {
     const meta = _testSuite.meta();
-    if (_testSuite.state() === "skipped") {
+    if (_testSuite.state() === "skipped" || !meta?.evaluation) {
       return;
     }
     if (!this._printedFlagOverrides) {
@@ -1473,7 +1495,7 @@ var AxiomReporter = class {
   }
   async onTestSuiteResult(testSuite) {
     const meta = testSuite.meta();
-    if (testSuite.state() === "skipped") {
+    if (testSuite.state() === "skipped" || !meta?.evaluation) {
       return;
     }
     const durationSeconds = Number((performance.now() - this.start) / 1e3).toFixed(2);
@@ -1509,8 +1531,11 @@ var AxiomReporter = class {
       baseline: suiteBaseline || null,
       configFlags: meta.evaluation.configFlags,
       flagConfig: meta.evaluation.flagConfig,
+      runId: meta.evaluation.runId,
+      orgId: meta.evaluation.orgId,
       cases,
-      outOfScopeFlags: meta.evaluation.outOfScopeFlags
+      outOfScopeFlags: meta.evaluation.outOfScopeFlags,
+      registrationStatus: meta.evaluation.registrationStatus
     });
     printEvalNameAndFileName(testSuite, meta);
     printBaselineNameAndVersion(meta);
@@ -1526,8 +1551,15 @@ var AxiomReporter = class {
     if (shouldClear) {
       process.stdout.write("\x1B[2J\x1B[0f");
     }
+    const registrationStatus = this._suiteData.map((suite) => ({
+      name: suite.name,
+      registered: suite.registrationStatus?.status === "success",
+      error: suite.registrationStatus?.status === "failed" ? suite.registrationStatus.error : void 0
+    }));
     printFinalReport({
-      suiteData: this._suiteData
+      suiteData: this._suiteData,
+      config: this._config,
+      registrationStatus
     });
     const DEBUG = process.env.AXIOM_DEBUG === "true";
     if (DEBUG && this._endOfRunConfigEnd) {
@@ -1693,11 +1725,11 @@ function setupEvalProvider(connection) {
   axiomProvider = new import_sdk_trace_node.NodeTracerProvider({
     resource: (0, import_resources.resourceFromAttributes)({
       ["service.name"]: "axiom",
-      ["service.version"]: "0.23.0"
+      ["service.version"]: "0.24.0"
     }),
     spanProcessors: [processor]
   });
-  axiomTracer = axiomProvider.getTracer("axiom", "0.23.0");
+  axiomTracer = axiomProvider.getTracer("axiom", "0.24.0");
 }
 async function initInstrumentation(config) {
   if (initialized) {
@@ -1709,7 +1741,7 @@ async function initInstrumentation(config) {
   }
   initializationPromise = (async () => {
     if (!config.enabled) {
-      axiomTracer = import_api10.trace.getTracer("axiom", "0.23.0");
+      axiomTracer = import_api10.trace.getTracer("axiom", "0.24.0");
       initialized = true;
       return;
     }
@@ -1809,7 +1841,8 @@ var runVitest = async (dir, opts) => {
       baseline: opts.baseline,
       debug: opts.debug,
       overrides: opts.overrides,
-      axiomConfig: providedConfig
+      axiomConfig: providedConfig,
+      runId: opts.runId
     }
   });
   await vi.start();
@@ -1868,6 +1901,7 @@ function isGlob(str) {
 }
 
 // src/cli/commands/eval.command.ts
+var createRunId = (0, import_nanoid.customAlphabet)("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ", 10);
 var loadEvalCommand = (program2, flagOverrides = {}) => {
   return program2.addCommand(
     new import_commander3.Command("eval").description("run evals locally").addArgument(
@@ -1909,6 +1943,7 @@ var loadEvalCommand = (program2, flagOverrides = {}) => {
           );
           console.log("");
         }
+        const runId = createRunId();
         await runEvalWithContext(flagOverrides, async () => {
           return runVitest(".", {
             watch: options.watch,
@@ -1918,7 +1953,8 @@ var loadEvalCommand = (program2, flagOverrides = {}) => {
             testNamePattern,
             debug: options.debug,
             overrides: flagOverrides,
-            config
+            config,
+            runId
           });
         });
       } catch (error) {
@@ -2042,7 +2078,7 @@ var import_commander4 = require("commander");
 var loadVersionCommand = (program2) => {
   return program2.addCommand(
     new import_commander4.Command("version").description("cli version").action(() => {
-      console.log("0.23.0");
+      console.log("0.24.0");
     })
   );
 };
@@ -2052,7 +2088,7 @@ var { loadEnvConfig } = import_env.default;
 loadEnvConfig(process.cwd());
 var { cleanedArgv, overrides } = extractOverrides(process.argv.slice(2));
 var program = new import_commander5.Command();
-program.name("axiom").description("Axiom's CLI to manage your objects and run evals").version("0.23.0");
+program.name("axiom").description("Axiom's CLI to manage your objects and run evals").version("0.24.0");
 loadPushCommand(program);
 loadPullCommand(program);
 loadEvalCommand(program, overrides);