axiom 0.42.0 → 0.44.0

package/README.md

@@ -82,8 +82,9 @@ const result = await withSpan(
 For running scorers in production (without vitest dependency):
 
 ```ts
-import { withSpan, onlineEval } from 'axiom/ai';
+import { withSpan } from 'axiom/ai';
 import { Scorer } from 'axiom/ai/evals/scorers';
+import { onlineEval } from 'axiom/ai/evals/online';
 
 const formatScorer = Scorer('format-check', ({ output }: { output: string }) => {
   return output.length > 0;

package/dist/bin.cjs

@@ -553,7 +553,7 @@ var init_package = __esm({
   "package.json"() {
     package_default = {
       name: "axiom",
-      version: "0.42.0",
+      version: "0.44.0",
       type: "module",
       author: "Axiom, Inc.",
       contributors: [
@@ -563,7 +563,8 @@ var init_package = __esm({
       ],
       scripts: {
         dev: "tsup --watch",
-        build: "tsup && chmod +x dist/bin.js",
+        build: "tsup && chmod +x dist/bin.js && pnpm check:vitest-entrypoints",
+        "check:vitest-entrypoints": "node ./scripts/check-vitest-entrypoints.mjs",
         format: "prettier --write .",
         "format:check": "prettier --check .",
         lint: "eslint './**/*.{js,ts}'",
@@ -619,6 +620,16 @@ var init_package = __esm({
             default: "./dist/evals/scorers.cjs"
           }
         },
+        "./ai/evals/online": {
+          import: {
+            types: "./dist/evals/online.d.ts",
+            default: "./dist/evals/online.js"
+          },
+          require: {
+            types: "./dist/evals/online.d.cts",
+            default: "./dist/evals/online.cjs"
+          }
+        },
         "./ai/config": {
           import: {
             types: "./dist/config.d.ts",
@@ -2218,27 +2229,40 @@ function printFinalReport({
   logger("");
   logger(u.bgBlue(u.white(" FINAL EVALUATION REPORT ")));
   logger("");
-  for (const suite of suiteData) {
-    const scorerAverages = calculateScorerAverages(suite);
-    const flagDiff = calculateFlagDiff(suite);
-    printSuiteBox({ suite, scorerAverages, calculateBaselineScorerAverage, flagDiff, logger });
-    logger("");
-  }
   const runId = suiteData[0]?.runId;
   const orgId = suiteData[0]?.orgId;
   const anyRegistered = registrationStatus.some((s2) => s2.registered);
   const anyFailed = registrationStatus.some((s2) => !s2.registered);
+  const allFailed = registrationStatus.length > 0 && registrationStatus.every((s2) => !s2.registered);
+  const hasAnyScores = suiteData.some(
+    (suite) => suite.cases.some((caseData) => Object.keys(caseData.scores ?? {}).length > 0)
+  );
+  const shouldPrintSuiteBoxes = isDebug || !allFailed || hasAnyScores;
+  if (shouldPrintSuiteBoxes) {
+    for (const suite of suiteData) {
+      const scorerAverages = calculateScorerAverages(suite);
+      const flagDiff = calculateFlagDiff(suite);
+      printSuiteBox({
+        suite,
+        scorerAverages,
+        calculateBaselineScorerAverage,
+        flagDiff,
+        logger
+      });
+      logger("");
+    }
+  }
   if (anyRegistered && orgId && config?.consoleEndpointUrl) {
     if (suiteData.length === 1) {
       const suite = suiteData[0];
       const baselineParam = suite.baseline?.traceId ? `?baselineId=${suite.baseline.traceId}` : "";
       logger("View eval result:");
       logger(
-        `${config.consoleEndpointUrl}/${orgId}/ai-engineering/evaluations/${suite.name}/${suite.version}${baselineParam}`
+        `${config.consoleEndpointUrl}/${orgId}/ai/evaluations/${suite.name}/${suite.version}${baselineParam}`
       );
     } else {
       logger("View full report:");
-      logger(`${config.consoleEndpointUrl}/${orgId}/ai-engineering/evaluations?runId=${runId}`);
+      logger(`${config.consoleEndpointUrl}/${orgId}/ai/evaluations?runId=${runId}`);
     }
   } else if (isDebug) {
     logger(u.dim("Results not uploaded to Axiom (debug mode)"));
@@ -2490,11 +2514,11 @@ function setupEvalProvider(connection) {
   axiomProvider = new import_sdk_trace_node.NodeTracerProvider({
     resource: (0, import_resources.resourceFromAttributes)({
       ["service.name"]: "axiom",
-      ["service.version"]: "0.42.0"
+      ["service.version"]: "0.44.0"
     }),
     spanProcessors: [processor]
   });
-  axiomTracer = axiomProvider.getTracer("axiom", "0.42.0");
+  axiomTracer = axiomProvider.getTracer("axiom", "0.44.0");
 }
 async function initInstrumentation(config) {
   if (initialized) {
@@ -2506,7 +2530,7 @@ async function initInstrumentation(config) {
   }
   initializationPromise = (async () => {
     if (!config.enabled) {
-      axiomTracer = import_api10.trace.getTracer("axiom", "0.42.0");
+      axiomTracer = import_api10.trace.getTracer("axiom", "0.44.0");
       initialized = true;
       return;
     }
@@ -2768,7 +2792,7 @@ var init_eval_command = __esm({
             ".",
             "any *.eval.ts file in current directory"
           )
-        ).option("-w, --watch true", "keep server running and watch for changes", false).option("-t, --token <TOKEN>", "axiom token", getDefaultToken).option("-d, --dataset <DATASET>", "axiom dataset name", process.env.AXIOM_DATASET).option("-u, --url <AXIOM URL>", "axiom url", getDefaultUrl).option("-o, --org-id <ORG ID>", "axiom organization id", getDefaultOrgId).option("-b, --baseline <BASELINE ID>", "id of baseline evaluation to compare against").option("--debug", "run locally without any network operations", false).option("--list", "list evaluations and test cases without running them", false).addOption(new import_commander.Option("-c, --console-url <URL>", "console url override").hideHelp()).action(async (target, options) => {
+        ).option("-w, --watch true", "keep server running and watch for changes", false).option("-t, --token <TOKEN>", "axiom token", getDefaultToken).option("-d, --dataset <DATASET>", "axiom dataset name").option("-u, --url <AXIOM URL>", "axiom url", getDefaultUrl).option("-o, --org-id <ORG ID>", "axiom organization id", getDefaultOrgId).option("-b, --baseline <BASELINE ID>", "id of baseline evaluation to compare against").option("--debug", "run locally without any network operations", false).option("--list", "list evaluations and test cases without running them", false).addOption(new import_commander.Option("-c, --console-url <URL>", "console url override").hideHelp()).action(async (target, options) => {
           try {
             if (options.debug) {
               process.env.AXIOM_DEBUG = "true";
@@ -3249,7 +3273,7 @@ var import_commander2 = require("commander");
 var loadVersionCommand = (program2) => {
   return program2.addCommand(
     new import_commander2.Command("version").description("cli version").action(() => {
-      console.log("0.42.0");
+      console.log("0.44.0");
     })
   );
 };
@@ -3259,7 +3283,7 @@ var { loadEnvConfig } = import_env.default;
 loadEnvConfig(process.cwd());
 var { cleanedArgv, overrides } = extractOverrides(process.argv.slice(2));
 var program = new import_commander3.Command();
-program.name("axiom").description("Axiom's CLI to manage your objects and run evals").version("0.42.0");
+program.name("axiom").description("Axiom's CLI to manage your objects and run evals").version("0.44.0");
 program.hook("preAction", async (_, actionCommand) => {
   const commandName = actionCommand.name();
   const parentCommand = actionCommand.parent;