axiom 0.22.1 → 0.23.0

package/README.md

@@ -10,6 +10,8 @@ Axiom AI SDK provides
 npm install axiom
 ```
 
+Evals require Node 22.20 or higher.
+
 ## Model Wrapping
 
 ```ts

package/dist/bin.cjs

@@ -478,21 +478,6 @@ var u = p(r);
 // src/evals/run-vitest.ts
 var import_node = require("vitest/node");
 
-// src/evals/context/global-flags.ts
-var GLOBAL_OVERRIDES_SYMBOL = Symbol.for("axiom.global_flag_overrides");
-function getRoot() {
-  return globalThis[GLOBAL_OVERRIDES_SYMBOL] ?? {};
-}
-function setRoot(val) {
-  globalThis[GLOBAL_OVERRIDES_SYMBOL] = val;
-}
-function setGlobalFlagOverrides(overrides2) {
-  setRoot(overrides2);
-}
-function getGlobalFlagOverrides() {
-  return getRoot();
-}
-
 // src/evals/context/storage.ts
 var import_api9 = require("@opentelemetry/api");
 
@@ -563,6 +548,15 @@ function createAsyncHook(_name) {
   };
 }
 
+// src/evals/context/global-flags.ts
+var GLOBAL_OVERRIDES_SYMBOL = Symbol.for("axiom.global_flag_overrides");
+function setRoot(val) {
+  globalThis[GLOBAL_OVERRIDES_SYMBOL] = val;
+}
+function setGlobalFlagOverrides(overrides2) {
+  setRoot(overrides2);
+}
+
 // src/validate-flags.ts
 var import_zod3 = require("zod");
 
@@ -610,7 +604,7 @@ var import_api4 = require("@opentelemetry/api");
 // package.json
 var package_default = {
   name: "axiom",
-  version: "0.22.1",
+  version: "0.23.0",
   type: "module",
   author: "Axiom, Inc.",
   contributors: [
@@ -764,9 +758,6 @@ var import_api7 = require("@opentelemetry/api");
 
 // src/evals/context/storage.ts
 var CONFIG_SCOPE_SYMBOL = Symbol.for("axiom.eval.configScope");
-function getGlobalConfigScope() {
-  return globalThis[CONFIG_SCOPE_SYMBOL];
-}
 var AXIOM_CONFIG_SYMBOL = Symbol.for("axiom.eval.config");
 function getAxiomConfig() {
   return globalThis[AXIOM_CONFIG_SYMBOL];
@@ -814,15 +805,20 @@ function withEvalContext(options = {}, fn) {
     fn
   );
 }
-function getConfigScope() {
-  const current = EVAL_CONTEXT.get();
-  return current?.configScope ?? getGlobalConfigScope();
-}
 
 // src/config/resolver.ts
+var buildConsoleUrl = (urlString) => {
+  const url = new URL(urlString);
+  return `${url.protocol}//app.${url.host.split("api.").at(-1)}`;
+};
 function resolveAxiomConnection(config) {
+  let consoleEndpointUrl = buildConsoleUrl(config.eval.url);
+  if ("__overrideEndpointUrl" in config.eval) {
+    consoleEndpointUrl = config.eval.__overrideEndpointUrl;
+  }
   return {
     url: config.eval.url,
+    consoleEndpointUrl,
     token: config.eval.token,
     dataset: config.eval.dataset
   };
@@ -1157,8 +1153,10 @@ function printTestCaseSuccessOrFailed(testMeta, ok) {
 function printTestCaseScores(testMeta, baseline) {
   const index = testMeta.case.index;
   Object.keys(testMeta.case.scores).forEach((k) => {
-    const v = testMeta.case.scores[k].score ? testMeta.case.scores[k].score : 0;
-    const scoreValue = Number(v * 100).toFixed(2) + "%";
+    const scoreData = testMeta.case.scores[k];
+    const hasError = scoreData.metadata?.error;
+    const v = scoreData.score ? scoreData.score : 0;
+    const scoreValue = hasError ? u.dim("N/A") : Number(v * 100).toFixed(2) + "%";
     if (baseline?.cases[index]?.scores[k]) {
       const baselineScoreValue = baseline.cases[index].scores[k].value;
       const diff = v - baselineScoreValue;
@@ -1169,11 +1167,16 @@ function printTestCaseScores(testMeta, baseline) {
         k,
         u.magentaBright(blScoreText),
         "->",
-        u.blueBright(scoreValue),
-        diff > 0 ? u.green("+" + diffText) : diff < 0 ? u.red(diffText) : diffText
+        hasError ? scoreValue : u.blueBright(scoreValue),
+        hasError ? u.dim("(scorer not run)") : diff > 0 ? u.green("+" + diffText) : diff < 0 ? u.red(diffText) : diffText
       );
     } else {
-      console.log("   ", k, u.blueBright(scoreValue));
+      console.log(
+        "   ",
+        k,
+        hasError ? scoreValue : u.blueBright(scoreValue),
+        hasError ? u.dim("(scorer not run)") : ""
+      );
     }
     return [k, scoreValue];
   });
@@ -1260,7 +1263,7 @@ function printGlobalFlagOverrides(overrides2, defaults) {
 function printSuiteBox({
   suite,
   scorerAverages,
-  calculateBaselineScorerAverage,
+  calculateBaselineScorerAverage: calculateBaselineScorerAverage2,
   flagDiff
 }) {
   const filename = suite.file.split("/").pop();
@@ -1269,29 +1272,33 @@ function printSuiteBox({
   console.log("\u251C\u2500");
   const scorerNames = Object.keys(scorerAverages);
   const maxNameLength = Math.max(...scorerNames.map((name) => name.length));
+  const allCasesErrored = (scorerName) => {
+    return suite.cases.every((caseData) => caseData.scores[scorerName]?.metadata?.error);
+  };
   for (const scorerName of scorerNames) {
     const avg = scorerAverages[scorerName];
     const paddedName = scorerName.padEnd(maxNameLength);
+    const hasAllErrors = allCasesErrored(scorerName);
     if (suite.baseline) {
-      const baselineAvg = calculateBaselineScorerAverage(suite.baseline, scorerName);
+      const baselineAvg = calculateBaselineScorerAverage2(suite.baseline, scorerName);
       if (baselineAvg !== null) {
-        const currentPercent = (avg * 100).toFixed(2) + "%";
+        const currentPercent = hasAllErrors ? u.dim("N/A") : (avg * 100).toFixed(2) + "%";
         const baselinePercent = (baselineAvg * 100).toFixed(2) + "%";
         const diff = avg - baselineAvg;
         const diffText = (diff >= 0 ? "+" : "") + (diff * 100).toFixed(2) + "%";
         const diffColor = diff > 0 ? u.green : diff < 0 ? u.red : u.dim;
         const paddedBaseline = baselinePercent.padStart(7);
-        const paddedCurrent = currentPercent.padStart(7);
-        const paddedDiff = diffText.padStart(8);
+        const paddedCurrent = hasAllErrors ? currentPercent : currentPercent.padStart(7);
+        const paddedDiff = hasAllErrors ? u.dim("(all cases failed)") : diffText.padStart(8);
         console.log(
-          `\u2502  ${paddedName}  ${u.blueBright(paddedBaseline)} \u2192 ${u.magentaBright(paddedCurrent)}  (${diffColor(paddedDiff)})`
+          `\u2502  ${paddedName}  ${u.blueBright(paddedBaseline)} \u2192 ${hasAllErrors ? paddedCurrent : u.magentaBright(paddedCurrent)}  (${hasAllErrors ? paddedDiff : diffColor(paddedDiff)})`
         );
       } else {
-        const currentPercent = (avg * 100).toFixed(2) + "%";
+        const currentPercent = hasAllErrors ? u.red("N/A (all cases failed)") : (avg * 100).toFixed(2) + "%";
         console.log(`\u2502   \u2022 ${paddedName}  ${currentPercent}`);
       }
     } else {
-      const currentPercent = (avg * 100).toFixed(2) + "%";
+      const currentPercent = hasAllErrors ? u.red("N/A (all cases failed)") : (avg * 100).toFixed(2) + "%";
       console.log(`\u2502   \u2022 ${paddedName}  ${currentPercent}`);
     }
   }
@@ -1332,12 +1339,62 @@ function printSuiteBox({
   }
   console.log("\u2514\u2500");
 }
-function printFinalReport({
-  suiteData,
-  calculateScorerAverages,
-  calculateBaselineScorerAverage,
-  calculateFlagDiff
-}) {
+function calculateScorerAverages(suite) {
+  const scorerTotals = {};
+  for (const caseData of suite.cases) {
+    for (const [scorerName, score] of Object.entries(caseData.scores)) {
+      if (!scorerTotals[scorerName]) {
+        scorerTotals[scorerName] = { sum: 0, count: 0 };
+      }
+      if (!score.metadata?.error) {
+        scorerTotals[scorerName].sum += score.score || 0;
+        scorerTotals[scorerName].count += 1;
+      }
+    }
+  }
+  const averages = {};
+  for (const [scorerName, totals] of Object.entries(scorerTotals)) {
+    averages[scorerName] = totals.count > 0 ? totals.sum / totals.count : 0;
+  }
+  return averages;
+}
+function calculateBaselineScorerAverage(baseline, scorerName) {
+  const scores = [];
+  for (const caseData of baseline.cases) {
+    if (caseData.scores[scorerName]) {
+      scores.push(caseData.scores[scorerName].value);
+    }
+  }
+  if (scores.length === 0) return null;
+  const sum = scores.reduce((acc, val) => acc + val, 0);
+  return sum / scores.length;
+}
+function calculateFlagDiff(suite) {
+  if (!suite.baseline || !suite.configFlags || suite.configFlags.length === 0) {
+    return [];
+  }
+  const diffs = [];
+  const currentConfig = suite.flagConfig || {};
+  const baselineConfig = suite.baseline.flagConfig || {};
+  const currentFlat = flattenObject(currentConfig);
+  const baselineFlat = flattenObject(baselineConfig);
+  const allKeys = /* @__PURE__ */ new Set([...Object.keys(currentFlat), ...Object.keys(baselineFlat)]);
+  for (const key of allKeys) {
+    const isInScope = suite.configFlags.some((pattern) => key.startsWith(pattern));
+    if (!isInScope) continue;
+    const currentValue = currentFlat[key];
+    const baselineValue = baselineFlat[key];
+    if (JSON.stringify(currentValue) !== JSON.stringify(baselineValue)) {
+      diffs.push({
+        flag: key,
+        current: currentValue ? JSON.stringify(currentValue) : void 0,
+        baseline: baselineValue ? JSON.stringify(baselineValue) : void 0
+      });
+    }
+  }
+  return diffs;
+}
+function printFinalReport({ suiteData }) {
   console.log("");
   console.log(u.bgBlue(u.white(" FINAL EVALUATION REPORT ")));
   console.log("");
@@ -1376,19 +1433,25 @@ var AxiomReporter = class {
     __publicField(this, "_endOfRunConfigEnd");
     __publicField(this, "_suiteData", []);
     __publicField(this, "_baselines", /* @__PURE__ */ new Map());
+    __publicField(this, "_printedFlagOverrides", false);
   }
   onTestRunStart() {
     this.start = performance.now();
     this.startTime = (/* @__PURE__ */ new Date()).getTime();
-    const overrides2 = getGlobalFlagOverrides();
-    const defaults = getConfigScope()?.getAllDefaultFlags?.() ?? {};
-    printGlobalFlagOverrides(overrides2, defaults);
   }
   async onTestSuiteReady(_testSuite) {
     const meta = _testSuite.meta();
     if (_testSuite.state() === "skipped") {
       return;
     }
+    if (!this._printedFlagOverrides) {
+      const defaultsFromConfigEnd = meta.evaluation.configEnd?.flags ?? {};
+      const overridesFromConfigEnd = meta.evaluation.configEnd?.overrides ?? {};
+      if (Object.keys(overridesFromConfigEnd).length > 0) {
+        printGlobalFlagOverrides(overridesFromConfigEnd, defaultsFromConfigEnd);
+      }
+      this._printedFlagOverrides = true;
+    }
     const baseline = meta.evaluation.baseline;
     if (baseline) {
       const config = getAxiomConfig();
@@ -1464,10 +1527,7 @@ var AxiomReporter = class {
       process.stdout.write("\x1B[2J\x1B[0f");
     }
     printFinalReport({
-      suiteData: this._suiteData,
-      calculateScorerAverages: this.calculateScorerAverages.bind(this),
-      calculateBaselineScorerAverage: this.calculateBaselineScorerAverage.bind(this),
-      calculateFlagDiff: this.calculateFlagDiff.bind(this)
+      suiteData: this._suiteData
     });
     const DEBUG = process.env.AXIOM_DEBUG === "true";
     if (DEBUG && this._endOfRunConfigEnd) {
@@ -1485,68 +1545,6 @@ var AxiomReporter = class {
     printRuntimeFlags(testMeta);
     printOutOfScopeFlags(testMeta);
   }
-  /**
-   * Calculate average scores per scorer for a suite
-   */
-  calculateScorerAverages(suite) {
-    const scorerTotals = {};
-    for (const caseData of suite.cases) {
-      for (const [scorerName, score] of Object.entries(caseData.scores)) {
-        if (!scorerTotals[scorerName]) {
-          scorerTotals[scorerName] = { sum: 0, count: 0 };
-        }
-        scorerTotals[scorerName].sum += score.score || 0;
-        scorerTotals[scorerName].count += 1;
-      }
-    }
-    const averages = {};
-    for (const [scorerName, totals] of Object.entries(scorerTotals)) {
-      averages[scorerName] = totals.count > 0 ? totals.sum / totals.count : 0;
-    }
-    return averages;
-  }
-  /**
-   * Calculate average score for a specific scorer from baseline data
-   */
-  calculateBaselineScorerAverage(baseline, scorerName) {
-    const scores = [];
-    for (const caseData of baseline.cases) {
-      if (caseData.scores[scorerName]) {
-        scores.push(caseData.scores[scorerName].value);
-      }
-    }
-    if (scores.length === 0) return null;
-    const sum = scores.reduce((acc, val) => acc + val, 0);
-    return sum / scores.length;
-  }
-  /**
-   * Calculate flag diff between current run and baseline (filtered by configFlags)
-   */
-  calculateFlagDiff(suite) {
-    if (!suite.baseline || !suite.configFlags || suite.configFlags.length === 0) {
-      return [];
-    }
-    const diffs = [];
-    const currentConfig = suite.flagConfig || {};
-    const baselineConfig = suite.baseline.flagConfig || {};
-    const currentFlat = flattenObject(currentConfig);
-    const baselineFlat = flattenObject(baselineConfig);
-    const allKeys = /* @__PURE__ */ new Set([...Object.keys(currentFlat), ...Object.keys(baselineFlat)]);
-    for (const key of allKeys) {
-      const isInScope = suite.configFlags.some((pattern) => key.startsWith(pattern));
-      if (!isInScope) continue;
-      const currentValue = currentFlat[key];
-      const baselineValue = baselineFlat[key];
-      if (JSON.stringify(currentValue) !== JSON.stringify(baselineValue)) {
-        diffs.push({
-          flag: key,
-          current: currentValue ? JSON.stringify(currentValue) : void 0,
-          baseline: baselineValue ? JSON.stringify(baselineValue) : void 0
-        });
-      }
-    }
-    return diffs;
-  }
   /**
    * End-of-suite config summary (console only)
    */
@@ -1561,11 +1559,111 @@ var import_sdk_trace_node = require("@opentelemetry/sdk-trace-node");
 var import_resources = require("@opentelemetry/resources");
 var import_exporter_trace_otlp_http = require("@opentelemetry/exporter-trace-otlp-http");
 var import_api10 = require("@opentelemetry/api");
+
+// src/config/loader.ts
+var import_c12 = require("c12");
+var import_defu = require("defu");
+
+// src/config/index.ts
+var DEFAULT_EVAL_INCLUDE = ["**/*.eval.{ts,js,mts,mjs,cts,cjs}"];
+function createPartialDefaults() {
+  return {
+    eval: {
+      url: process.env.AXIOM_URL || "https://api.axiom.co",
+      token: process.env.AXIOM_TOKEN,
+      dataset: process.env.AXIOM_DATASET,
+      instrumentation: null,
+      include: [...DEFAULT_EVAL_INCLUDE],
+      exclude: [],
+      timeoutMs: 6e4
+    }
+  };
+}
+function validateConfig(config) {
+  const errors = [];
+  if (!config.eval?.token) {
+    errors.push(
+      "eval.token is required (set in axiom.config.ts or AXIOM_TOKEN environment variable)"
+    );
+  }
+  if (!config.eval?.dataset) {
+    errors.push(
+      "eval.dataset is required (set in axiom.config.ts or AXIOM_DATASET environment variable)"
+    );
+  }
+  if (!config.eval?.url) {
+    console.log(
+      "eval.url was not specified. Defaulting to `https://api.axiom.co`. Please set it in axiom.config.ts or AXIOM_URL environment variable if you want to use a different endpoint."
+    );
+  }
+  const instrumentation = config.eval?.instrumentation;
+  if (instrumentation !== null && instrumentation !== void 0 && typeof instrumentation !== "function") {
+    errors.push(
+      "eval.instrumentation must be a function returning OTEL setup information or null."
+    );
+  }
+  if (errors.length > 0) {
+    throw new AxiomCLIError(`Invalid Axiom configuration:
+  - ${errors.join("\n  - ")}`);
+  }
+  return config;
+}
+
+// src/config/loader.ts
+function customMerger(target, source) {
+  const merged = (0, import_defu.defu)(source, target);
+  if (source?.eval && "include" in source.eval) {
+    merged.eval.include = source.eval.include;
+  }
+  return merged;
+}
+async function loadConfig(cwd = process.cwd()) {
+  try {
+    const defaults = createPartialDefaults();
+    const result = await (0, import_c12.loadConfig)({
+      name: "axiom",
+      cwd,
+      // Support common config file extensions
+      configFile: "axiom.config",
+      // Don't use defaultConfig - we'll merge manually to control array behavior
+      // Disable configs other than .ts/.js/.mts/.mjs/.cts/.cjs
+      rcFile: false,
+      globalRc: false,
+      packageJson: false,
+      giget: false
+    });
+    const mergedConfig = customMerger(defaults, result.config);
+    const validatedConfig = validateConfig(mergedConfig);
+    return {
+      config: validatedConfig
+    };
+  } catch (error) {
+    if (error instanceof AxiomCLIError) {
+      throw error;
+    }
+    throw new AxiomCLIError(`Failed to load config file: ${errorToString(error)}`);
+  }
+}
+
+// src/evals/instrument.ts
 var axiomProvider;
 var axiomTracer;
 var userProvider;
 var initializationPromise = null;
 var initialized = false;
+async function resolveInstrumentationHook(config) {
+  if (config.eval.instrumentation) {
+    return config.eval.instrumentation;
+  }
+  try {
+    const { config: loadedConfig } = await loadConfig(process.cwd());
+    return loadedConfig.eval.instrumentation ?? null;
+  } catch (error) {
+    throw new AxiomCLIError(
+      `Failed to reload instrumentation from config: ${errorToString(error)}`
+    );
+  }
+}
 async function runInstrumentationHook(hook, options) {
   try {
     return await hook(options);
@@ -1595,11 +1693,11 @@ function setupEvalProvider(connection) {
   axiomProvider = new import_sdk_trace_node.NodeTracerProvider({
     resource: (0, import_resources.resourceFromAttributes)({
       ["service.name"]: "axiom",
-      ["service.version"]: "0.22.1"
+      ["service.version"]: "0.23.0"
     }),
     spanProcessors: [processor]
   });
-  axiomTracer = axiomProvider.getTracer("axiom", "0.22.1");
+  axiomTracer = axiomProvider.getTracer("axiom", "0.23.0");
 }
 async function initInstrumentation(config) {
   if (initialized) {
@@ -1611,12 +1709,12 @@ async function initInstrumentation(config) {
   }
   initializationPromise = (async () => {
     if (!config.enabled) {
-      axiomTracer = import_api10.trace.getTracer("axiom", "0.22.1");
+      axiomTracer = import_api10.trace.getTracer("axiom", "0.23.0");
       initialized = true;
       return;
     }
     const connection = resolveAxiomConnection(config.config);
-    const hook = config.config.eval.instrumentation;
+    const hook = await resolveInstrumentationHook(config.config);
     let hookResult = void 0;
     if (hook) {
       config.config.eval.instrumentation = hook;
@@ -1769,91 +1867,6 @@ function isGlob(str) {
   return /[*?[\]{}!]/.test(str);
 }
 
-// src/config/loader.ts
-var import_c12 = require("c12");
-var import_defu = require("defu");
-
-// src/config/index.ts
-var DEFAULT_EVAL_INCLUDE = ["**/*.eval.{ts,js,mts,mjs,cts,cjs}"];
-function createPartialDefaults() {
-  return {
-    eval: {
-      url: process.env.AXIOM_URL || "https://api.axiom.co",
-      token: process.env.AXIOM_TOKEN,
-      dataset: process.env.AXIOM_DATASET,
-      instrumentation: null,
-      include: [...DEFAULT_EVAL_INCLUDE],
-      exclude: [],
-      timeoutMs: 6e4
-    }
-  };
-}
-function validateConfig(config) {
-  const errors = [];
-  if (!config.eval?.token) {
-    errors.push(
-      "eval.token is required (set in axiom.config.ts or AXIOM_TOKEN environment variable)"
-    );
-  }
-  if (!config.eval?.dataset) {
-    errors.push(
-      "eval.dataset is required (set in axiom.config.ts or AXIOM_DATASET environment variable)"
-    );
-  }
-  if (!config.eval?.url) {
-    console.log(
-      "eval.url was not specified. Defaulting to `https://api.axiom.co`. Please set it in axiom.config.ts or AXIOM_URL environment variable if you want to use a different endpoint."
-    );
-  }
-  const instrumentation = config.eval?.instrumentation;
-  if (instrumentation !== null && instrumentation !== void 0 && typeof instrumentation !== "function") {
-    errors.push(
-      "eval.instrumentation must be a function returning OTEL setup information or null."
-    );
-  }
-  if (errors.length > 0) {
-    throw new AxiomCLIError(`Invalid Axiom configuration:
-  - ${errors.join("\n  - ")}`);
-  }
-  return config;
-}
-
-// src/config/loader.ts
-function customMerger(target, source) {
-  const merged = (0, import_defu.defu)(source, target);
-  if (source?.eval && "include" in source.eval) {
-    merged.eval.include = source.eval.include;
-  }
-  return merged;
-}
-async function loadConfig(cwd = process.cwd()) {
-  try {
-    const defaults = createPartialDefaults();
-    const result = await (0, import_c12.loadConfig)({
-      name: "axiom",
-      cwd,
-      // Support common config file extensions
-      configFile: "axiom.config",
-      // Don't use defaultConfig - we'll merge manually to control array behavior
-      // Disable configs other than .ts/.js/.mts/.mjs/.cts/.cjs
-      rcFile: false,
-      globalRc: false,
-      packageJson: false,
-      giget: false
-    });
-    const mergedConfig = customMerger(defaults, result.config);
-    const validatedConfig = validateConfig(mergedConfig);
-    return {
-      config: validatedConfig
-    };
-  } catch (error) {
-    if (error instanceof AxiomCLIError) {
-      throw error;
-    }
-    throw new AxiomCLIError(`Failed to load config file: ${errorToString(error)}`);
-  }
-}
-
 // src/cli/commands/eval.command.ts
 var loadEvalCommand = (program2, flagOverrides = {}) => {
   return program2.addCommand(
@@ -2029,7 +2042,7 @@ var import_commander4 = require("commander");
 var loadVersionCommand = (program2) => {
   return program2.addCommand(
     new import_commander4.Command("version").description("cli version").action(() => {
-      console.log("0.22.1");
+      console.log("0.23.0");
     })
   );
 };
@@ -2039,7 +2052,7 @@ var { loadEnvConfig } = import_env.default;
 loadEnvConfig(process.cwd());
 var { cleanedArgv, overrides } = extractOverrides(process.argv.slice(2));
 var program = new import_commander5.Command();
-program.name("axiom").description("Axiom's CLI to manage your objects and run evals").version("0.22.1");
+program.name("axiom").description("Axiom's CLI to manage your objects and run evals").version("0.23.0");
 loadPushCommand(program);
 loadPullCommand(program);
 loadEvalCommand(program, overrides);