axiom 0.18.0 → 0.20.0

package/dist/bin.cjs

@@ -255,10 +255,10 @@ async function askConfirmation(message) {
     input: process.stdin,
     output: process.stdout
   });
-  return new Promise((resolve2) => {
+  return new Promise((resolve3) => {
     rl.question(`${message} (y/N): `, (answer) => {
       rl.close();
-      resolve2(answer.toLowerCase() === "y" || answer.toLowerCase() === "yes");
+      resolve3(answer.toLowerCase() === "y" || answer.toLowerCase() === "yes");
     });
   });
 }
@@ -349,8 +349,8 @@ var loadPullCommand = (program2) => {
   ).option("--version <version>", "The version to pull, default: latest", "latest").option("--output <path>", "Output file path (optional, defaults to <slug>.prompt.ts)").action(async (slug, options) => {
     try {
       console.log(`Pulling prompt: ${slug} (version: ${options.version})`);
-      const url2 = `${process.env.AXIOM_URL}/v1/prompts/${slug}`;
-      const response = await fetch(url2, {
+      const url = `${process.env.AXIOM_URL}/v1/prompts/${slug}`;
+      const response = await fetch(url, {
         method: "GET",
         headers: {
           Authorization: `Bearer ${process.env.AXIOM_TOKEN}`,
@@ -479,11 +479,16 @@ var r = process.env.FORCE_TTY !== void 0 || (0, import_tty.isatty)(1);
 var u = p(r);
 
 // src/evals/eval.service.ts
-var datasetName = process.env.AXIOM_DATASET ?? "";
-var url = process.env.AXIOM_URL ?? "https://api.axiom.co";
-var token = process.env.AXIOM_TOKEN;
+var getEnvVars = () => {
+  return {
+    datasetName: process.env.AXIOM_DATASET ?? "",
+    url: process.env.AXIOM_URL ?? "https://api.axiom.co",
+    token: process.env.AXIOM_TOKEN
+  };
+};
 var findEvaluationCases = async (evalId) => {
   try {
+    const { datasetName, url, token } = getEnvVars();
     const apl = `['${datasetName}'] | where trace_id == "${evalId}" | order by _time`;
     const headers = new Headers({
       Authorization: `Bearer ${token}`,
@@ -646,6 +651,9 @@ var AxiomReporter = class {
   }
   async onTestSuiteReady(_testSuite) {
     const meta = _testSuite.meta();
+    if (_testSuite.state() === "skipped") {
+      return;
+    }
     const baseline = meta.evaluation.baseline;
     if (baseline) {
       this.baseline = await findEvaluationCases(baseline.id);
@@ -673,16 +681,20 @@ var AxiomReporter = class {
   }
   onTestCaseReady(test) {
     const meta = test.meta();
+    if (!meta.case) return;
     console.log(u.blue(` \u2713 evaluating case ${meta.case.index}`));
   }
   onTestSuiteResult(testSuite) {
+    if (testSuite.state() === "skipped") {
+      return;
+    }
     const duration = Number((performance.now() - this.start) / 1e3).toFixed(2);
     console.log(" ");
     console.log(" ", u.dim("Cases"), testSuite.children.size);
     console.log(" ", u.dim("Start at"), new Date(this.startTime).toTimeString());
     console.log(" ", u.dim("Duration"), `${duration}s`);
     const meta = testSuite.meta();
-    const url2 = `https://app.axiom.co/evaluations/${meta.evaluation.name}/${meta.evaluation.id}`;
+    const url = `https://app.axiom.co/evaluations/${meta.evaluation.name}/${meta.evaluation.id}`;
     for (const test of testSuite.children) {
       if (test.type !== "test") return;
       this.printCaseResult(test);
@@ -690,7 +702,7 @@ var AxiomReporter = class {
     console.log("");
     console.log(
       " ",
-      `see results for ${meta.evaluation.name}-${meta.evaluation.version} at ${url2}`
+      `see results for ${meta.evaluation.name}-${meta.evaluation.version} at ${url}`
     );
     console.log(
       " ",
@@ -736,6 +748,20 @@ var AxiomReporter = class {
       }
       return [k, scoreValue];
     });
+    if (testMeta.case.outOfScopeFlags && testMeta.case.outOfScopeFlags.length > 0) {
+      const pickedFlagsText = testMeta.case.pickedFlags ? `(picked: ${testMeta.case.pickedFlags.map((f2) => `'${f2}'`).join(", ")})` : "(none)";
+      console.log("   ", u.yellow(`\u26A0 Out-of-scope flags: ${pickedFlagsText}`));
+      testMeta.case.outOfScopeFlags.forEach((flag) => {
+        const timeStr = new Date(flag.accessedAt).toLocaleTimeString();
+        console.log("     ", `${flag.flagPath} (at ${timeStr})`);
+        if (flag.stackTrace && flag.stackTrace.length > 0) {
+          flag.stackTrace.forEach((frame, i) => {
+            const prefix = i === flag.stackTrace.length - 1 ? " \u2514\u2500" : " \u251C\u2500";
+            console.log("     ", u.dim(`${prefix} ${frame}`));
+          });
+        }
+      });
+    }
   }
 };
 
@@ -751,7 +777,7 @@ var import_api2 = require("@opentelemetry/api");
 // package.json
 var package_default = {
   name: "axiom",
-  version: "0.18.0",
+  version: "0.20.0",
   type: "module",
   author: "Axiom, Inc.",
   contributors: [
@@ -823,14 +849,12 @@ var package_default = {
     "@sinclair/typebox": "^0.34.37",
     commander: "^14.0.0",
     "console-table-printer": "^2.14.6",
-    esbuild: "^0.25.8",
     handlebars: "^4.7.8",
-    nanoid: "^5.1.5",
-    vitest: "catalog:",
-    zod: "catalog:"
+    nanoid: "^5.1.5"
   },
   peerDependencies: {
-    "@opentelemetry/api": "^1.9.0"
+    "@opentelemetry/api": "^1.9.0",
+    zod: "^3.25.0 || ^4.0.0"
   },
   devDependencies: {
     "@ai-sdk/anthropicv1": "npm:@ai-sdk/anthropic@^1.2.12",
@@ -848,17 +872,19 @@ var package_default = {
     "@vitest/coverage-v8": "^3.2.4",
     aiv4: "npm:ai@^4.3.19",
     aiv5: "npm:ai@^5.0.0",
+    esbuild: "^0.25.8",
     eslint: "catalog:",
     prettier: "catalog:",
     tinyrainbow: "^2.0.0",
     tsup: "catalog:",
     typescript: "catalog:",
-    vitest: "catalog:"
+    vitest: "catalog:",
+    zod: "catalog:"
   },
   files: [
     "dist"
   ],
-  packageManager: "pnpm@10.11.1"
+  packageManager: "pnpm@10.16.1"
 };
 
 // src/otel/utils/redaction.ts
@@ -913,12 +939,12 @@ var processor = new import_sdk_trace_node.BatchSpanProcessor(exporter, {
 var provider = new import_sdk_trace_node.NodeTracerProvider({
   resource: (0, import_resources.resourceFromAttributes)({
     ["service.name"]: "axiom",
-    ["service.version"]: "0.18.0"
+    ["service.version"]: "0.20.0"
   }),
   spanProcessors: [processor]
 });
 provider.register();
-var tracer = import_api3.trace.getTracer("axiom", "0.18.0");
+var tracer = import_api3.trace.getTracer("axiom", "0.20.0");
 var flush = async () => {
   await provider.forceFlush();
 };
@@ -931,7 +957,8 @@ var runVitest = async (dir, opts) => {
     root: dir ? dir : process.cwd(),
     mode: "test",
     include: opts.include,
-    reporters: [new AxiomReporter()],
+    testNamePattern: opts.testNamePattern,
+    reporters: ["verbose", new AxiomReporter()],
     environment: "node",
     browser: void 0,
     watch: opts.watch,
@@ -958,30 +985,337 @@ var runVitest = async (dir, opts) => {
 
 // src/cli/commands/eval.command.ts
 var import_node_fs = require("fs");
-var loadEvalCommand = (program2) => {
+
+// src/evals/context/storage.ts
+var import_api10 = require("@opentelemetry/api");
+
+// src/evals/context/manager.ts
+var import_node_module = require("module");
+var import_meta = {};
+var CONTEXT_MANAGER_SYMBOL = Symbol.for("axiom.context_manager");
+function getGlobalContextManager() {
+  return globalThis[CONTEXT_MANAGER_SYMBOL];
+}
+function setGlobalContextManager(manager) {
+  globalThis[CONTEXT_MANAGER_SYMBOL] = manager;
+}
+var isNodeJS = typeof process !== "undefined" && !!process.versions?.node;
+function getContextManager() {
+  const existing = getGlobalContextManager();
+  if (existing) return existing;
+  let manager;
+  if (isNodeJS) {
+    try {
+      let AsyncLocalStorage;
+      const req = (0, import_node_module.createRequire)(import_meta.url);
+      try {
+        AsyncLocalStorage = req("node:async_hooks").AsyncLocalStorage;
+      } catch {
+        AsyncLocalStorage = req("async_hooks").AsyncLocalStorage;
+      }
+      manager = new AsyncLocalStorage();
+    } catch (error) {
+      console.warn("AsyncLocalStorage not available, using fallback context manager:", error);
+      manager = createFallbackManager();
+    }
+  } else {
+    console.warn("AsyncLocalStorage not available, using fallback context manager");
+    manager = createFallbackManager();
+  }
+  setGlobalContextManager(manager);
+  return manager;
+}
+function createFallbackManager() {
+  let currentContext = null;
+  return {
+    getStore: () => currentContext,
+    run: (value, fn) => {
+      const prev = currentContext;
+      currentContext = value;
+      try {
+        return fn();
+      } finally {
+        currentContext = prev;
+      }
+    }
+  };
+}
+function createAsyncHook(_name) {
+  return {
+    get() {
+      const manager = getContextManager();
+      if (manager.getStore) {
+        return manager.getStore();
+      }
+      return void 0;
+    },
+    run(value, fn) {
+      const manager = getContextManager();
+      return manager.run(value, fn);
+    }
+  };
+}
+
+// src/evals/context/global-flags.ts
+var GLOBAL_OVERRIDES_SYMBOL = Symbol.for("axiom.global_flag_overrides");
+function setRoot(val) {
+  globalThis[GLOBAL_OVERRIDES_SYMBOL] = val;
+}
+function setGlobalFlagOverrides(overrides2) {
+  setRoot(overrides2);
+}
+
+// src/validate-flags.ts
+var import_zod3 = require("zod");
+
+// src/cli/utils/format-zod-errors.ts
+var import_zod = require("zod");
+
+// src/util/dot-path.ts
+var import_zod2 = require("zod");
+
+// src/app-scope.ts
+var import_api9 = require("@opentelemetry/api");
+var import_zod4 = require("zod");
+
+// src/otel/utils/to-otel-attribute.ts
+var import_api4 = require("@opentelemetry/api");
+
+// src/otel/withSpan.ts
+var import_api7 = require("@opentelemetry/api");
+
+// src/otel/utils/wrapperUtils.ts
+var import_api6 = require("@opentelemetry/api");
+
+// src/otel/semconv/attributes.ts
+var import_semantic_conventions = require("@opentelemetry/semantic-conventions");
+var import_incubating = require("@opentelemetry/semantic-conventions/incubating");
+
+// src/otel/startActiveSpan.ts
+var import_api5 = require("@opentelemetry/api");
+
+// src/otel/middleware.ts
+var import_api8 = require("@opentelemetry/api");
+
+// src/evals/context/storage.ts
+var EVAL_CONTEXT = createAsyncHook("eval-context");
+function getEvalContext() {
+  const ctx = EVAL_CONTEXT.get();
+  if (!ctx) {
+    return {
+      flags: {},
+      facts: {},
+      pickedFlags: void 0,
+      outOfScopeFlags: void 0
+    };
+  }
+  return {
+    flags: ctx.flags,
+    facts: ctx.facts,
+    pickedFlags: ctx.pickedFlags,
+    outOfScopeFlags: ctx.outOfScopeFlags,
+    parent: ctx.parent,
+    overrides: ctx.overrides
+  };
+}
+function putOnSpan(kind, key, value) {
+  const span = import_api10.trace.getActiveSpan();
+  if (span?.isRecording()) {
+    span.setAttributes({ [`${kind}.${key}`]: value });
+  }
+}
+function withEvalContext(options = {}, fn) {
+  const { initialFlags = {}, pickedFlags = [] } = options;
+  return EVAL_CONTEXT.run(
+    { flags: { ...initialFlags }, facts: {}, pickedFlags, outOfScopeFlags: [] },
+    fn
+  );
+}
+
+// src/context.ts
+function overrideFlags(partial) {
+  const current = getEvalContext();
+  if (!current) {
+    if (process.env.NODE_ENV !== "test") {
+      console.warn("overrideFlags called outside of evaluation context");
+    }
+    return;
+  }
+  const overlayContext = {
+    ...current,
+    flags: { ...current.flags, ...partial },
+    // Merge for backwards compatibility
+    parent: current,
+    overrides: { ...partial }
+  };
+  const currentCtx = EVAL_CONTEXT.get();
+  if (currentCtx) {
+    Object.assign(currentCtx, overlayContext);
+  }
+  for (const [key, value] of Object.entries(partial)) {
+    putOnSpan("flag", key, value);
+  }
+}
+
+// src/cli/utils/eval-context-runner.ts
+async function runEvalWithContext(overrides2, runFn) {
+  setGlobalFlagOverrides(overrides2);
+  return withEvalContext({ initialFlags: overrides2 }, async () => {
+    if (Object.keys(overrides2).length > 0) {
+      overrideFlags(overrides2);
+    }
+    return runFn();
+  });
+}
+
+// src/cli/utils/glob-utils.ts
+function isGlob(str) {
+  return /[*?[\]{}!]/.test(str);
+}
+
+// src/cli/commands/eval.command.ts
+var loadEvalCommand = (program2, flagOverrides = {}) => {
   return program2.addCommand(
     new import_commander3.Command("eval").description("run evals locally").addArgument(
-      new import_commander3.Argument("[path]", "path of base directory or *.eval.ts file").default(
+      new import_commander3.Argument("[target]", "file, directory, glob pattern, or eval name").default(
         ".",
         "any *.eval.ts file in current directory"
       )
-    ).option("-w, --watch true", "keep server running and watch for changes", false).option("-t, --token <TOKEN>", "axiom token", process.env.AXIOM_TOKEN).option("-d, --dataset <DATASET>", "axiom dataset name", process.env.AXIOM_DATASET).option("-u, --url <AXIOM URL>", "axiom url", process.env.AXIOM_URL ?? "https://api.axiom.co").option("-b, --baseline <BASELINE ID>", "id of baseline evaluation to compare against").action(async (path3, options) => {
+    ).option("-w, --watch true", "keep server running and watch for changes", false).option("-t, --token <TOKEN>", "axiom token", process.env.AXIOM_TOKEN).option("-d, --dataset <DATASET>", "axiom dataset name", process.env.AXIOM_DATASET).option("-u, --url <AXIOM URL>", "axiom url", process.env.AXIOM_URL ?? "https://api.axiom.co").option("-b, --baseline <BASELINE ID>", "id of baseline evaluation to compare against").action(async (target, options) => {
       if (!options.token || !options.dataset) {
         throw new Error("AXIOM_TOKEN, and AXIOM_DATASET must be set");
       }
-      const f2 = (0, import_node_fs.lstatSync)(path3);
-      const isDir = f2.isDirectory();
-      const targetPath = isDir ? path3 : ".";
-      const includedfiles = isDir ? ["**/*.eval.ts"] : [path3];
-      await runVitest(targetPath, {
-        watch: options.watch,
-        baseline: options.baseline,
-        include: includedfiles
+      let targetPath = ".";
+      let include = ["**/*.eval.ts"];
+      let testNamePattern;
+      const isGlobPattern = isGlob(target);
+      if (isGlobPattern) {
+        include = [target];
+      } else {
+        try {
+          const stat = (0, import_node_fs.lstatSync)(target);
+          if (stat.isDirectory()) {
+            targetPath = target;
+            include = ["**/*.eval.ts"];
+          } else {
+            include = [target];
+          }
+        } catch {
+          testNamePattern = new RegExp(target, "i");
+        }
+      }
+      await runEvalWithContext(flagOverrides, async () => {
+        return runVitest(targetPath, {
+          watch: options.watch,
+          baseline: options.baseline,
+          include,
+          testNamePattern
+        });
       });
     })
   );
 };
 
+// src/cli/utils/parse-flag-overrides.ts
+var import_zod5 = require("zod");
+var import_node_fs2 = require("fs");
+var import_node_path3 = require("path");
+var FLAG_RE = /^--flag\.([^=]+)(?:=(.*))?$/;
+var CONFIG_RE = /^--flags-config(?:=(.*))?$/;
+function ensureNoSpaceSeparatedSyntax(flagName, value, nextToken, flagType) {
+  if (value === void 0 && nextToken !== void 0) {
+    if (flagType === "flag" && !nextToken.startsWith("-") && nextToken !== "true" && nextToken !== "false") {
+      console.error(`\u274C Invalid syntax: --flag.${flagName} ${nextToken}`);
+      console.error(`\u{1F4A1} Use: --flag.${flagName}=${nextToken}`);
+      process.exit(1);
+    } else if (flagType === "config" && !nextToken.startsWith("-")) {
+      console.error(`\u274C Invalid syntax: --flags-config ${nextToken}`);
+      console.error(`\u{1F4A1} Use: --flags-config=${nextToken}`);
+      process.exit(1);
+    }
+  }
+}
+function coerceValue(raw) {
+  if (raw === "true") return true;
+  if (raw === "false") return false;
+  const num = Number(raw);
+  if (!Number.isNaN(num) && raw.trim() === num.toString()) {
+    return num;
+  }
+  try {
+    return JSON.parse(raw);
+  } catch {
+    return raw;
+  }
+}
+function loadConfigFile(path3) {
+  const abs = (0, import_node_path3.resolve)(process.cwd(), path3);
+  try {
+    const contents = (0, import_node_fs2.readFileSync)(abs, "utf8");
+    const parsed = JSON.parse(contents);
+    if (typeof parsed !== "object" || Array.isArray(parsed) || parsed === null) {
+      console.error(
+        `\u274C Flags config must be a JSON object, got ${Array.isArray(parsed) ? "array" : typeof parsed}`
+      );
+      process.exit(1);
+    }
+    return parsed;
+  } catch (err) {
+    console.error(`\u274C Could not read or parse flags config "${path3}": ${err.message}`);
+    process.exit(1);
+  }
+}
+function extractOverrides(argv) {
+  const cleanedArgv2 = [];
+  const overrides2 = {};
+  let configPath = null;
+  let hasCliFlags = false;
+  let configPathCount = 0;
+  for (let i = 0; i < argv.length; i++) {
+    const token = argv[i];
+    const configMatch = token.match(CONFIG_RE);
+    const flagMatch = token.match(FLAG_RE);
+    if (configMatch) {
+      configPathCount++;
+      if (configPathCount > 1) {
+        console.error("\u274C Only one --flags-config can be supplied.");
+        process.exit(1);
+      }
+      const value = configMatch[1];
+      const nextToken = argv.length > i + 1 ? argv[i + 1] : void 0;
+      ensureNoSpaceSeparatedSyntax("flags-config", value, nextToken, "config");
+      if (!value) {
+        console.error("\u274C --flags-config requires a file path");
+        console.error("\u{1F4A1} Use: --flags-config=path/to/config.json");
+        process.exit(1);
+      }
+      configPath = value;
+    } else if (flagMatch) {
+      hasCliFlags = true;
+      const key = flagMatch[1];
+      const value = flagMatch[2];
+      const nextToken = argv.length > i + 1 ? argv[i + 1] : void 0;
+      ensureNoSpaceSeparatedSyntax(key, value, nextToken, "flag");
+      const finalValue = value === void 0 ? "true" : value;
+      overrides2[key] = coerceValue(finalValue);
+    } else {
+      cleanedArgv2.push(token);
+    }
+  }
+  if (configPath && hasCliFlags) {
+    console.error("\u274C Cannot use both --flags-config and --flag.* arguments together.");
+    console.error("Choose one approach:");
+    console.error("  \u2022 Config file: --flags-config=my-flags.json");
+    console.error("  \u2022 CLI flags: --flag.temperature=0.9 --flag.model=gpt-4o");
+    process.exit(1);
+  }
+  if (configPath) {
+    const configOverrides = loadConfigFile(configPath);
+    return { cleanedArgv: cleanedArgv2, overrides: configOverrides };
+  }
+  return { cleanedArgv: cleanedArgv2, overrides: overrides2 };
+}
+
 // src/bin.ts
 var import_env = __toESM(require("@next/env"), 1);
 
@@ -990,7 +1324,7 @@ var import_commander4 = require("commander");
 var loadVersionCommand = (program2) => {
   return program2.addCommand(
     new import_commander4.Command("version").description("cli version").action(() => {
-      console.log("0.18.0");
+      console.log("0.20.0");
     })
   );
 };
@@ -998,13 +1332,14 @@ var loadVersionCommand = (program2) => {
 // src/bin.ts
 var { loadEnvConfig } = import_env.default;
 loadEnvConfig(process.cwd());
+var { cleanedArgv, overrides } = extractOverrides(process.argv.slice(2));
 var program = new import_commander5.Command();
-program.name("axiom").description("Axiom's CLI to manage your objects and run evals").version("0.18.0");
+program.name("axiom").description("Axiom's CLI to manage your objects and run evals").version("0.20.0");
 loadPushCommand(program);
 loadPullCommand(program);
-loadEvalCommand(program);
+loadEvalCommand(program, overrides);
 loadVersionCommand(program);
-program.parse();
+program.parse(["node", "axiom", ...cleanedArgv]);
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   program