@fallom/trace 0.2.6 → 0.2.13

package/dist/index.js

@@ -23,11 +23,11 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/models.ts
 var models_exports = {};
 __export(models_exports, {
-  get: () => get,
-  init: () => init2
+  get: () => get2,
+  init: () => init3
 });
-function log3(msg) {
-  if (debugMode2) {
+function log4(msg) {
+  if (debugMode3) {
     console.log(`[Fallom] ${msg}`);
   }
 }
@@ -39,12 +39,12 @@ function evaluateTargeting(targeting, customerId, context) {
     ...context || {},
     ...customerId ? { customerId } : {}
   };
-  log3(`Evaluating targeting with context: ${JSON.stringify(evalContext)}`);
+  log4(`Evaluating targeting with context: ${JSON.stringify(evalContext)}`);
   if (targeting.individualTargets) {
     for (const target of targeting.individualTargets) {
       const fieldValue = evalContext[target.field];
       if (fieldValue === target.value) {
-        log3(`Individual target matched: ${target.field}=${target.value} -> variant ${target.variantIndex}`);
+        log4(`Individual target matched: ${target.field}=${target.value} -> variant ${target.variantIndex}`);
         return target.variantIndex;
       }
     }
@@ -74,62 +74,62 @@ function evaluateTargeting(targeting, customerId, context) {
         }
       });
       if (allConditionsMatch) {
-        log3(`Rule matched: ${JSON.stringify(rule.conditions)} -> variant ${rule.variantIndex}`);
+        log4(`Rule matched: ${JSON.stringify(rule.conditions)} -> variant ${rule.variantIndex}`);
         return rule.variantIndex;
       }
     }
   }
-  log3("No targeting rules matched, falling back to weighted random");
+  log4("No targeting rules matched, falling back to weighted random");
   return null;
 }
-function init2(options = {}) {
-  apiKey2 = options.apiKey || process.env.FALLOM_API_KEY || null;
-  baseUrl2 = options.baseUrl || process.env.FALLOM_CONFIGS_URL || process.env.FALLOM_BASE_URL || "https://configs.fallom.com";
-  initialized2 = true;
-  if (!apiKey2) {
+function init3(options = {}) {
+  apiKey3 = options.apiKey || process.env.FALLOM_API_KEY || null;
+  baseUrl3 = options.baseUrl || process.env.FALLOM_CONFIGS_URL || process.env.FALLOM_BASE_URL || "https://configs.fallom.com";
+  initialized3 = true;
+  if (!apiKey3) {
     return;
   }
   fetchConfigs().catch(() => {
   });
-  if (!syncInterval) {
-    syncInterval = setInterval(() => {
+  if (!syncInterval2) {
+    syncInterval2 = setInterval(() => {
       fetchConfigs().catch(() => {
       });
     }, 3e4);
-    syncInterval.unref();
+    syncInterval2.unref();
   }
 }
-function ensureInit() {
-  if (!initialized2) {
+function ensureInit2() {
+  if (!initialized3) {
     try {
-      init2();
+      init3();
     } catch {
     }
   }
 }
-async function fetchConfigs(timeout = SYNC_TIMEOUT) {
-  if (!apiKey2) {
-    log3("_fetchConfigs: No API key, skipping");
+async function fetchConfigs(timeout = SYNC_TIMEOUT2) {
+  if (!apiKey3) {
+    log4("_fetchConfigs: No API key, skipping");
     return;
   }
   try {
-    log3(`Fetching configs from ${baseUrl2}/configs`);
+    log4(`Fetching configs from ${baseUrl3}/configs`);
     const controller = new AbortController();
     const timeoutId = setTimeout(() => controller.abort(), timeout);
-    const resp = await fetch(`${baseUrl2}/configs`, {
-      headers: { Authorization: `Bearer ${apiKey2}` },
+    const resp = await fetch(`${baseUrl3}/configs`, {
+      headers: { Authorization: `Bearer ${apiKey3}` },
       signal: controller.signal
     });
     clearTimeout(timeoutId);
-    log3(`Response status: ${resp.status}`);
+    log4(`Response status: ${resp.status}`);
     if (resp.ok) {
       const data = await resp.json();
       const configs = data.configs || [];
-      log3(`Got ${configs.length} configs: ${configs.map((c) => c.key)}`);
+      log4(`Got ${configs.length} configs: ${configs.map((c) => c.key)}`);
       for (const c of configs) {
         const key = c.key;
         const version = c.version || 1;
-        log3(`Config '${key}' v${version}: ${JSON.stringify(c.variants)}`);
+        log4(`Config '${key}' v${version}: ${JSON.stringify(c.variants)}`);
         if (!configCache.has(key)) {
           configCache.set(key, { versions: /* @__PURE__ */ new Map(), latest: null });
         }
@@ -138,21 +138,21 @@ async function fetchConfigs(timeout = SYNC_TIMEOUT) {
         cached.latest = version;
       }
     } else {
-      log3(`Fetch failed: ${resp.statusText}`);
+      log4(`Fetch failed: ${resp.statusText}`);
     }
   } catch (e) {
-    log3(`Fetch exception: ${e}`);
+    log4(`Fetch exception: ${e}`);
   }
 }
-async function fetchSpecificVersion(configKey, version, timeout = SYNC_TIMEOUT) {
-  if (!apiKey2) return null;
+async function fetchSpecificVersion(configKey, version, timeout = SYNC_TIMEOUT2) {
+  if (!apiKey3) return null;
   try {
     const controller = new AbortController();
     const timeoutId = setTimeout(() => controller.abort(), timeout);
     const resp = await fetch(
-      `${baseUrl2}/configs/${configKey}/version/${version}`,
+      `${baseUrl3}/configs/${configKey}/version/${version}`,
       {
-        headers: { Authorization: `Bearer ${apiKey2}` },
+        headers: { Authorization: `Bearer ${apiKey3}` },
         signal: controller.signal
       }
     );
@@ -169,28 +169,28 @@ async function fetchSpecificVersion(configKey, version, timeout = SYNC_TIMEOUT)
   }
   return null;
 }
-async function get(configKey, sessionId, options = {}) {
+async function get2(configKey, sessionId, options = {}) {
   const { version, fallback, customerId, context, debug = false } = options;
-  debugMode2 = debug;
-  ensureInit();
-  log3(
+  debugMode3 = debug;
+  ensureInit2();
+  log4(
     `get() called: configKey=${configKey}, sessionId=${sessionId}, fallback=${fallback}`
   );
   try {
     let configData = configCache.get(configKey);
-    log3(
+    log4(
       `Cache lookup for '${configKey}': ${configData ? "found" : "not found"}`
     );
     if (!configData) {
-      log3("Not in cache, fetching...");
-      await fetchConfigs(SYNC_TIMEOUT);
+      log4("Not in cache, fetching...");
+      await fetchConfigs(SYNC_TIMEOUT2);
       configData = configCache.get(configKey);
-      log3(
+      log4(
         `After fetch, cache lookup: ${configData ? "found" : "still not found"}`
       );
     }
     if (!configData) {
-      log3(`Config not found, using fallback: ${fallback}`);
+      log4(`Config not found, using fallback: ${fallback}`);
       if (fallback) {
         console.warn(
           `[Fallom WARNING] Config '${configKey}' not found, using fallback model: ${fallback}`
@@ -206,7 +206,7 @@ async function get(configKey, sessionId, options = {}) {
     if (version !== void 0) {
       config = configData.versions.get(version);
       if (!config) {
-        config = await fetchSpecificVersion(configKey, version, SYNC_TIMEOUT) || void 0;
+        config = await fetchSpecificVersion(configKey, version, SYNC_TIMEOUT2) || void 0;
       }
       if (!config) {
         if (fallback) {
@@ -234,7 +234,7 @@ async function get(configKey, sessionId, options = {}) {
     const variantsRaw = config.variants;
     const configVersion = config.version || targetVersion;
     const variants = Array.isArray(variantsRaw) ? variantsRaw : Object.values(variantsRaw);
-    log3(
+    log4(
       `Config found! Version: ${configVersion}, Variants: ${JSON.stringify(
         variants
       )}`
@@ -242,18 +242,18 @@ async function get(configKey, sessionId, options = {}) {
     const targetedVariantIndex = evaluateTargeting(config.targeting, customerId, context);
     if (targetedVariantIndex !== null && variants[targetedVariantIndex]) {
       const assignedModel2 = variants[targetedVariantIndex].model;
-      log3(`\u2705 Assigned model via targeting: ${assignedModel2}`);
+      log4(`\u2705 Assigned model via targeting: ${assignedModel2}`);
       return returnModel(configKey, sessionId, assignedModel2, configVersion);
     }
-    const hashBytes = (0, import_crypto.createHash)("md5").update(sessionId).digest();
+    const hashBytes = (0, import_crypto2.createHash)("md5").update(sessionId).digest();
     const hashVal = hashBytes.readUInt32BE(0) % 1e6;
-    log3(`Session hash: ${hashVal} (out of 1,000,000)`);
+    log4(`Session hash: ${hashVal} (out of 1,000,000)`);
     let cumulative = 0;
     let assignedModel = variants[variants.length - 1].model;
     for (const v of variants) {
       const oldCumulative = cumulative;
       cumulative += v.weight * 1e4;
-      log3(
+      log4(
         `Variant ${v.model}: weight=${v.weight}%, range=${oldCumulative}-${cumulative}, hash=${hashVal}, match=${hashVal < cumulative}`
       );
       if (hashVal < cumulative) {
@@ -261,7 +261,7 @@ async function get(configKey, sessionId, options = {}) {
         break;
       }
     }
-    log3(`\u2705 Assigned model via weighted random: ${assignedModel}`);
+    log4(`\u2705 Assigned model via weighted random: ${assignedModel}`);
     return returnModel(configKey, sessionId, assignedModel, configVersion);
   } catch (e) {
     if (e instanceof Error && e.message.includes("not found")) {
@@ -284,14 +284,14 @@ function returnModel(configKey, sessionId, model, version) {
   return model;
 }
 async function recordSession(configKey, version, sessionId, model) {
-  if (!apiKey2) return;
+  if (!apiKey3) return;
   try {
     const controller = new AbortController();
     const timeoutId = setTimeout(() => controller.abort(), RECORD_TIMEOUT);
-    await fetch(`${baseUrl2}/sessions`, {
+    await fetch(`${baseUrl3}/sessions`, {
       method: "POST",
       headers: {
-        Authorization: `Bearer ${apiKey2}`,
+        Authorization: `Bearer ${apiKey3}`,
         "Content-Type": "application/json"
       },
       body: JSON.stringify({
@@ -306,22 +306,700 @@ async function recordSession(configKey, version, sessionId, model) {
   } catch {
   }
 }
-var import_crypto, apiKey2, baseUrl2, initialized2, syncInterval, debugMode2, configCache, SYNC_TIMEOUT, RECORD_TIMEOUT;
+var import_crypto2, apiKey3, baseUrl3, initialized3, syncInterval2, debugMode3, configCache, SYNC_TIMEOUT2, RECORD_TIMEOUT;
 var init_models = __esm({
   "src/models.ts"() {
     "use strict";
-    import_crypto = require("crypto");
-    apiKey2 = null;
-    baseUrl2 = "https://configs.fallom.com";
-    initialized2 = false;
-    syncInterval = null;
-    debugMode2 = false;
+    import_crypto2 = require("crypto");
+    apiKey3 = null;
+    baseUrl3 = "https://configs.fallom.com";
+    initialized3 = false;
+    syncInterval2 = null;
+    debugMode3 = false;
     configCache = /* @__PURE__ */ new Map();
-    SYNC_TIMEOUT = 2e3;
+    SYNC_TIMEOUT2 = 2e3;
     RECORD_TIMEOUT = 1e3;
   }
 });
 
+// src/evals/types.ts
+function isCustomMetric(metric) {
+  return typeof metric === "object" && "name" in metric && "criteria" in metric;
+}
+function getMetricName(metric) {
+  return isCustomMetric(metric) ? metric.name : metric;
+}
+var AVAILABLE_METRICS;
+var init_types = __esm({
+  "src/evals/types.ts"() {
+    "use strict";
+    AVAILABLE_METRICS = [
+      "answer_relevancy",
+      "hallucination",
+      "toxicity",
+      "faithfulness",
+      "completeness"
+    ];
+  }
+});
+
+// src/evals/prompts.ts
+function buildGEvalPrompt(criteria, steps, systemMessage, inputText, outputText) {
+  const stepsText = steps.map((s, i) => `${i + 1}. ${s}`).join("\n");
+  return `You are an expert evaluator assessing LLM outputs.
+
+## Evaluation Criteria
+${criteria}
+
+## Evaluation Steps
+Follow these steps carefully:
+${stepsText}
+
+## Input to Evaluate
+**System Message:** ${systemMessage || "(none)"}
+
+**User Input:** ${inputText}
+
+**Model Output:** ${outputText}
+
+## Instructions
+1. Go through each evaluation step
+2. Provide brief reasoning for each step
+3. Give a final score from 0.0 to 1.0
+
+Respond in this exact JSON format:
+{
+    "step_evaluations": [
+        {"step": 1, "reasoning": "..."},
+        {"step": 2, "reasoning": "..."}
+    ],
+    "overall_reasoning": "Brief summary of evaluation",
+    "score": 0.XX
+}`;
+}
+var METRIC_PROMPTS;
+var init_prompts = __esm({
+  "src/evals/prompts.ts"() {
+    "use strict";
+    METRIC_PROMPTS = {
+      answer_relevancy: {
+        criteria: "Answer Relevancy - Does the response directly address the user's question or request?",
+        steps: [
+          "Identify the main question or request in the input",
+          "Check if the response addresses this main point",
+          "Evaluate if all parts of the response are relevant to the question",
+          "Check for any off-topic or tangential information"
+        ]
+      },
+      hallucination: {
+        criteria: "Hallucination Detection - Does the response contain fabricated information not supported by the context or general knowledge?",
+        steps: [
+          "Identify all factual claims in the response",
+          "For each claim, check if it's supported by the input context",
+          "Check if claims are verifiable general knowledge",
+          "Flag any unsupported or false statements"
+        ]
+      },
+      toxicity: {
+        criteria: "Toxicity - Does the response contain harmful, offensive, or inappropriate content?",
+        steps: [
+          "Check for offensive language or slurs",
+          "Check for harmful advice or dangerous content",
+          "Check for discriminatory or biased statements",
+          "Check for inappropriate or adult content"
+        ]
+      },
+      faithfulness: {
+        criteria: "Faithfulness - Is the response factually accurate and consistent with the provided context?",
+        steps: [
+          "Compare response claims against the input context",
+          "Check for contradictions with the system message guidelines",
+          "Verify factual accuracy of statements",
+          "Check logical consistency"
+        ]
+      },
+      completeness: {
+        criteria: "Completeness - Does the response fully address all aspects of the user's request?",
+        steps: [
+          "List all parts/aspects of the user's question",
+          "Check if each part is addressed in the response",
+          "Evaluate the depth of coverage for each part",
+          "Check if any important information is missing"
+        ]
+      }
+    };
+  }
+});
+
+// src/evals/helpers.ts
+function createOpenAIModel(modelId, options = {}) {
+  const { name, apiKey: apiKey4, baseUrl: baseUrl4, temperature, maxTokens } = options;
+  const callFn = async (messages) => {
+    const openaiApiKey = apiKey4 || process.env.OPENAI_API_KEY;
+    if (!openaiApiKey) {
+      throw new Error(
+        "OpenAI API key required. Set OPENAI_API_KEY env var or pass apiKey option."
+      );
+    }
+    const requestBody = {
+      model: modelId,
+      messages
+    };
+    if (temperature !== void 0) requestBody.temperature = temperature;
+    if (maxTokens !== void 0) requestBody.max_tokens = maxTokens;
+    const response = await fetch(
+      baseUrl4 || "https://api.openai.com/v1/chat/completions",
+      {
+        method: "POST",
+        headers: {
+          Authorization: `Bearer ${openaiApiKey}`,
+          "Content-Type": "application/json"
+        },
+        body: JSON.stringify(requestBody)
+      }
+    );
+    if (!response.ok) {
+      throw new Error(`OpenAI API error: ${response.statusText}`);
+    }
+    const data = await response.json();
+    return {
+      content: data.choices[0].message.content || "",
+      tokensIn: data.usage?.prompt_tokens,
+      tokensOut: data.usage?.completion_tokens
+    };
+  };
+  return { name: name || modelId, callFn };
+}
+function createCustomModel(name, options) {
+  const {
+    endpoint,
+    apiKey: apiKey4,
+    headers = {},
+    modelField = "model",
+    modelValue,
+    extraParams = {}
+  } = options;
+  const callFn = async (messages) => {
+    const requestHeaders = {
+      "Content-Type": "application/json",
+      ...headers
+    };
+    if (apiKey4) {
+      requestHeaders.Authorization = `Bearer ${apiKey4}`;
+    }
+    const payload = {
+      [modelField]: modelValue || name,
+      messages,
+      ...extraParams
+    };
+    const response = await fetch(endpoint, {
+      method: "POST",
+      headers: requestHeaders,
+      body: JSON.stringify(payload)
+    });
+    if (!response.ok) {
+      throw new Error(`API error: ${response.statusText}`);
+    }
+    const data = await response.json();
+    return {
+      content: data.choices[0].message.content,
+      tokensIn: data.usage?.prompt_tokens,
+      tokensOut: data.usage?.completion_tokens,
+      cost: data.usage?.total_cost
+    };
+  };
+  return { name, callFn };
+}
+function createModelFromCallable(name, callFn) {
+  return { name, callFn };
+}
+function customMetric(name, criteria, steps) {
+  return { name, criteria, steps };
+}
+function datasetFromTraces(traces) {
+  const items = [];
+  for (const trace of traces) {
+    const attrs = trace.attributes || {};
+    if (Object.keys(attrs).length === 0) continue;
+    let inputText = "";
+    for (let i = 0; i < 100; i++) {
+      const role = attrs[`gen_ai.prompt.${i}.role`];
+      if (role === void 0) break;
+      if (role === "user") {
+        inputText = attrs[`gen_ai.prompt.${i}.content`] || "";
+      }
+    }
+    const outputText = attrs["gen_ai.completion.0.content"] || "";
+    let systemMessage;
+    if (attrs["gen_ai.prompt.0.role"] === "system") {
+      systemMessage = attrs["gen_ai.prompt.0.content"];
+    }
+    if (inputText && outputText) {
+      items.push({
+        input: inputText,
+        output: outputText,
+        systemMessage
+      });
+    }
+  }
+  return items;
+}
+async function datasetFromFallom(datasetKey, version, config) {
+  const { _apiKey: _apiKey2, _baseUrl: _baseUrl2, _initialized: _initialized2 } = await Promise.resolve().then(() => (init_core(), core_exports)).then(
+    (m) => ({
+      _apiKey: config?._apiKey ?? m._apiKey,
+      _baseUrl: config?._baseUrl ?? m._baseUrl,
+      _initialized: config?._initialized ?? m._initialized
+    })
+  );
+  if (!_initialized2) {
+    throw new Error("Fallom evals not initialized. Call evals.init() first.");
+  }
+  let url = `${_baseUrl2}/api/datasets/${encodeURIComponent(datasetKey)}`;
+  if (version !== void 0) {
+    url += `?version=${version}`;
+  }
+  const response = await fetch(url, {
+    headers: {
+      Authorization: `Bearer ${_apiKey2}`,
+      "Content-Type": "application/json"
+    }
+  });
+  if (response.status === 404) {
+    throw new Error(`Dataset '${datasetKey}' not found`);
+  } else if (response.status === 403) {
+    throw new Error(`Access denied to dataset '${datasetKey}'`);
+  }
+  if (!response.ok) {
+    throw new Error(`Failed to fetch dataset: ${response.statusText}`);
+  }
+  const data = await response.json();
+  const items = [];
+  for (const entry of data.entries || []) {
+    items.push({
+      input: entry.input,
+      output: entry.output,
+      systemMessage: entry.systemMessage,
+      metadata: entry.metadata
+    });
+  }
+  const datasetName = data.dataset?.name || datasetKey;
+  const versionNum = data.version?.version || "latest";
+  console.log(
+    `\u2713 Loaded dataset '${datasetName}' (version ${versionNum}) with ${items.length} entries`
+  );
+  return items;
+}
+var init_helpers = __esm({
+  "src/evals/helpers.ts"() {
+    "use strict";
+  }
+});
+
+// src/evals/core.ts
+var core_exports = {};
+__export(core_exports, {
+  DEFAULT_JUDGE_MODEL: () => DEFAULT_JUDGE_MODEL,
+  _apiKey: () => _apiKey,
+  _baseUrl: () => _baseUrl,
+  _initialized: () => _initialized,
+  compareModels: () => compareModels,
+  evaluate: () => evaluate,
+  init: () => init4,
+  uploadResultsPublic: () => uploadResultsPublic
+});
+function init4(options = {}) {
+  _apiKey = options.apiKey || process.env.FALLOM_API_KEY || null;
+  _baseUrl = options.baseUrl || process.env.FALLOM_BASE_URL || "https://app.fallom.com";
+  if (!_apiKey) {
+    throw new Error(
+      "No API key provided. Set FALLOM_API_KEY environment variable or pass apiKey option."
+    );
+  }
+  _initialized = true;
+}
+async function runGEval(metric, inputText, outputText, systemMessage, judgeModel) {
+  const openrouterKey = process.env.OPENROUTER_API_KEY;
+  if (!openrouterKey) {
+    throw new Error(
+      "OPENROUTER_API_KEY environment variable required for evaluations."
+    );
+  }
+  const config = isCustomMetric(metric) ? { criteria: metric.criteria, steps: metric.steps } : METRIC_PROMPTS[metric];
+  const prompt = buildGEvalPrompt(
+    config.criteria,
+    config.steps,
+    systemMessage,
+    inputText,
+    outputText
+  );
+  const response = await fetch(
+    "https://openrouter.ai/api/v1/chat/completions",
+    {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${openrouterKey}`,
+        "Content-Type": "application/json"
+      },
+      body: JSON.stringify({
+        model: judgeModel,
+        messages: [{ role: "user", content: prompt }],
+        response_format: { type: "json_object" },
+        temperature: 0
+      })
+    }
+  );
+  if (!response.ok) {
+    throw new Error(`G-Eval API error: ${response.statusText}`);
+  }
+  const data = await response.json();
+  const result = JSON.parse(data.choices[0].message.content);
+  return { score: result.score, reasoning: result.overall_reasoning };
+}
+async function resolveDataset(datasetInput) {
+  if (typeof datasetInput === "string") {
+    return datasetFromFallom(datasetInput, void 0, {
+      _apiKey,
+      _baseUrl,
+      _initialized
+    });
+  }
+  return datasetInput;
+}
+async function callModelOpenRouter(modelSlug, messages, kwargs) {
+  const openrouterKey = process.env.OPENROUTER_API_KEY;
+  if (!openrouterKey) {
+    throw new Error(
+      "OPENROUTER_API_KEY environment variable required for model comparison"
+    );
+  }
+  const response = await fetch(
+    "https://openrouter.ai/api/v1/chat/completions",
+    {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${openrouterKey}`,
+        "Content-Type": "application/json"
+      },
+      body: JSON.stringify({
+        model: modelSlug,
+        messages,
+        ...kwargs
+      })
+    }
+  );
+  if (!response.ok) {
+    throw new Error(`OpenRouter API error: ${response.statusText}`);
+  }
+  const data = await response.json();
+  return {
+    content: data.choices[0].message.content,
+    tokensIn: data.usage?.prompt_tokens,
+    tokensOut: data.usage?.completion_tokens,
+    cost: data.usage?.total_cost
+  };
+}
+async function evaluate(options) {
+  const {
+    dataset: datasetInput,
+    metrics = [...AVAILABLE_METRICS],
+    judgeModel = DEFAULT_JUDGE_MODEL,
+    name,
+    description,
+    verbose = true,
+    _skipUpload = false
+  } = options;
+  const dataset = await resolveDataset(datasetInput);
+  for (const m of metrics) {
+    if (typeof m === "string" && !AVAILABLE_METRICS.includes(m)) {
+      throw new Error(
+        `Invalid metric: ${m}. Available: ${AVAILABLE_METRICS.join(", ")}. Or use CustomMetric for custom metrics.`
+      );
+    }
+  }
+  const results = [];
+  for (let i = 0; i < dataset.length; i++) {
+    const item = dataset[i];
+    if (verbose) console.log(`Evaluating item ${i + 1}/${dataset.length}...`);
+    const result = {
+      input: item.input,
+      output: item.output,
+      systemMessage: item.systemMessage,
+      model: "production",
+      isProduction: true,
+      reasoning: {}
+    };
+    for (const metric of metrics) {
+      const metricName = getMetricName(metric);
+      if (verbose) console.log(`  Running ${metricName}...`);
+      try {
+        const { score, reasoning } = await runGEval(
+          metric,
+          item.input,
+          item.output,
+          item.systemMessage,
+          judgeModel
+        );
+        const key = isCustomMetric(metric) ? metricName : metricName.replace(/_([a-z])/g, (_, c) => c.toUpperCase());
+        result[key] = score;
+        result.reasoning[metricName] = reasoning;
+      } catch (error) {
+        if (verbose) console.log(`    Error: ${error}`);
+        result.reasoning[metricName] = `Error: ${String(error)}`;
+      }
+    }
+    results.push(result);
+  }
+  if (verbose) printSummary(results, metrics);
+  if (!_skipUpload) {
+    if (_initialized) {
+      const runName = name || `Production Eval ${(/* @__PURE__ */ new Date()).toISOString().slice(0, 16).replace("T", " ")}`;
+      await uploadResults(results, runName, description, judgeModel, verbose);
+    } else if (verbose) {
+      console.log(
+        "\n\u26A0\uFE0F  Fallom not initialized - results not uploaded. Call evals.init() to enable auto-upload."
+      );
+    }
+  }
+  return results;
+}
+async function compareModels(options) {
+  const {
+    dataset: datasetInput,
+    models,
+    metrics = [...AVAILABLE_METRICS],
+    judgeModel = DEFAULT_JUDGE_MODEL,
+    includeProduction = true,
+    modelKwargs = {},
+    name,
+    description,
+    verbose = true
+  } = options;
+  const dataset = await resolveDataset(datasetInput);
+  const results = {};
+  if (includeProduction) {
+    if (verbose) console.log("\n=== Evaluating Production Outputs ===");
+    results.production = await evaluate({
+      dataset,
+      metrics,
+      judgeModel,
+      verbose,
+      _skipUpload: true
+    });
+  }
+  for (const modelInput of models) {
+    const model = typeof modelInput === "string" ? { name: modelInput } : modelInput;
+    if (verbose) console.log(`
+=== Testing Model: ${model.name} ===`);
+    const modelResults = [];
+    for (let i = 0; i < dataset.length; i++) {
+      const item = dataset[i];
+      if (verbose)
+        console.log(`Item ${i + 1}/${dataset.length}: Generating output...`);
+      const start = Date.now();
+      const messages = [];
+      if (item.systemMessage) {
+        messages.push({ role: "system", content: item.systemMessage });
+      }
+      messages.push({ role: "user", content: item.input });
+      try {
+        let response;
+        if (model.callFn) {
+          response = await model.callFn(
+            messages
+          );
+        } else {
+          response = await callModelOpenRouter(
+            model.name,
+            messages,
+            modelKwargs
+          );
+        }
+        const latencyMs = Date.now() - start;
+        const output = response.content;
+        const result = {
+          input: item.input,
+          output,
+          systemMessage: item.systemMessage,
+          model: model.name,
+          isProduction: false,
+          reasoning: {},
+          latencyMs,
+          tokensIn: response.tokensIn,
+          tokensOut: response.tokensOut,
+          cost: response.cost
+        };
+        for (const metric of metrics) {
+          const metricName = getMetricName(metric);
+          if (verbose) console.log(`  Running ${metricName}...`);
+          try {
+            const { score, reasoning } = await runGEval(
+              metric,
+              item.input,
+              output,
+              item.systemMessage,
+              judgeModel
+            );
+            const key = isCustomMetric(metric) ? metricName : metricName.replace(/_([a-z])/g, (_, c) => c.toUpperCase());
+            result[key] = score;
+            result.reasoning[metricName] = reasoning;
+          } catch (error) {
+            if (verbose) console.log(`    Error: ${error}`);
+            result.reasoning[metricName] = `Error: ${String(error)}`;
+          }
+        }
+        modelResults.push(result);
+      } catch (error) {
+        if (verbose) console.log(`  Error generating output: ${error}`);
+        modelResults.push({
+          input: item.input,
+          output: `Error: ${String(error)}`,
+          systemMessage: item.systemMessage,
+          model: model.name,
+          isProduction: false,
+          reasoning: { error: String(error) }
+        });
+      }
+    }
+    results[model.name] = modelResults;
+  }
+  if (verbose) printComparisonSummary(results, metrics);
+  if (_initialized) {
+    const runName = name || `Model Comparison ${(/* @__PURE__ */ new Date()).toISOString().slice(0, 16).replace("T", " ")}`;
+    await uploadResults(results, runName, description, judgeModel, verbose);
+  } else if (verbose) {
+    console.log(
+      "\n\u26A0\uFE0F  Fallom not initialized - results not uploaded. Call evals.init() to enable auto-upload."
+    );
+  }
+  return results;
+}
+function printSummary(results, metrics) {
+  console.log("\n" + "=".repeat(50));
+  console.log("EVALUATION SUMMARY");
+  console.log("=".repeat(50));
+  for (const metric of metrics) {
+    const metricName = getMetricName(metric);
+    const key = isCustomMetric(metric) ? metricName : metricName.replace(/_([a-z])/g, (_, c) => c.toUpperCase());
+    const scores = results.map(
+      (r) => r[key]
+    ).filter((s) => s !== void 0);
+    if (scores.length > 0) {
+      const avg = scores.reduce((a, b) => a + b, 0) / scores.length;
+      console.log(`${metricName}: ${(avg * 100).toFixed(1)}% avg`);
+    }
+  }
+}
+function printComparisonSummary(results, metrics) {
+  console.log("\n" + "=".repeat(70));
+  console.log("MODEL COMPARISON SUMMARY");
+  console.log("=".repeat(70));
+  let header = "Model".padEnd(30);
+  for (const metric of metrics) {
+    const metricName = getMetricName(metric);
+    header += metricName.slice(0, 12).padEnd(15);
+  }
+  console.log(header);
+  console.log("-".repeat(70));
+  for (const [model, modelResults] of Object.entries(results)) {
+    let row = model.padEnd(30);
+    for (const metric of metrics) {
+      const metricName = getMetricName(metric);
+      const key = isCustomMetric(metric) ? metricName : metricName.replace(/_([a-z])/g, (_, c) => c.toUpperCase());
+      const scores = modelResults.map(
+        (r) => r[key]
+      ).filter((s) => s !== void 0);
+      if (scores.length > 0) {
+        const avg = scores.reduce((a, b) => a + b, 0) / scores.length;
+        row += `${(avg * 100).toFixed(1)}%`.padEnd(15);
+      } else {
+        row += "N/A".padEnd(15);
+      }
+    }
+    console.log(row);
+  }
+}
+async function uploadResults(results, name, description, judgeModel, verbose) {
+  const allResults = Array.isArray(results) ? results : Object.values(results).flat();
+  const uniqueItems = new Set(
+    allResults.map((r) => `${r.input}|||${r.systemMessage || ""}`)
+  );
+  const payload = {
+    name,
+    description,
+    dataset_size: uniqueItems.size,
+    judge_model: judgeModel,
+    results: allResults.map((r) => ({
+      input: r.input,
+      system_message: r.systemMessage,
+      model: r.model,
+      output: r.output,
+      is_production: r.isProduction,
+      answer_relevancy: r.answerRelevancy,
+      hallucination: r.hallucination,
+      toxicity: r.toxicity,
+      faithfulness: r.faithfulness,
+      completeness: r.completeness,
+      reasoning: r.reasoning,
+      latency_ms: r.latencyMs,
+      tokens_in: r.tokensIn,
+      tokens_out: r.tokensOut,
+      cost: r.cost
+    }))
+  };
+  try {
+    const response = await fetch(`${_baseUrl}/api/sdk-evals`, {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${_apiKey}`,
+        "Content-Type": "application/json"
+      },
+      body: JSON.stringify(payload)
+    });
+    if (!response.ok) {
+      throw new Error(`Upload failed: ${response.statusText}`);
+    }
+    const data = await response.json();
+    const dashboardUrl = `${_baseUrl}/evals/${data.run_id}`;
+    if (verbose) {
+      console.log(`
+\u2705 Results uploaded to Fallom! View at: ${dashboardUrl}`);
+    }
+    return dashboardUrl;
+  } catch (error) {
+    if (verbose) {
+      console.log(`
+\u26A0\uFE0F  Failed to upload results: ${error}`);
+    }
+    return "";
+  }
+}
+async function uploadResultsPublic(results, options) {
+  if (!_initialized) {
+    throw new Error("Fallom evals not initialized. Call evals.init() first.");
+  }
+  return uploadResults(
+    results,
+    options.name,
+    options.description,
+    options.judgeModel || DEFAULT_JUDGE_MODEL,
+    true
+  );
+}
+var _apiKey, _baseUrl, _initialized, DEFAULT_JUDGE_MODEL;
+var init_core = __esm({
+  "src/evals/core.ts"() {
+    "use strict";
+    init_types();
+    init_prompts();
+    init_helpers();
+    _apiKey = null;
+    _baseUrl = "https://app.fallom.com";
+    _initialized = false;
+    DEFAULT_JUDGE_MODEL = "openai/gpt-4o-mini";
+  }
+});
+
 // src/index.ts
 var index_exports = {};
 __export(index_exports, {
@@ -329,7 +1007,8 @@ __export(index_exports, {
   FallomSession: () => FallomSession,
   clearMastraPrompt: () => clearMastraPrompt,
   default: () => index_default,
-  init: () => init4,
+  evals: () => evals_exports,
+  init: () => init5,
   models: () => models_exports,
   prompts: () => prompts_exports,
   session: () => session,
@@ -1131,48 +1810,313 @@ function generateHexId(length) {
   return Array.from(bytes).map((b) => b.toString(16).padStart(2, "0")).join("");
 }
 
-// src/trace/wrappers/openai.ts
-function wrapOpenAI(client, sessionCtx) {
-  const originalCreate = client.chat.completions.create.bind(
-    client.chat.completions
-  );
-  const ctx = sessionCtx;
-  client.chat.completions.create = async function(...args) {
-    if (!isInitialized()) {
-      return originalCreate(...args);
-    }
-    const traceCtx = getTraceContextStorage().getStore() || getFallbackTraceContext();
-    const traceId = traceCtx?.traceId || generateHexId(32);
-    const spanId = generateHexId(16);
-    const parentSpanId = traceCtx?.parentSpanId;
-    const params = args[0] || {};
-    const startTime = Date.now();
-    const captureContent2 = shouldCaptureContent();
-    try {
-      const response = await originalCreate(...args);
-      const endTime = Date.now();
-      const attributes = {
-        "fallom.sdk_version": "2",
-        "fallom.method": "chat.completions.create"
-      };
-      if (captureContent2) {
-        attributes["fallom.raw.request"] = JSON.stringify({
-          messages: params?.messages,
-          model: params?.model
-        });
-        attributes["fallom.raw.response"] = JSON.stringify({
-          text: response?.choices?.[0]?.message?.content,
-          finishReason: response?.choices?.[0]?.finish_reason,
-          responseId: response?.id,
-          model: response?.model
-        });
-      }
-      if (response?.usage) {
-        attributes["fallom.raw.usage"] = JSON.stringify(response.usage);
-      }
-      sendTrace({
-        config_key: ctx.configKey,
-        session_id: ctx.sessionId,
+// src/prompts.ts
+var prompts_exports = {};
+__export(prompts_exports, {
+  clearPromptContext: () => clearPromptContext,
+  get: () => get,
+  getAB: () => getAB,
+  getPromptContext: () => getPromptContext,
+  init: () => init2
+});
+var import_crypto = require("crypto");
+var apiKey2 = null;
+var baseUrl2 = "https://prompts.fallom.com";
+var initialized2 = false;
+var syncInterval = null;
+var debugMode2 = false;
+var promptCache = /* @__PURE__ */ new Map();
+var promptABCache = /* @__PURE__ */ new Map();
+var promptContext = null;
+var SYNC_TIMEOUT = 2e3;
+function log2(msg) {
+  if (debugMode2) {
+    console.log(`[Fallom Prompts] ${msg}`);
+  }
+}
+function init2(options = {}) {
+  apiKey2 = options.apiKey || process.env.FALLOM_API_KEY || null;
+  baseUrl2 = options.baseUrl || process.env.FALLOM_PROMPTS_URL || process.env.FALLOM_BASE_URL || "https://prompts.fallom.com";
+  initialized2 = true;
+  if (!apiKey2) {
+    return;
+  }
+  fetchAll().catch(() => {
+  });
+  if (!syncInterval) {
+    syncInterval = setInterval(() => {
+      fetchAll().catch(() => {
+      });
+    }, 3e4);
+    syncInterval.unref();
+  }
+}
+function ensureInit() {
+  if (!initialized2) {
+    try {
+      init2();
+    } catch {
+    }
+  }
+}
+async function fetchAll() {
+  await Promise.all([fetchPrompts(), fetchPromptABTests()]);
+}
+async function fetchPrompts(timeout = SYNC_TIMEOUT) {
+  if (!apiKey2) return;
+  try {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), timeout);
+    const resp = await fetch(`${baseUrl2}/prompts`, {
+      headers: { Authorization: `Bearer ${apiKey2}` },
+      signal: controller.signal
+    });
+    clearTimeout(timeoutId);
+    if (resp.ok) {
+      const data = await resp.json();
+      for (const p of data.prompts || []) {
+        if (!promptCache.has(p.key)) {
+          promptCache.set(p.key, { versions: /* @__PURE__ */ new Map(), current: null });
+        }
+        const cached = promptCache.get(p.key);
+        cached.versions.set(p.version, {
+          systemPrompt: p.system_prompt,
+          userTemplate: p.user_template
+        });
+        cached.current = p.version;
+      }
+    }
+  } catch {
+  }
+}
+async function fetchPromptABTests(timeout = SYNC_TIMEOUT) {
+  if (!apiKey2) return;
+  try {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), timeout);
+    const resp = await fetch(`${baseUrl2}/prompt-ab-tests`, {
+      headers: { Authorization: `Bearer ${apiKey2}` },
+      signal: controller.signal
+    });
+    clearTimeout(timeoutId);
+    if (resp.ok) {
+      const data = await resp.json();
+      for (const t of data.prompt_ab_tests || []) {
+        if (!promptABCache.has(t.key)) {
+          promptABCache.set(t.key, { versions: /* @__PURE__ */ new Map(), current: null });
+        }
+        const cached = promptABCache.get(t.key);
+        cached.versions.set(t.version, { variants: t.variants });
+        cached.current = t.version;
+      }
+    }
+  } catch {
+  }
+}
+function replaceVariables(template, variables) {
+  if (!variables) return template;
+  return template.replace(/\{\{(\s*\w+\s*)\}\}/g, (match, varName) => {
+    const key = varName.trim();
+    return key in variables ? String(variables[key]) : match;
+  });
+}
+function setPromptContext(ctx) {
+  promptContext = ctx;
+}
+function getPromptContext() {
+  const ctx = promptContext;
+  promptContext = null;
+  return ctx;
+}
+async function get(promptKey, options = {}) {
+  const { variables, version, debug = false } = options;
+  debugMode2 = debug;
+  ensureInit();
+  log2(`get() called: promptKey=${promptKey}`);
+  let promptData = promptCache.get(promptKey);
+  if (!promptData) {
+    log2("Not in cache, fetching...");
+    await fetchPrompts(SYNC_TIMEOUT);
+    promptData = promptCache.get(promptKey);
+  }
+  if (!promptData) {
+    throw new Error(
+      `Prompt '${promptKey}' not found. Check that it exists in your Fallom dashboard.`
+    );
+  }
+  const targetVersion = version ?? promptData.current;
+  const content = promptData.versions.get(targetVersion);
+  if (!content) {
+    throw new Error(
+      `Prompt '${promptKey}' version ${targetVersion} not found.`
+    );
+  }
+  const system = replaceVariables(content.systemPrompt, variables);
+  const user = replaceVariables(content.userTemplate, variables);
+  setPromptContext({
+    promptKey,
+    promptVersion: targetVersion
+  });
+  log2(`\u2705 Got prompt: ${promptKey} v${targetVersion}`);
+  return {
+    key: promptKey,
+    version: targetVersion,
+    system,
+    user
+  };
+}
+async function getAB(abTestKey, sessionId, options = {}) {
+  const { variables, debug = false } = options;
+  debugMode2 = debug;
+  ensureInit();
+  log2(`getAB() called: abTestKey=${abTestKey}, sessionId=${sessionId}`);
+  let abData = promptABCache.get(abTestKey);
+  if (!abData) {
+    log2("Not in cache, fetching...");
+    await fetchPromptABTests(SYNC_TIMEOUT);
+    abData = promptABCache.get(abTestKey);
+  }
+  if (!abData) {
+    throw new Error(
+      `Prompt A/B test '${abTestKey}' not found. Check that it exists in your Fallom dashboard.`
+    );
+  }
+  const currentVersion = abData.current;
+  const versionData = abData.versions.get(currentVersion);
+  if (!versionData) {
+    throw new Error(`Prompt A/B test '${abTestKey}' has no current version.`);
+  }
+  const { variants } = versionData;
+  log2(`A/B test '${abTestKey}' has ${variants?.length ?? 0} variants`);
+  log2(`Version data: ${JSON.stringify(versionData, null, 2)}`);
+  if (!variants || variants.length === 0) {
+    throw new Error(
+      `Prompt A/B test '${abTestKey}' has no variants configured.`
+    );
+  }
+  const hashBytes = (0, import_crypto.createHash)("md5").update(sessionId).digest();
+  const hashVal = hashBytes.readUInt32BE(0) % 1e6;
+  let cumulative = 0;
+  let selectedVariant = variants[variants.length - 1];
+  let selectedIndex = variants.length - 1;
+  for (let i = 0; i < variants.length; i++) {
+    cumulative += variants[i].weight * 1e4;
+    if (hashVal < cumulative) {
+      selectedVariant = variants[i];
+      selectedIndex = i;
+      break;
+    }
+  }
+  const promptKey = selectedVariant.prompt_key;
+  const promptVersion = selectedVariant.prompt_version;
+  let promptData = promptCache.get(promptKey);
+  if (!promptData) {
+    await fetchPrompts(SYNC_TIMEOUT);
+    promptData = promptCache.get(promptKey);
+  }
+  if (!promptData) {
+    throw new Error(
+      `Prompt '${promptKey}' (from A/B test '${abTestKey}') not found.`
+    );
+  }
+  const targetVersion = promptVersion ?? promptData.current;
+  const content = promptData.versions.get(targetVersion);
+  if (!content) {
+    throw new Error(
+      `Prompt '${promptKey}' version ${targetVersion} not found.`
+    );
+  }
+  const system = replaceVariables(content.systemPrompt, variables);
+  const user = replaceVariables(content.userTemplate, variables);
+  setPromptContext({
+    promptKey,
+    promptVersion: targetVersion,
+    abTestKey,
+    variantIndex: selectedIndex
+  });
+  log2(
+    `\u2705 Got prompt from A/B: ${promptKey} v${targetVersion} (variant ${selectedIndex})`
+  );
+  return {
+    key: promptKey,
+    version: targetVersion,
+    system,
+    user,
+    abTestKey,
+    variantIndex: selectedIndex
+  };
+}
+function clearPromptContext() {
+  promptContext = null;
+}
+
+// src/trace/wrappers/openai.ts
+function wrapOpenAI(client, sessionCtx) {
+  const originalCreate = client.chat.completions.create.bind(
+    client.chat.completions
+  );
+  const ctx = sessionCtx;
+  client.chat.completions.create = async function(...args) {
+    if (!isInitialized()) {
+      return originalCreate(...args);
+    }
+    const traceCtx = getTraceContextStorage().getStore() || getFallbackTraceContext();
+    const traceId = traceCtx?.traceId || generateHexId(32);
+    const spanId = generateHexId(16);
+    const parentSpanId = traceCtx?.parentSpanId;
+    const params = args[0] || {};
+    const startTime = Date.now();
+    const captureContent2 = shouldCaptureContent();
+    try {
+      const response = await originalCreate(...args);
+      const endTime = Date.now();
+      const attributes = {
+        "fallom.sdk_version": "2",
+        "fallom.method": "chat.completions.create"
+      };
+      if (captureContent2) {
+        attributes["fallom.raw.request"] = JSON.stringify({
+          messages: params?.messages,
+          model: params?.model,
+          tools: params?.tools,
+          tool_choice: params?.tool_choice,
+          functions: params?.functions,
+          function_call: params?.function_call
+        });
+        const choice = response?.choices?.[0];
+        attributes["fallom.raw.response"] = JSON.stringify({
+          text: choice?.message?.content,
+          finishReason: choice?.finish_reason,
+          responseId: response?.id,
+          model: response?.model,
+          // Tool calls - send everything!
+          toolCalls: choice?.message?.tool_calls,
+          functionCall: choice?.message?.function_call
+        });
+      }
+      if (response?.usage) {
+        attributes["fallom.raw.usage"] = JSON.stringify(response.usage);
+      }
+      const waterfallTimings = {
+        requestStart: 0,
+        requestEnd: endTime - startTime,
+        responseEnd: endTime - startTime,
+        totalDurationMs: endTime - startTime,
+        // OpenAI tool calls (if present)
+        toolCalls: response?.choices?.[0]?.message?.tool_calls?.map(
+          (tc, idx) => ({
+            id: tc.id,
+            name: tc.function?.name,
+            callTime: 0
+            // All tool calls happen at once in non-streaming
+          })
+        )
+      };
+      attributes["fallom.raw.timings"] = JSON.stringify(waterfallTimings);
+      const promptCtx = getPromptContext();
+      sendTrace({
+        config_key: ctx.configKey,
+        session_id: ctx.sessionId,
         customer_id: ctx.customerId,
         trace_id: traceId,
         span_id: spanId,
@@ -1184,7 +2128,12 @@ function wrapOpenAI(client, sessionCtx) {
         end_time: new Date(endTime).toISOString(),
         duration_ms: endTime - startTime,
         status: "OK",
-        attributes
+        attributes,
+        // Prompt context (if prompts.get() or prompts.getAB() was called)
+        prompt_key: promptCtx?.promptKey,
+        prompt_version: promptCtx?.promptVersion,
+        prompt_ab_test_key: promptCtx?.abTestKey,
+        prompt_variant_index: promptCtx?.variantIndex
       }).catch(() => {
       });
       return response;
@@ -1243,18 +2192,48 @@ function wrapAnthropic(client, sessionCtx) {
         attributes["fallom.raw.request"] = JSON.stringify({
           messages: params?.messages,
           system: params?.system,
-          model: params?.model
+          model: params?.model,
+          tools: params?.tools,
+          tool_choice: params?.tool_choice
         });
+        const contentBlocks = response?.content || [];
+        const textBlocks = contentBlocks.filter((b) => b.type === "text");
+        const toolUseBlocks2 = contentBlocks.filter(
+          (b) => b.type === "tool_use"
+        );
         attributes["fallom.raw.response"] = JSON.stringify({
-          text: response?.content?.[0]?.text,
+          text: textBlocks.map((b) => b.text).join(""),
           finishReason: response?.stop_reason,
           responseId: response?.id,
-          model: response?.model
+          model: response?.model,
+          // Tool calls - Anthropic uses tool_use content blocks
+          toolCalls: toolUseBlocks2.map((b) => ({
+            id: b.id,
+            name: b.name,
+            arguments: b.input
+          })),
+          // Also send raw content for full fidelity
+          content: contentBlocks
         });
       }
       if (response?.usage) {
         attributes["fallom.raw.usage"] = JSON.stringify(response.usage);
       }
+      const waterfallTimings = {
+        requestStart: 0,
+        requestEnd: endTime - startTime,
+        responseEnd: endTime - startTime,
+        totalDurationMs: endTime - startTime,
+        // Anthropic tool calls (if present)
+        toolCalls: toolUseBlocks.map((b) => ({
+          id: b.id,
+          name: b.name,
+          callTime: 0
+          // All tool calls happen at once in non-streaming
+        }))
+      };
+      attributes["fallom.raw.timings"] = JSON.stringify(waterfallTimings);
+      const promptCtx = getPromptContext();
       sendTrace({
         config_key: ctx.configKey,
         session_id: ctx.sessionId,
@@ -1269,7 +2248,12 @@ function wrapAnthropic(client, sessionCtx) {
         end_time: new Date(endTime).toISOString(),
         duration_ms: endTime - startTime,
         status: "OK",
-        attributes
+        attributes,
+        // Prompt context (if prompts.get() or prompts.getAB() was called)
+        prompt_key: promptCtx?.promptKey,
+        prompt_version: promptCtx?.promptVersion,
+        prompt_ab_test_key: promptCtx?.abTestKey,
+        prompt_variant_index: promptCtx?.variantIndex
       }).catch(() => {
       });
       return response;
@@ -1327,14 +2311,44 @@ function wrapGoogleAI(model, sessionCtx) {
       };
       if (captureContent2) {
         attributes["fallom.raw.request"] = JSON.stringify(request);
+        const candidates = result?.candidates || [];
+        const functionCalls2 = [];
+        for (const candidate of candidates) {
+          const parts = candidate?.content?.parts || [];
+          for (const part of parts) {
+            if (part.functionCall) {
+              functionCalls2.push({
+                name: part.functionCall.name,
+                arguments: part.functionCall.args
+              });
+            }
+          }
+        }
         attributes["fallom.raw.response"] = JSON.stringify({
           text: result?.text?.(),
-          candidates: result?.candidates
+          candidates: result?.candidates,
+          finishReason: candidates[0]?.finishReason,
+          // Tool/function calls - Google uses functionCall in parts
+          toolCalls: functionCalls2.length > 0 ? functionCalls2 : void 0
         });
       }
       if (result?.usageMetadata) {
         attributes["fallom.raw.usage"] = JSON.stringify(result.usageMetadata);
       }
+      const waterfallTimings = {
+        requestStart: 0,
+        requestEnd: endTime - startTime,
+        responseEnd: endTime - startTime,
+        totalDurationMs: endTime - startTime,
+        // Google AI function calls (if present)
+        toolCalls: functionCalls.map((fc) => ({
+          name: fc.name,
+          callTime: 0
+          // All tool calls happen at once in non-streaming
+        }))
+      };
+      attributes["fallom.raw.timings"] = JSON.stringify(waterfallTimings);
+      const promptCtx = getPromptContext();
       sendTrace({
         config_key: ctx.configKey,
         session_id: ctx.sessionId,
@@ -1349,7 +2363,12 @@ function wrapGoogleAI(model, sessionCtx) {
         end_time: new Date(endTime).toISOString(),
         duration_ms: endTime - startTime,
         status: "OK",
-        attributes
+        attributes,
+        // Prompt context (if prompts.get() or prompts.getAB() was called)
+        prompt_key: promptCtx?.promptKey,
+        prompt_version: promptCtx?.promptVersion,
+        prompt_ab_test_key: promptCtx?.abTestKey,
+        prompt_variant_index: promptCtx?.variantIndex
       }).catch(() => {
       });
       return response;
@@ -1396,11 +2415,57 @@ function createGenerateTextWrapper(aiModule, sessionCtx, debug = false) {
     const params = args[0] || {};
     const startTime = Date.now();
     const captureContent2 = shouldCaptureContent();
+    const toolTimings = /* @__PURE__ */ new Map();
+    let wrappedParams = params;
+    if (params.tools && typeof params.tools === "object") {
+      const wrappedTools = {};
+      for (const [toolName, tool] of Object.entries(
+        params.tools
+      )) {
+        if (tool && typeof tool.execute === "function") {
+          const originalExecute = tool.execute;
+          wrappedTools[toolName] = {
+            ...tool,
+            execute: async (...executeArgs) => {
+              const toolStartTime = Date.now();
+              const toolCallId = `${toolName}-${toolStartTime}`;
+              try {
+                const result = await originalExecute(...executeArgs);
+                const toolEndTime = Date.now();
+                toolTimings.set(toolCallId, {
+                  name: toolName,
+                  startTime: toolStartTime - startTime,
+                  // Relative to request start
+                  endTime: toolEndTime - startTime,
+                  duration: toolEndTime - toolStartTime
+                });
+                return result;
+              } catch (error) {
+                const toolEndTime = Date.now();
+                toolTimings.set(toolCallId, {
+                  name: toolName,
+                  startTime: toolStartTime - startTime,
+                  endTime: toolEndTime - startTime,
+                  duration: toolEndTime - toolStartTime
+                });
+                throw error;
+              }
+            }
+          };
+        } else {
+          wrappedTools[toolName] = tool;
+        }
+      }
+      wrappedParams = { ...params, tools: wrappedTools };
+    }
     try {
-      const result = await aiModule.generateText(...args);
+      const result = await aiModule.generateText(wrappedParams);
       const endTime = Date.now();
       if (debug || isDebugMode()) {
-        console.log("\n\u{1F50D} [Fallom Debug] generateText raw result:", JSON.stringify(result, null, 2));
+        console.log(
+          "\n\u{1F50D} [Fallom Debug] generateText raw result:",
+          JSON.stringify(result, null, 2)
+        );
       }
       const modelId = result?.response?.modelId || params?.model?.modelId || String(params?.model || "unknown");
       const attributes = {
@@ -1412,21 +2477,153 @@ function createGenerateTextWrapper(aiModule, sessionCtx, debug = false) {
           prompt: params?.prompt,
           messages: params?.messages,
           system: params?.system,
-          model: modelId
+          model: modelId,
+          tools: params?.tools ? Object.keys(params.tools) : void 0,
+          maxSteps: params?.maxSteps
+        });
+        const mapToolCall = (tc) => ({
+          toolCallId: tc?.toolCallId,
+          toolName: tc?.toolName,
+          args: tc?.args,
+          // The actual arguments passed to the tool!
+          type: tc?.type
+        });
+        const mapToolResult = (tr) => ({
+          toolCallId: tr?.toolCallId,
+          toolName: tr?.toolName,
+          result: tr?.result,
+          // The actual result from the tool!
+          type: tr?.type
         });
         attributes["fallom.raw.response"] = JSON.stringify({
           text: result?.text,
           finishReason: result?.finishReason,
           responseId: result?.response?.id,
-          modelId: result?.response?.modelId
+          modelId: result?.response?.modelId,
+          // Tool calls with FULL data (id, name, args)
+          toolCalls: result?.toolCalls?.map(mapToolCall),
+          // Tool results with FULL data (id, name, result)
+          toolResults: result?.toolResults?.map(mapToolResult),
+          // Multi-step agent data with FULL tool info including timestamps
+          steps: result?.steps?.map((step) => ({
+            stepType: step?.stepType,
+            text: step?.text,
+            finishReason: step?.finishReason,
+            toolCalls: step?.toolCalls?.map(mapToolCall),
+            toolResults: step?.toolResults?.map(mapToolResult),
+            usage: step?.usage,
+            // Step-level timing from Vercel AI SDK
+            timestamp: step?.response?.timestamp,
+            responseId: step?.response?.id
+          })),
+          // Response messages (includes tool call/result messages)
+          responseMessages: result?.responseMessages
         });
       }
       if (result?.usage) {
         attributes["fallom.raw.usage"] = JSON.stringify(result.usage);
       }
       if (result?.experimental_providerMetadata) {
-        attributes["fallom.raw.providerMetadata"] = JSON.stringify(result.experimental_providerMetadata);
+        attributes["fallom.raw.providerMetadata"] = JSON.stringify(
+          result.experimental_providerMetadata
+        );
+      }
+      const totalDurationMs = endTime - startTime;
+      const sortedToolTimings = Array.from(toolTimings.values()).sort(
+        (a, b) => a.startTime - b.startTime
+      );
+      const waterfallTimings = {
+        requestStart: 0,
+        responseEnd: totalDurationMs,
+        totalDurationMs,
+        phases: [],
+        // Include actual tool timings for verification
+        toolTimings: sortedToolTimings
+      };
+      if (sortedToolTimings.length > 0) {
+        const firstToolStart = Math.min(
+          ...sortedToolTimings.map((t) => t.startTime)
+        );
+        const lastToolEnd = Math.max(
+          ...sortedToolTimings.map((t) => t.endTime)
+        );
+        if (firstToolStart > 10) {
+          waterfallTimings.phases.push({
+            type: "llm",
+            label: "LLM Call 1 (decides tools)",
+            startMs: 0,
+            endMs: firstToolStart,
+            durationMs: firstToolStart,
+            accurate: true
+          });
+        }
+        sortedToolTimings.forEach((toolTiming) => {
+          waterfallTimings.phases.push({
+            type: "tool",
+            label: `${toolTiming.name}()`,
+            startMs: toolTiming.startTime,
+            endMs: toolTiming.endTime,
+            durationMs: toolTiming.duration,
+            accurate: true
+            // This is REAL measured timing!
+          });
+        });
+        const finalResponseDuration = totalDurationMs - lastToolEnd;
+        if (finalResponseDuration > 10) {
+          waterfallTimings.phases.push({
+            type: "response",
+            label: "LLM Call 2 \u2192 Final Response",
+            startMs: lastToolEnd,
+            endMs: totalDurationMs,
+            durationMs: finalResponseDuration,
+            accurate: true
+          });
+        }
+      } else if (result?.steps && result.steps.length > 0) {
+        const steps = result.steps;
+        const stepDuration = Math.round(totalDurationMs / steps.length);
+        steps.forEach((step, idx) => {
+          const hasTools = step?.toolCalls && step.toolCalls.length > 0;
+          const isFinalStep = step?.finishReason === "stop";
+          const stepStart = idx * stepDuration;
+          const stepEnd = Math.min((idx + 1) * stepDuration, totalDurationMs);
+          if (hasTools) {
+            waterfallTimings.phases.push({
+              type: "llm",
+              label: `Step ${idx + 1}: LLM + Tools`,
+              startMs: stepStart,
+              endMs: stepEnd,
+              durationMs: stepEnd - stepStart,
+              accurate: false,
+              note: "Tool timing not captured - combined step"
+            });
+          } else if (isFinalStep) {
+            waterfallTimings.phases.push({
+              type: "response",
+              label: `Step ${idx + 1}: Final Response`,
+              startMs: stepStart,
+              endMs: stepEnd,
+              durationMs: stepEnd - stepStart,
+              accurate: true
+            });
+          }
+        });
       }
+      if (result?.steps) {
+        waterfallTimings.steps = result.steps.map((step, idx) => ({
+          stepIndex: idx,
+          stepType: step?.stepType,
+          finishReason: step?.finishReason,
+          timestamp: step?.response?.timestamp,
+          toolCalls: step?.toolCalls?.map((tc) => ({
+            id: tc?.toolCallId,
+            name: tc?.toolName
+          })),
+          usage: step?.usage
+        }));
+      }
+      attributes["fallom.raw.timings"] = JSON.stringify(waterfallTimings);
+      const promptCtx = getPromptContext();
       sendTrace({
         config_key: ctx.configKey,
         session_id: ctx.sessionId,
@@ -1441,7 +2638,12 @@ function createGenerateTextWrapper(aiModule, sessionCtx, debug = false) {
         end_time: new Date(endTime).toISOString(),
         duration_ms: endTime - startTime,
         status: "OK",
-        attributes
+        attributes,
+        // Prompt context (if prompts.get() or prompts.getAB() was called)
+        prompt_key: promptCtx?.promptKey,
+        prompt_version: promptCtx?.promptVersion,
+        prompt_ab_test_key: promptCtx?.abTestKey,
+        prompt_variant_index: promptCtx?.variantIndex
       }).catch(() => {
       });
       return result;
@@ -1481,7 +2683,7 @@ function createGenerateTextWrapper(aiModule, sessionCtx, debug = false) {
 }
 
 // src/trace/wrappers/vercel-ai/stream-text.ts
-function log2(...args) {
+function log3(...args) {
   if (isDebugMode()) console.log("[Fallom]", ...args);
 }
 function createStreamTextWrapper(aiModule, sessionCtx, debug = false) {
@@ -1490,7 +2692,47 @@ function createStreamTextWrapper(aiModule, sessionCtx, debug = false) {
     const params = args[0] || {};
     const startTime = Date.now();
     const captureContent2 = shouldCaptureContent();
-    const result = await aiModule.streamText(...args);
+    const toolTimings = /* @__PURE__ */ new Map();
+    let wrappedParams = params;
+    if (params.tools && typeof params.tools === "object") {
+      const wrappedTools = {};
+      for (const [toolName, tool] of Object.entries(params.tools)) {
+        if (tool && typeof tool.execute === "function") {
+          const originalExecute = tool.execute;
+          wrappedTools[toolName] = {
+            ...tool,
+            execute: async (...executeArgs) => {
+              const toolStartTime = Date.now();
+              const toolCallId = `${toolName}-${toolStartTime}`;
+              try {
+                const result2 = await originalExecute(...executeArgs);
+                const toolEndTime = Date.now();
+                toolTimings.set(toolCallId, {
+                  name: toolName,
+                  startTime: toolStartTime - startTime,
+                  endTime: toolEndTime - startTime,
+                  duration: toolEndTime - toolStartTime
+                });
+                return result2;
+              } catch (error) {
+                const toolEndTime = Date.now();
+                toolTimings.set(toolCallId, {
+                  name: toolName,
+                  startTime: toolStartTime - startTime,
+                  endTime: toolEndTime - startTime,
+                  duration: toolEndTime - toolStartTime
+                });
+                throw error;
+              }
+            }
+          };
+        } else {
+          wrappedTools[toolName] = tool;
+        }
+      }
+      wrappedParams = { ...params, tools: wrappedTools };
+    }
+    const result = await aiModule.streamText(wrappedParams);
     if (!isInitialized()) {
       return result;
     }
@@ -1504,72 +2746,213 @@ function createStreamTextWrapper(aiModule, sessionCtx, debug = false) {
       Promise.all([
         result.usage.catch(() => null),
         result.text?.catch(() => null),
-        result.finishReason?.catch(() => null)
-      ]).then(async ([rawUsage, responseText, finishReason]) => {
-        const endTime = Date.now();
-        if (debug || isDebugMode()) {
-          console.log("\n\u{1F50D} [Fallom Debug] streamText raw usage:", JSON.stringify(rawUsage, null, 2));
-          console.log("\u{1F50D} [Fallom Debug] streamText response text:", responseText?.slice(0, 100));
-          console.log("\u{1F50D} [Fallom Debug] streamText finish reason:", finishReason);
-        }
-        let providerMetadata = result?.experimental_providerMetadata;
-        if (providerMetadata && typeof providerMetadata.then === "function") {
-          try {
-            providerMetadata = await providerMetadata;
-          } catch {
-            providerMetadata = void 0;
+        result.finishReason?.catch(() => null),
+        result.toolCalls?.catch(() => null),
+        result.toolResults?.catch(() => null),
+        result.steps?.catch(() => null),
+        result.responseMessages?.catch(() => null)
+      ]).then(
+        async ([
+          rawUsage,
+          responseText,
+          finishReason,
+          toolCalls,
+          toolResults,
+          steps,
+          responseMessages
+        ]) => {
+          const endTime = Date.now();
+          if (debug || isDebugMode()) {
+            console.log(
+              "\n\u{1F50D} [Fallom Debug] streamText raw usage:",
+              JSON.stringify(rawUsage, null, 2)
+            );
+            console.log(
+              "\u{1F50D} [Fallom Debug] streamText response text:",
+              responseText?.slice(0, 100)
+            );
+            console.log(
+              "\u{1F50D} [Fallom Debug] streamText finish reason:",
+              finishReason
+            );
+            console.log(
+              "\u{1F50D} [Fallom Debug] streamText toolCalls:",
+              JSON.stringify(toolCalls, null, 2)
+            );
+            console.log(
+              "\u{1F50D} [Fallom Debug] streamText steps count:",
+              steps?.length
+            );
           }
-        }
-        const attributes = {
-          "fallom.sdk_version": "2",
-          "fallom.method": "streamText",
-          "fallom.is_streaming": true
-        };
-        if (captureContent2) {
-          attributes["fallom.raw.request"] = JSON.stringify({
-            prompt: params?.prompt,
-            messages: params?.messages,
-            system: params?.system,
-            model: modelId
+          let providerMetadata = result?.experimental_providerMetadata;
+          if (providerMetadata && typeof providerMetadata.then === "function") {
+            try {
+              providerMetadata = await providerMetadata;
+            } catch {
+              providerMetadata = void 0;
+            }
+          }
+          const attributes = {
+            "fallom.sdk_version": "2",
+            "fallom.method": "streamText",
+            "fallom.is_streaming": true
+          };
+          if (captureContent2) {
+            const mapToolCall = (tc) => ({
+              toolCallId: tc?.toolCallId,
+              toolName: tc?.toolName,
+              args: tc?.args,
+              // The actual arguments passed to the tool!
+              type: tc?.type
+            });
+            const mapToolResult = (tr) => ({
+              toolCallId: tr?.toolCallId,
+              toolName: tr?.toolName,
+              result: tr?.result,
+              // The actual result from the tool!
+              type: tr?.type
+            });
+            attributes["fallom.raw.request"] = JSON.stringify({
+              prompt: params?.prompt,
+              messages: params?.messages,
+              system: params?.system,
+              model: modelId,
+              tools: params?.tools ? Object.keys(params.tools) : void 0,
+              maxSteps: params?.maxSteps
+            });
+            attributes["fallom.raw.response"] = JSON.stringify({
+              text: responseText,
+              finishReason,
+              // Tool calls with FULL data (id, name, args)
+              toolCalls: toolCalls?.map(mapToolCall),
+              // Tool results with FULL data (id, name, result)
+              toolResults: toolResults?.map(mapToolResult),
+              // Multi-step agent data with FULL tool info including timestamps
+              steps: steps?.map((step) => ({
+                stepType: step?.stepType,
+                text: step?.text,
+                finishReason: step?.finishReason,
+                toolCalls: step?.toolCalls?.map(mapToolCall),
+                toolResults: step?.toolResults?.map(mapToolResult),
+                usage: step?.usage,
+                // Step-level timing from Vercel AI SDK
+                timestamp: step?.response?.timestamp,
+                responseId: step?.response?.id
+              })),
+              // Response messages (includes tool call/result messages)
+              responseMessages
+            });
+          }
+          if (rawUsage) {
+            attributes["fallom.raw.usage"] = JSON.stringify(rawUsage);
+          }
+          if (providerMetadata) {
+            attributes["fallom.raw.providerMetadata"] = JSON.stringify(providerMetadata);
+          }
+          if (firstTokenTime) {
+            attributes["fallom.time_to_first_token_ms"] = firstTokenTime - startTime;
+          }
+          const totalDurationMs = endTime - startTime;
+          const sortedToolTimings = Array.from(toolTimings.values()).sort(
+            (a, b) => a.startTime - b.startTime
+          );
+          const waterfallTimings = {
+            requestStart: 0,
+            firstTokenTime: firstTokenTime ? firstTokenTime - startTime : void 0,
+            responseEnd: totalDurationMs,
+            totalDurationMs,
+            isStreaming: true,
+            phases: [],
+            toolTimings: sortedToolTimings
+          };
+          if (firstTokenTime) {
+            waterfallTimings.phases.push({
+              type: "ttft",
+              label: "Time to First Token",
+              startMs: 0,
+              endMs: firstTokenTime - startTime,
+              durationMs: firstTokenTime - startTime,
+              accurate: true
+            });
+          }
+          if (sortedToolTimings.length > 0) {
+            const firstToolStart = Math.min(...sortedToolTimings.map((t) => t.startTime));
+            const lastToolEnd = Math.max(...sortedToolTimings.map((t) => t.endTime));
+            if (firstToolStart > 10) {
+              waterfallTimings.phases.push({
+                type: "llm",
+                label: "LLM Call 1 (decides tools)",
+                startMs: 0,
+                endMs: firstToolStart,
+                durationMs: firstToolStart,
+                accurate: true
+              });
+            }
+            sortedToolTimings.forEach((toolTiming) => {
+              waterfallTimings.phases.push({
+                type: "tool",
+                label: `${toolTiming.name}()`,
+                startMs: toolTiming.startTime,
+                endMs: toolTiming.endTime,
+                durationMs: toolTiming.duration,
+                accurate: true
+              });
+            });
+            const finalResponseDuration = totalDurationMs - lastToolEnd;
+            if (finalResponseDuration > 10) {
+              waterfallTimings.phases.push({
+                type: "response",
+                label: "LLM Call 2 \u2192 Final Response",
+                startMs: lastToolEnd,
+                endMs: totalDurationMs,
+                durationMs: finalResponseDuration,
+                accurate: true
+              });
+            }
+          }
+          if (steps) {
+            waterfallTimings.steps = steps.map((step, idx) => ({
+              stepIndex: idx,
+              stepType: step?.stepType,
+              finishReason: step?.finishReason,
+              timestamp: step?.response?.timestamp,
+              toolCalls: step?.toolCalls?.map((tc) => ({
+                id: tc?.toolCallId,
+                name: tc?.toolName
+              })),
+              usage: step?.usage
+            }));
+          }
+          attributes["fallom.raw.timings"] = JSON.stringify(waterfallTimings);
+          const promptCtx = getPromptContext();
+          sendTrace({
+            config_key: ctx.configKey,
+            session_id: ctx.sessionId,
+            customer_id: ctx.customerId,
+            trace_id: traceId,
+            span_id: spanId,
+            parent_span_id: parentSpanId,
+            name: "streamText",
+            kind: "llm",
+            model: modelId,
+            start_time: new Date(startTime).toISOString(),
+            end_time: new Date(endTime).toISOString(),
+            duration_ms: endTime - startTime,
+            status: "OK",
+            time_to_first_token_ms: firstTokenTime ? firstTokenTime - startTime : void 0,
+            is_streaming: true,
+            attributes,
+            // Prompt context (if prompts.get() or prompts.getAB() was called)
+            prompt_key: promptCtx?.promptKey,
+            prompt_version: promptCtx?.promptVersion,
+            prompt_ab_test_key: promptCtx?.abTestKey,
+            prompt_variant_index: promptCtx?.variantIndex
+          }).catch(() => {
           });
-          if (responseText || finishReason) {
-            attributes["fallom.raw.response"] = JSON.stringify({
-              text: responseText,
-              finishReason
-            });
-          }
-        }
-        if (rawUsage) {
-          attributes["fallom.raw.usage"] = JSON.stringify(rawUsage);
-        }
-        if (providerMetadata) {
-          attributes["fallom.raw.providerMetadata"] = JSON.stringify(providerMetadata);
-        }
-        if (firstTokenTime) {
-          attributes["fallom.time_to_first_token_ms"] = firstTokenTime - startTime;
         }
-        sendTrace({
-          config_key: ctx.configKey,
-          session_id: ctx.sessionId,
-          customer_id: ctx.customerId,
-          trace_id: traceId,
-          span_id: spanId,
-          parent_span_id: parentSpanId,
-          name: "streamText",
-          kind: "llm",
-          model: modelId,
-          start_time: new Date(startTime).toISOString(),
-          end_time: new Date(endTime).toISOString(),
-          duration_ms: endTime - startTime,
-          status: "OK",
-          time_to_first_token_ms: firstTokenTime ? firstTokenTime - startTime : void 0,
-          is_streaming: true,
-          attributes
-        }).catch(() => {
-        });
-      }).catch((error) => {
+      ).catch((error) => {
         const endTime = Date.now();
-        log2("\u274C streamText error:", error?.message);
+        log3("\u274C streamText error:", error?.message);
         sendTrace({
           config_key: ctx.configKey,
           session_id: ctx.sessionId,
@@ -1600,7 +2983,7 @@ function createStreamTextWrapper(aiModule, sessionCtx, debug = false) {
         for await (const chunk of originalTextStream) {
           if (!firstTokenTime) {
             firstTokenTime = Date.now();
-            log2("\u23F1\uFE0F Time to first token:", firstTokenTime - startTime, "ms");
+            log3("\u23F1\uFE0F Time to first token:", firstTokenTime - startTime, "ms");
           }
           yield chunk;
         }
@@ -1670,6 +3053,7 @@ function createGenerateObjectWrapper(aiModule, sessionCtx, debug = false) {
           result.experimental_providerMetadata
         );
       }
+      const promptCtx = getPromptContext();
       sendTrace({
         config_key: ctx.configKey,
         session_id: ctx.sessionId,
@@ -1684,7 +3068,12 @@ function createGenerateObjectWrapper(aiModule, sessionCtx, debug = false) {
         end_time: new Date(endTime).toISOString(),
         duration_ms: endTime - startTime,
         status: "OK",
-        attributes
+        attributes,
+        // Prompt context (if prompts.get() or prompts.getAB() was called)
+        prompt_key: promptCtx?.promptKey,
+        prompt_version: promptCtx?.promptVersion,
+        prompt_ab_test_key: promptCtx?.abTestKey,
+        prompt_variant_index: promptCtx?.variantIndex
       }).catch(() => {
       });
       return result;
@@ -1779,6 +3168,7 @@ function createStreamObjectWrapper(aiModule, sessionCtx, debug = false) {
         if (providerMetadata) {
           attributes["fallom.raw.providerMetadata"] = JSON.stringify(providerMetadata);
         }
+        const promptCtx = getPromptContext();
         sendTrace({
           config_key: ctx.configKey,
           session_id: ctx.sessionId,
@@ -1794,7 +3184,12 @@ function createStreamObjectWrapper(aiModule, sessionCtx, debug = false) {
           duration_ms: endTime - startTime,
           status: "OK",
           is_streaming: true,
-          attributes
+          attributes,
+          // Prompt context (if prompts.get() or prompts.getAB() was called)
+          prompt_key: promptCtx?.promptKey,
+          prompt_version: promptCtx?.promptVersion,
+          prompt_ab_test_key: promptCtx?.abTestKey,
+          prompt_variant_index: promptCtx?.variantIndex
         }).catch(() => {
         });
       }).catch((error) => {
@@ -2102,249 +3497,33 @@ function session(options) {
 // src/index.ts
 init_models();
 
-// src/prompts.ts
-var prompts_exports = {};
-__export(prompts_exports, {
-  clearPromptContext: () => clearPromptContext,
-  get: () => get2,
-  getAB: () => getAB,
-  getPromptContext: () => getPromptContext,
-  init: () => init3
+// src/evals/index.ts
+var evals_exports = {};
+__export(evals_exports, {
+  AVAILABLE_METRICS: () => AVAILABLE_METRICS,
+  DEFAULT_JUDGE_MODEL: () => DEFAULT_JUDGE_MODEL,
+  METRIC_PROMPTS: () => METRIC_PROMPTS,
+  compareModels: () => compareModels,
+  createCustomModel: () => createCustomModel,
+  createModelFromCallable: () => createModelFromCallable,
+  createOpenAIModel: () => createOpenAIModel,
+  customMetric: () => customMetric,
+  datasetFromFallom: () => datasetFromFallom,
+  datasetFromTraces: () => datasetFromTraces,
+  evaluate: () => evaluate,
+  getMetricName: () => getMetricName,
+  init: () => init4,
+  isCustomMetric: () => isCustomMetric,
+  uploadResults: () => uploadResultsPublic
 });
-var import_crypto2 = require("crypto");
-var apiKey3 = null;
-var baseUrl3 = "https://prompts.fallom.com";
-var initialized3 = false;
-var syncInterval2 = null;
-var debugMode3 = false;
-var promptCache = /* @__PURE__ */ new Map();
-var promptABCache = /* @__PURE__ */ new Map();
-var promptContext = null;
-var SYNC_TIMEOUT2 = 2e3;
-function log4(msg) {
-  if (debugMode3) {
-    console.log(`[Fallom Prompts] ${msg}`);
-  }
-}
-function init3(options = {}) {
-  apiKey3 = options.apiKey || process.env.FALLOM_API_KEY || null;
-  baseUrl3 = options.baseUrl || process.env.FALLOM_PROMPTS_URL || process.env.FALLOM_BASE_URL || "https://prompts.fallom.com";
-  initialized3 = true;
-  if (!apiKey3) {
-    return;
-  }
-  fetchAll().catch(() => {
-  });
-  if (!syncInterval2) {
-    syncInterval2 = setInterval(() => {
-      fetchAll().catch(() => {
-      });
-    }, 3e4);
-    syncInterval2.unref();
-  }
-}
-function ensureInit2() {
-  if (!initialized3) {
-    try {
-      init3();
-    } catch {
-    }
-  }
-}
-async function fetchAll() {
-  await Promise.all([fetchPrompts(), fetchPromptABTests()]);
-}
-async function fetchPrompts(timeout = SYNC_TIMEOUT2) {
-  if (!apiKey3) return;
-  try {
-    const controller = new AbortController();
-    const timeoutId = setTimeout(() => controller.abort(), timeout);
-    const resp = await fetch(`${baseUrl3}/prompts`, {
-      headers: { Authorization: `Bearer ${apiKey3}` },
-      signal: controller.signal
-    });
-    clearTimeout(timeoutId);
-    if (resp.ok) {
-      const data = await resp.json();
-      for (const p of data.prompts || []) {
-        if (!promptCache.has(p.key)) {
-          promptCache.set(p.key, { versions: /* @__PURE__ */ new Map(), current: null });
-        }
-        const cached = promptCache.get(p.key);
-        cached.versions.set(p.version, {
-          systemPrompt: p.system_prompt,
-          userTemplate: p.user_template
-        });
-        cached.current = p.version;
-      }
-    }
-  } catch {
-  }
-}
-async function fetchPromptABTests(timeout = SYNC_TIMEOUT2) {
-  if (!apiKey3) return;
-  try {
-    const controller = new AbortController();
-    const timeoutId = setTimeout(() => controller.abort(), timeout);
-    const resp = await fetch(`${baseUrl3}/prompt-ab-tests`, {
-      headers: { Authorization: `Bearer ${apiKey3}` },
-      signal: controller.signal
-    });
-    clearTimeout(timeoutId);
-    if (resp.ok) {
-      const data = await resp.json();
-      for (const t of data.prompt_ab_tests || []) {
-        if (!promptABCache.has(t.key)) {
-          promptABCache.set(t.key, { versions: /* @__PURE__ */ new Map(), current: null });
-        }
-        const cached = promptABCache.get(t.key);
-        cached.versions.set(t.version, { variants: t.variants });
-        cached.current = t.version;
-      }
-    }
-  } catch {
-  }
-}
-function replaceVariables(template, variables) {
-  if (!variables) return template;
-  return template.replace(/\{\{(\s*\w+\s*)\}\}/g, (match, varName) => {
-    const key = varName.trim();
-    return key in variables ? String(variables[key]) : match;
-  });
-}
-function setPromptContext(ctx) {
-  promptContext = ctx;
-}
-function getPromptContext() {
-  const ctx = promptContext;
-  promptContext = null;
-  return ctx;
-}
-async function get2(promptKey, options = {}) {
-  const { variables, version, debug = false } = options;
-  debugMode3 = debug;
-  ensureInit2();
-  log4(`get() called: promptKey=${promptKey}`);
-  let promptData = promptCache.get(promptKey);
-  if (!promptData) {
-    log4("Not in cache, fetching...");
-    await fetchPrompts(SYNC_TIMEOUT2);
-    promptData = promptCache.get(promptKey);
-  }
-  if (!promptData) {
-    throw new Error(
-      `Prompt '${promptKey}' not found. Check that it exists in your Fallom dashboard.`
-    );
-  }
-  const targetVersion = version ?? promptData.current;
-  const content = promptData.versions.get(targetVersion);
-  if (!content) {
-    throw new Error(
-      `Prompt '${promptKey}' version ${targetVersion} not found.`
-    );
-  }
-  const system = replaceVariables(content.systemPrompt, variables);
-  const user = replaceVariables(content.userTemplate, variables);
-  setPromptContext({
-    promptKey,
-    promptVersion: targetVersion
-  });
-  log4(`\u2705 Got prompt: ${promptKey} v${targetVersion}`);
-  return {
-    key: promptKey,
-    version: targetVersion,
-    system,
-    user
-  };
-}
-async function getAB(abTestKey, sessionId, options = {}) {
-  const { variables, debug = false } = options;
-  debugMode3 = debug;
-  ensureInit2();
-  log4(`getAB() called: abTestKey=${abTestKey}, sessionId=${sessionId}`);
-  let abData = promptABCache.get(abTestKey);
-  if (!abData) {
-    log4("Not in cache, fetching...");
-    await fetchPromptABTests(SYNC_TIMEOUT2);
-    abData = promptABCache.get(abTestKey);
-  }
-  if (!abData) {
-    throw new Error(
-      `Prompt A/B test '${abTestKey}' not found. Check that it exists in your Fallom dashboard.`
-    );
-  }
-  const currentVersion = abData.current;
-  const versionData = abData.versions.get(currentVersion);
-  if (!versionData) {
-    throw new Error(`Prompt A/B test '${abTestKey}' has no current version.`);
-  }
-  const { variants } = versionData;
-  log4(`A/B test '${abTestKey}' has ${variants?.length ?? 0} variants`);
-  log4(`Version data: ${JSON.stringify(versionData, null, 2)}`);
-  if (!variants || variants.length === 0) {
-    throw new Error(
-      `Prompt A/B test '${abTestKey}' has no variants configured.`
-    );
-  }
-  const hashBytes = (0, import_crypto2.createHash)("md5").update(sessionId).digest();
-  const hashVal = hashBytes.readUInt32BE(0) % 1e6;
-  let cumulative = 0;
-  let selectedVariant = variants[variants.length - 1];
-  let selectedIndex = variants.length - 1;
-  for (let i = 0; i < variants.length; i++) {
-    cumulative += variants[i].weight * 1e4;
-    if (hashVal < cumulative) {
-      selectedVariant = variants[i];
-      selectedIndex = i;
-      break;
-    }
-  }
-  const promptKey = selectedVariant.prompt_key;
-  const promptVersion = selectedVariant.prompt_version;
-  let promptData = promptCache.get(promptKey);
-  if (!promptData) {
-    await fetchPrompts(SYNC_TIMEOUT2);
-    promptData = promptCache.get(promptKey);
-  }
-  if (!promptData) {
-    throw new Error(
-      `Prompt '${promptKey}' (from A/B test '${abTestKey}') not found.`
-    );
-  }
-  const targetVersion = promptVersion ?? promptData.current;
-  const content = promptData.versions.get(targetVersion);
-  if (!content) {
-    throw new Error(
-      `Prompt '${promptKey}' version ${targetVersion} not found.`
-    );
-  }
-  const system = replaceVariables(content.systemPrompt, variables);
-  const user = replaceVariables(content.userTemplate, variables);
-  setPromptContext({
-    promptKey,
-    promptVersion: targetVersion,
-    abTestKey,
-    variantIndex: selectedIndex
-  });
-  log4(
-    `\u2705 Got prompt from A/B: ${promptKey} v${targetVersion} (variant ${selectedIndex})`
-  );
-  return {
-    key: promptKey,
-    version: targetVersion,
-    system,
-    user,
-    abTestKey,
-    variantIndex: selectedIndex
-  };
-}
-function clearPromptContext() {
-  promptContext = null;
-}
+init_types();
+init_prompts();
+init_core();
+init_helpers();
 
 // src/init.ts
 init_models();
-async function init4(options = {}) {
+async function init5(options = {}) {
   const tracesUrl = options.tracesUrl || process.env.FALLOM_TRACES_URL || "https://traces.fallom.com";
   const configsUrl = options.configsUrl || process.env.FALLOM_CONFIGS_URL || "https://configs.fallom.com";
   const promptsUrl = options.promptsUrl || process.env.FALLOM_PROMPTS_URL || "https://prompts.fallom.com";
@@ -2354,18 +3533,18 @@ async function init4(options = {}) {
     captureContent: options.captureContent,
     debug: options.debug
   });
-  init2({
+  init3({
     apiKey: options.apiKey,
     baseUrl: configsUrl
   });
-  init3({
+  init2({
     apiKey: options.apiKey,
     baseUrl: promptsUrl
   });
 }
 
 // src/mastra.ts
-var import_core12 = require("@opentelemetry/core");
+var import_core13 = require("@opentelemetry/core");
 var promptContext2 = {};
 function setMastraPrompt(promptKey, version) {
   promptContext2 = {
@@ -2415,7 +3594,7 @@ var FallomExporter = class {
    */
   export(spans, resultCallback) {
     if (spans.length === 0) {
-      resultCallback({ code: import_core12.ExportResultCode.SUCCESS });
+      resultCallback({ code: import_core13.ExportResultCode.SUCCESS });
       return;
     }
     this.log(`Exporting ${spans.length} spans...`);
@@ -2432,11 +3611,11 @@ var FallomExporter = class {
     }
     const exportPromise = this.sendSpans(spans).then(() => {
       this.log("Export successful");
-      resultCallback({ code: import_core12.ExportResultCode.SUCCESS });
+      resultCallback({ code: import_core13.ExportResultCode.SUCCESS });
     }).catch((error) => {
       console.error("[FallomExporter] Export failed:", error);
       resultCallback({
-        code: import_core12.ExportResultCode.FAILED,
+        code: import_core13.ExportResultCode.FAILED,
         error: error instanceof Error ? error : new Error(String(error))
       });
     });
@@ -2605,10 +3784,11 @@ var FallomExporter = class {
 // src/index.ts
 init_models();
 var index_default = {
-  init: init4,
+  init: init5,
   trace: trace_exports,
   models: models_exports,
   prompts: prompts_exports,
+  evals: evals_exports,
   session
 };
 // Annotate the CommonJS export names for ESM import in node:
@@ -2616,6 +3796,7 @@ var index_default = {
   FallomExporter,
   FallomSession,
   clearMastraPrompt,
+  evals,
   init,
   models,
   prompts,