@fallom/trace 0.2.17 → 0.2.21

package/dist/index.mjs

@@ -7,6 +7,8 @@ import {
   DEFAULT_JUDGE_MODEL,
   EvaluationDataset,
   METRIC_PROMPTS,
+  buildGEvalPrompt,
+  calculateAggregateScores,
   compareModels,
   createCustomModel,
   createModelFromCallable,
@@ -14,12 +16,14 @@ import {
   customMetric,
   datasetFromFallom,
   datasetFromTraces,
+  detectRegression,
   evaluate,
   getMetricName,
   init as init2,
   isCustomMetric,
+  runGEval,
   uploadResultsPublic
-} from "./chunk-3HBKT4HK.mjs";
+} from "./chunk-GZ6TE7G4.mjs";
 import {
   __export
 } from "./chunk-7P6ASYW6.mjs";
@@ -41,7 +45,7 @@ import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
 // node_modules/@opentelemetry/resources/build/esm/Resource.js
 import { diag } from "@opentelemetry/api";
 
-// node_modules/@opentelemetry/resources/node_modules/@opentelemetry/semantic-conventions/build/esm/resource/SemanticResourceAttributes.js
+// node_modules/@opentelemetry/semantic-conventions/build/esm/resource/SemanticResourceAttributes.js
 var SemanticResourceAttributes = {
   /**
    * Name of the cloud provider.
@@ -1567,20 +1571,36 @@ function createGenerateTextWrapper(aiModule, sessionCtx, debug = false) {
           tools: params?.tools ? Object.keys(params.tools) : void 0,
           maxSteps: params?.maxSteps
         });
-        const mapToolCall = (tc) => ({
-          toolCallId: tc?.toolCallId,
-          toolName: tc?.toolName,
-          args: tc?.args ?? tc?.input,
-          // v4: args, v5: input
-          type: tc?.type
-        });
-        const mapToolResult = (tr) => ({
-          toolCallId: tr?.toolCallId,
-          toolName: tr?.toolName,
-          result: tr?.result ?? tr?.output,
-          // v4: result, v5: output
-          type: tr?.type
-        });
+        const mapToolCall = (tc) => {
+          let args2 = tc?.args ?? tc?.input;
+          if (args2 === void 0 && tc) {
+            const { type, toolCallId, toolName, providerExecuted, dynamic, invalid, error, providerMetadata, ...rest } = tc;
+            if (Object.keys(rest).length > 0) {
+              args2 = rest;
+            }
+          }
+          return {
+            toolCallId: tc?.toolCallId,
+            toolName: tc?.toolName,
+            args: args2,
+            type: tc?.type
+          };
+        };
+        const mapToolResult = (tr) => {
+          let result2 = tr?.result ?? tr?.output;
+          if (result2 === void 0 && tr) {
+            const { type, toolCallId, toolName, ...rest } = tr;
+            if (Object.keys(rest).length > 0) {
+              result2 = rest;
+            }
+          }
+          return {
+            toolCallId: tr?.toolCallId,
+            toolName: tr?.toolName,
+            result: result2,
+            type: tr?.type
+          };
+        };
         attributes["fallom.raw.response"] = JSON.stringify({
           text: result?.text,
           finishReason: result?.finishReason,
@@ -1793,7 +1813,9 @@ function createStreamTextWrapper(aiModule, sessionCtx, debug = false) {
     let wrappedParams = params;
     if (params.tools && typeof params.tools === "object") {
       const wrappedTools = {};
-      for (const [toolName, tool] of Object.entries(params.tools)) {
+      for (const [toolName, tool] of Object.entries(
+        params.tools
+      )) {
         if (tool && typeof tool.execute === "function") {
           const originalExecute = tool.execute;
           wrappedTools[toolName] = {
@@ -1876,10 +1898,54 @@ function createStreamTextWrapper(aiModule, sessionCtx, debug = false) {
               "\u{1F50D} [Fallom Debug] streamText toolCalls:",
               JSON.stringify(toolCalls, null, 2)
             );
+            if (toolCalls?.[0]) {
+              console.log(
+                "\u{1F50D} [Fallom Debug] streamText toolCalls[0] keys:",
+                Object.keys(toolCalls[0])
+              );
+              console.log(
+                "\u{1F50D} [Fallom Debug] streamText toolCalls[0] full:",
+                JSON.stringify(
+                  toolCalls[0],
+                  Object.getOwnPropertyNames(toolCalls[0]),
+                  2
+                )
+              );
+            }
             console.log(
               "\u{1F50D} [Fallom Debug] streamText steps count:",
               steps?.length
             );
+            if (steps?.[0]?.toolCalls?.[0]) {
+              const tc = steps[0].toolCalls[0];
+              console.log(
+                "\u{1F50D} [Fallom Debug] steps[0].toolCalls[0] keys:",
+                Object.keys(tc)
+              );
+              console.log(
+                "\u{1F50D} [Fallom Debug] steps[0].toolCalls[0].args (v4):",
+                tc.args
+              );
+              console.log(
+                "\u{1F50D} [Fallom Debug] steps[0].toolCalls[0].input (v5):",
+                tc.input
+              );
+            }
+            if (steps?.[0]?.toolResults?.[0]) {
+              const tr = steps[0].toolResults[0];
+              console.log(
+                "\u{1F50D} [Fallom Debug] steps[0].toolResults[0] keys:",
+                Object.keys(tr)
+              );
+              console.log(
+                "\u{1F50D} [Fallom Debug] steps[0].toolResults[0].result (v4):",
+                typeof tr.result === "string" ? tr.result.slice(0, 200) : tr.result
+              );
+              console.log(
+                "\u{1F50D} [Fallom Debug] steps[0].toolResults[0].output (v5):",
+                typeof tr.output === "string" ? tr.output.slice(0, 200) : tr.output
+              );
+            }
           }
           let providerMetadata = result?.experimental_providerMetadata;
           if (providerMetadata && typeof providerMetadata.then === "function") {
@@ -1895,20 +1961,46 @@ function createStreamTextWrapper(aiModule, sessionCtx, debug = false) {
             "fallom.is_streaming": true
           };
           if (captureContent2) {
-            const mapToolCall = (tc) => ({
-              toolCallId: tc?.toolCallId,
-              toolName: tc?.toolName,
-              args: tc?.args ?? tc?.input,
-              // v4: args, v5: input
-              type: tc?.type
-            });
-            const mapToolResult = (tr) => ({
-              toolCallId: tr?.toolCallId,
-              toolName: tr?.toolName,
-              result: tr?.result ?? tr?.output,
-              // v4: result, v5: output
-              type: tr?.type
-            });
+            const mapToolCall = (tc) => {
+              let args2 = tc?.args ?? tc?.input;
+              if (args2 === void 0 && tc) {
+                const {
+                  type,
+                  toolCallId,
+                  toolName,
+                  providerExecuted,
+                  dynamic,
+                  invalid,
+                  error,
+                  providerMetadata: providerMetadata2,
+                  ...rest
+                } = tc;
+                if (Object.keys(rest).length > 0) {
+                  args2 = rest;
+                }
+              }
+              return {
+                toolCallId: tc?.toolCallId,
+                toolName: tc?.toolName,
+                args: args2,
+                type: tc?.type
+              };
+            };
+            const mapToolResult = (tr) => {
+              let result2 = tr?.result ?? tr?.output;
+              if (result2 === void 0 && tr) {
+                const { type, toolCallId, toolName, ...rest } = tr;
+                if (Object.keys(rest).length > 0) {
+                  result2 = rest;
+                }
+              }
+              return {
+                toolCallId: tr?.toolCallId,
+                toolName: tr?.toolName,
+                result: result2,
+                type: tr?.type
+              };
+            };
             attributes["fallom.raw.request"] = JSON.stringify({
               prompt: params?.prompt,
               messages: params?.messages,
@@ -1950,7 +2042,10 @@ function createStreamTextWrapper(aiModule, sessionCtx, debug = false) {
             attributes["fallom.time_to_first_token_ms"] = firstTokenTime - startTime;
           }
           try {
-            attributes["fallom.raw.metadata"] = JSON.stringify(result, sanitizeMetadataOnly);
+            attributes["fallom.raw.metadata"] = JSON.stringify(
+              result,
+              sanitizeMetadataOnly
+            );
           } catch {
           }
           const totalDurationMs = endTime - startTime;
@@ -1977,8 +2072,12 @@ function createStreamTextWrapper(aiModule, sessionCtx, debug = false) {
             });
           }
           if (sortedToolTimings.length > 0) {
-            const firstToolStart = Math.min(...sortedToolTimings.map((t) => t.startTime));
-            const lastToolEnd = Math.max(...sortedToolTimings.map((t) => t.endTime));
+            const firstToolStart = Math.min(
+              ...sortedToolTimings.map((t) => t.startTime)
+            );
+            const lastToolEnd = Math.max(
+              ...sortedToolTimings.map((t) => t.endTime)
+            );
             if (firstToolStart > 10) {
               waterfallTimings.phases.push({
                 type: "llm",
@@ -2643,6 +2742,8 @@ __export(evals_exports, {
   DEFAULT_JUDGE_MODEL: () => DEFAULT_JUDGE_MODEL,
   EvaluationDataset: () => EvaluationDataset,
   METRIC_PROMPTS: () => METRIC_PROMPTS,
+  buildGEvalPrompt: () => buildGEvalPrompt,
+  calculateAggregateScores: () => calculateAggregateScores,
   compareModels: () => compareModels,
   createCustomModel: () => createCustomModel,
   createModelFromCallable: () => createModelFromCallable,
@@ -2650,10 +2751,12 @@ __export(evals_exports, {
   customMetric: () => customMetric,
   datasetFromFallom: () => datasetFromFallom,
   datasetFromTraces: () => datasetFromTraces,
+  detectRegression: () => detectRegression,
   evaluate: () => evaluate,
   getMetricName: () => getMetricName,
   init: () => init2,
   isCustomMetric: () => isCustomMetric,
+  runGEval: () => runGEval,
   uploadResults: () => uploadResultsPublic
 });
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fallom/trace",
-  "version": "0.2.17",
+  "version": "0.2.21",
   "description": "Model A/B testing and tracing for LLM applications. Zero latency, production-ready.",
   "main": "./dist/index.js",
   "module": "./dist/index.mjs",

package/dist/chunk-KFD5AQ7V.mjs

@@ -1,308 +0,0 @@
-var __defProp = Object.defineProperty;
-var __export = (target, all) => {
-  for (var name in all)
-    __defProp(target, name, { get: all[name], enumerable: true });
-};
-
-// src/models.ts
-var models_exports = {};
-__export(models_exports, {
-  get: () => get,
-  init: () => init
-});
-import { createHash } from "crypto";
-var apiKey = null;
-var baseUrl = "https://configs.fallom.com";
-var initialized = false;
-var syncInterval = null;
-var debugMode = false;
-var configCache = /* @__PURE__ */ new Map();
-var SYNC_TIMEOUT = 2e3;
-var RECORD_TIMEOUT = 1e3;
-function log(msg) {
-  if (debugMode) {
-    console.log(`[Fallom] ${msg}`);
-  }
-}
-function evaluateTargeting(targeting, customerId, context) {
-  if (!targeting || targeting.enabled === false) {
-    return null;
-  }
-  const evalContext = {
-    ...context || {},
-    ...customerId ? { customerId } : {}
-  };
-  log(`Evaluating targeting with context: ${JSON.stringify(evalContext)}`);
-  if (targeting.individualTargets) {
-    for (const target of targeting.individualTargets) {
-      const fieldValue = evalContext[target.field];
-      if (fieldValue === target.value) {
-        log(`Individual target matched: ${target.field}=${target.value} -> variant ${target.variantIndex}`);
-        return target.variantIndex;
-      }
-    }
-  }
-  if (targeting.rules) {
-    for (const rule of targeting.rules) {
-      const allConditionsMatch = rule.conditions.every((condition) => {
-        const fieldValue = evalContext[condition.field];
-        if (fieldValue === void 0) return false;
-        switch (condition.operator) {
-          case "eq":
-            return fieldValue === condition.value;
-          case "neq":
-            return fieldValue !== condition.value;
-          case "in":
-            return Array.isArray(condition.value) && condition.value.includes(fieldValue);
-          case "nin":
-            return Array.isArray(condition.value) && !condition.value.includes(fieldValue);
-          case "contains":
-            return typeof condition.value === "string" && fieldValue.includes(condition.value);
-          case "startsWith":
-            return typeof condition.value === "string" && fieldValue.startsWith(condition.value);
-          case "endsWith":
-            return typeof condition.value === "string" && fieldValue.endsWith(condition.value);
-          default:
-            return false;
-        }
-      });
-      if (allConditionsMatch) {
-        log(`Rule matched: ${JSON.stringify(rule.conditions)} -> variant ${rule.variantIndex}`);
-        return rule.variantIndex;
-      }
-    }
-  }
-  log("No targeting rules matched, falling back to weighted random");
-  return null;
-}
-function init(options = {}) {
-  apiKey = options.apiKey || process.env.FALLOM_API_KEY || null;
-  baseUrl = options.baseUrl || process.env.FALLOM_CONFIGS_URL || process.env.FALLOM_BASE_URL || "https://configs.fallom.com";
-  initialized = true;
-  if (!apiKey) {
-    return;
-  }
-  fetchConfigs().catch(() => {
-  });
-  if (!syncInterval) {
-    syncInterval = setInterval(() => {
-      fetchConfigs().catch(() => {
-      });
-    }, 3e4);
-    syncInterval.unref();
-  }
-}
-function ensureInit() {
-  if (!initialized) {
-    try {
-      init();
-    } catch {
-    }
-  }
-}
-async function fetchConfigs(timeout = SYNC_TIMEOUT) {
-  if (!apiKey) {
-    log("_fetchConfigs: No API key, skipping");
-    return;
-  }
-  try {
-    log(`Fetching configs from ${baseUrl}/configs`);
-    const controller = new AbortController();
-    const timeoutId = setTimeout(() => controller.abort(), timeout);
-    const resp = await fetch(`${baseUrl}/configs`, {
-      headers: { Authorization: `Bearer ${apiKey}` },
-      signal: controller.signal
-    });
-    clearTimeout(timeoutId);
-    log(`Response status: ${resp.status}`);
-    if (resp.ok) {
-      const data = await resp.json();
-      const configs = data.configs || [];
-      log(`Got ${configs.length} configs: ${configs.map((c) => c.key)}`);
-      for (const c of configs) {
-        const key = c.key;
-        const version = c.version || 1;
-        log(`Config '${key}' v${version}: ${JSON.stringify(c.variants)}`);
-        if (!configCache.has(key)) {
-          configCache.set(key, { versions: /* @__PURE__ */ new Map(), latest: null });
-        }
-        const cached = configCache.get(key);
-        cached.versions.set(version, c);
-        cached.latest = version;
-      }
-    } else {
-      log(`Fetch failed: ${resp.statusText}`);
-    }
-  } catch (e) {
-    log(`Fetch exception: ${e}`);
-  }
-}
-async function fetchSpecificVersion(configKey, version, timeout = SYNC_TIMEOUT) {
-  if (!apiKey) return null;
-  try {
-    const controller = new AbortController();
-    const timeoutId = setTimeout(() => controller.abort(), timeout);
-    const resp = await fetch(
-      `${baseUrl}/configs/${configKey}/version/${version}`,
-      {
-        headers: { Authorization: `Bearer ${apiKey}` },
-        signal: controller.signal
-      }
-    );
-    clearTimeout(timeoutId);
-    if (resp.ok) {
-      const config = await resp.json();
-      if (!configCache.has(configKey)) {
-        configCache.set(configKey, { versions: /* @__PURE__ */ new Map(), latest: null });
-      }
-      configCache.get(configKey).versions.set(version, config);
-      return config;
-    }
-  } catch {
-  }
-  return null;
-}
-async function get(configKey, sessionId, options = {}) {
-  const { version, fallback, customerId, context, debug = false } = options;
-  debugMode = debug;
-  ensureInit();
-  log(
-    `get() called: configKey=${configKey}, sessionId=${sessionId}, fallback=${fallback}`
-  );
-  try {
-    let configData = configCache.get(configKey);
-    log(
-      `Cache lookup for '${configKey}': ${configData ? "found" : "not found"}`
-    );
-    if (!configData) {
-      log("Not in cache, fetching...");
-      await fetchConfigs(SYNC_TIMEOUT);
-      configData = configCache.get(configKey);
-      log(
-        `After fetch, cache lookup: ${configData ? "found" : "still not found"}`
-      );
-    }
-    if (!configData) {
-      log(`Config not found, using fallback: ${fallback}`);
-      if (fallback) {
-        console.warn(
-          `[Fallom WARNING] Config '${configKey}' not found, using fallback model: ${fallback}`
-        );
-        return returnModel(configKey, sessionId, fallback, 0);
-      }
-      throw new Error(
-        `Config '${configKey}' not found. Check that it exists in your Fallom dashboard.`
-      );
-    }
-    let config;
-    let targetVersion;
-    if (version !== void 0) {
-      config = configData.versions.get(version);
-      if (!config) {
-        config = await fetchSpecificVersion(configKey, version, SYNC_TIMEOUT) || void 0;
-      }
-      if (!config) {
-        if (fallback) {
-          console.warn(
-            `[Fallom WARNING] Config '${configKey}' version ${version} not found, using fallback: ${fallback}`
-          );
-          return returnModel(configKey, sessionId, fallback, 0);
-        }
-        throw new Error(`Config '${configKey}' version ${version} not found.`);
-      }
-      targetVersion = version;
-    } else {
-      targetVersion = configData.latest;
-      config = configData.versions.get(targetVersion);
-      if (!config) {
-        if (fallback) {
-          console.warn(
-            `[Fallom WARNING] Config '${configKey}' has no cached version, using fallback: ${fallback}`
-          );
-          return returnModel(configKey, sessionId, fallback, 0);
-        }
-        throw new Error(`Config '${configKey}' has no cached version.`);
-      }
-    }
-    const variantsRaw = config.variants;
-    const configVersion = config.version || targetVersion;
-    const variants = Array.isArray(variantsRaw) ? variantsRaw : Object.values(variantsRaw);
-    log(
-      `Config found! Version: ${configVersion}, Variants: ${JSON.stringify(
-        variants
-      )}`
-    );
-    const targetedVariantIndex = evaluateTargeting(config.targeting, customerId, context);
-    if (targetedVariantIndex !== null && variants[targetedVariantIndex]) {
-      const assignedModel2 = variants[targetedVariantIndex].model;
-      log(`\u2705 Assigned model via targeting: ${assignedModel2}`);
-      return returnModel(configKey, sessionId, assignedModel2, configVersion);
-    }
-    const hashBytes = createHash("md5").update(sessionId).digest();
-    const hashVal = hashBytes.readUInt32BE(0) % 1e6;
-    log(`Session hash: ${hashVal} (out of 1,000,000)`);
-    let cumulative = 0;
-    let assignedModel = variants[variants.length - 1].model;
-    for (const v of variants) {
-      const oldCumulative = cumulative;
-      cumulative += v.weight * 1e4;
-      log(
-        `Variant ${v.model}: weight=${v.weight}%, range=${oldCumulative}-${cumulative}, hash=${hashVal}, match=${hashVal < cumulative}`
-      );
-      if (hashVal < cumulative) {
-        assignedModel = v.model;
-        break;
-      }
-    }
-    log(`\u2705 Assigned model via weighted random: ${assignedModel}`);
-    return returnModel(configKey, sessionId, assignedModel, configVersion);
-  } catch (e) {
-    if (e instanceof Error && e.message.includes("not found")) {
-      throw e;
-    }
-    if (fallback) {
-      console.warn(
-        `[Fallom WARNING] Error getting model for '${configKey}': ${e}. Using fallback: ${fallback}`
-      );
-      return returnModel(configKey, sessionId, fallback, 0);
-    }
-    throw e;
-  }
-}
-function returnModel(configKey, sessionId, model, version) {
-  if (version > 0) {
-    recordSession(configKey, version, sessionId, model).catch(() => {
-    });
-  }
-  return model;
-}
-async function recordSession(configKey, version, sessionId, model) {
-  if (!apiKey) return;
-  try {
-    const controller = new AbortController();
-    const timeoutId = setTimeout(() => controller.abort(), RECORD_TIMEOUT);
-    await fetch(`${baseUrl}/sessions`, {
-      method: "POST",
-      headers: {
-        Authorization: `Bearer ${apiKey}`,
-        "Content-Type": "application/json"
-      },
-      body: JSON.stringify({
-        config_key: configKey,
-        config_version: version,
-        session_id: sessionId,
-        assigned_model: model
-      }),
-      signal: controller.signal
-    });
-    clearTimeout(timeoutId);
-  } catch {
-  }
-}
-
-export {
-  __export,
-  init,
-  get,
-  models_exports
-};

package/dist/models-SEFDGZU2.mjs

@@ -1,8 +0,0 @@
-import {
-  get,
-  init
-} from "./chunk-KFD5AQ7V.mjs";
-export {
-  get,
-  init
-};