@fallom/trace 0.2.24 → 0.2.26

package/dist/{core-Q3IHBEHB.mjs → core-46Z4Q54J.mjs}

@@ -7,7 +7,7 @@ import {
   evaluate,
   init,
   uploadResultsPublic
-} from "./chunk-3VWF2OJX.mjs";
+} from "./chunk-2NGJF2JZ.mjs";
 import "./chunk-7P6ASYW6.mjs";
 export {
   DEFAULT_JUDGE_MODEL,

package/dist/core-4L56QWI7.mjs

@@ -0,0 +1,21 @@
+import {
+  DEFAULT_JUDGE_MODEL,
+  _apiKey,
+  _baseUrl,
+  _initialized,
+  compareModels,
+  evaluate,
+  init,
+  uploadResultsPublic
+} from "./chunk-3HBKT4HK.mjs";
+import "./chunk-7P6ASYW6.mjs";
+export {
+  DEFAULT_JUDGE_MODEL,
+  _apiKey,
+  _baseUrl,
+  _initialized,
+  compareModels,
+  evaluate,
+  init,
+  uploadResultsPublic
+};

package/dist/core-DUG2SP2V.mjs

@@ -0,0 +1,21 @@
+import {
+  DEFAULT_JUDGE_MODEL,
+  _apiKey,
+  _baseUrl,
+  _initialized,
+  compareModels,
+  evaluate,
+  init,
+  uploadResultsPublic
+} from "./chunk-GZ6TE7G4.mjs";
+import "./chunk-7P6ASYW6.mjs";
+export {
+  DEFAULT_JUDGE_MODEL,
+  _apiKey,
+  _baseUrl,
+  _initialized,
+  compareModels,
+  evaluate,
+  init,
+  uploadResultsPublic
+};

package/dist/core-JLHYFVYS.mjs

@@ -0,0 +1,21 @@
+import {
+  DEFAULT_JUDGE_MODEL,
+  _apiKey,
+  _baseUrl,
+  _initialized,
+  compareModels,
+  evaluate,
+  init,
+  uploadResultsPublic
+} from "./chunk-XBZ3ESNV.mjs";
+import "./chunk-7P6ASYW6.mjs";
+export {
+  DEFAULT_JUDGE_MODEL,
+  _apiKey,
+  _baseUrl,
+  _initialized,
+  compareModels,
+  evaluate,
+  init,
+  uploadResultsPublic
+};

package/dist/core-NTEI2B5Z.mjs

@@ -0,0 +1,21 @@
+import {
+  DEFAULT_JUDGE_MODEL,
+  _apiKey,
+  _baseUrl,
+  _initialized,
+  compareModels,
+  evaluate,
+  init,
+  uploadResultsPublic
+} from "./chunk-FTZVXPQN.mjs";
+import "./chunk-7P6ASYW6.mjs";
+export {
+  DEFAULT_JUDGE_MODEL,
+  _apiKey,
+  _baseUrl,
+  _initialized,
+  compareModels,
+  evaluate,
+  init,
+  uploadResultsPublic
+};

package/dist/index.d.mts

@@ -445,6 +445,12 @@ interface EvalResult {
     input: string;
     output: string;
     systemMessage?: string;
+    /** Expected/golden output for comparison (if provided) */
+    expectedOutput?: string;
+    /** Retrieved documents/context for RAG evaluation */
+    context?: string[];
+    /** Additional metadata */
+    metadata?: Record<string, unknown>;
     model: string;
     isProduction: boolean;
     answerRelevancy?: number;

package/dist/index.d.ts

@@ -445,6 +445,12 @@ interface EvalResult {
     input: string;
     output: string;
     systemMessage?: string;
+    /** Expected/golden output for comparison (if provided) */
+    expectedOutput?: string;
+    /** Retrieved documents/context for RAG evaluation */
+    context?: string[];
+    /** Additional metadata */
+    metadata?: Record<string, unknown>;
     model: string;
     isProduction: boolean;
     answerRelevancy?: number;

package/dist/index.js

@@ -1047,13 +1047,22 @@ async function evaluate(options) {
     _skipUpload = false
   } = options;
   let dataset;
+  let testCaseExtras = /* @__PURE__ */ new Map();
   if (testCases !== void 0 && testCases.length > 0) {
-    dataset = testCases.map((tc) => ({
-      input: tc.input,
-      output: tc.actualOutput,
-      systemMessage: tc.systemMessage,
-      metadata: tc.metadata
-    }));
+    dataset = testCases.map((tc, idx) => {
+      if (tc.expectedOutput || tc.context) {
+        testCaseExtras.set(idx, {
+          expectedOutput: tc.expectedOutput,
+          context: tc.context
+        });
+      }
+      return {
+        input: tc.input,
+        output: tc.actualOutput,
+        systemMessage: tc.systemMessage,
+        metadata: tc.metadata
+      };
+    });
   } else if (datasetInput !== void 0) {
     dataset = await resolveDataset(datasetInput);
   } else {
@@ -1072,10 +1081,14 @@ async function evaluate(options) {
   for (let i = 0; i < dataset.length; i++) {
     const item = dataset[i];
     if (verbose) console.log(`Evaluating item ${i + 1}/${dataset.length}...`);
+    const extras = testCaseExtras.get(i);
     const result = {
       input: item.input,
       output: item.output,
       systemMessage: item.systemMessage,
+      expectedOutput: extras?.expectedOutput,
+      context: extras?.context,
+      metadata: item.metadata,
       model: "production",
       isProduction: true,
       reasoning: {}
@@ -1175,6 +1188,7 @@ async function compareModels(options) {
           input: item.input,
           output,
           systemMessage: item.systemMessage,
+          metadata: item.metadata,
           model: model.name,
           isProduction: false,
           reasoning: {},
@@ -1286,6 +1300,9 @@ async function uploadResults(results, name, description, judgeModel, verbose) {
     results: allResults.map((r) => ({
       input: r.input,
       system_message: r.systemMessage,
+      expected_output: r.expectedOutput,
+      context: r.context,
+      metadata: r.metadata,
       model: r.model,
       output: r.output,
       is_production: r.isProduction,
@@ -1395,7 +1412,7 @@ var import_exporter_trace_otlp_http = require("@opentelemetry/exporter-trace-otl
 // node_modules/@opentelemetry/resources/build/esm/Resource.js
 var import_api = require("@opentelemetry/api");
 
-// node_modules/@opentelemetry/resources/node_modules/@opentelemetry/semantic-conventions/build/esm/resource/SemanticResourceAttributes.js
+// node_modules/@opentelemetry/semantic-conventions/build/esm/resource/SemanticResourceAttributes.js
 var SemanticResourceAttributes = {
   /**
    * Name of the cloud provider.
@@ -2842,6 +2859,101 @@ function wrapGoogleAI(model, sessionCtx) {
   return model;
 }
 
+// src/trace/wrappers/vercel-ai/utils.ts
+function extractProviderInfo(model, aiModule, result) {
+  const info = {};
+  try {
+    if (aiModule) {
+      info.aiSdkVersion = aiModule.version ?? aiModule.VERSION ?? void 0;
+    }
+    if (!model) return info;
+    info.modelId = model.modelId ?? model.id ?? String(model);
+    if (model.provider) {
+      if (typeof model.provider === "string") {
+        info.provider = model.provider;
+      } else if (typeof model.provider === "object") {
+        info.provider = model.provider.id ?? model.provider.name;
+        info.providerId = model.provider.id;
+      }
+    }
+    if (model.providerId) {
+      info.providerId = model.providerId;
+    }
+    const baseUrl4 = model.config?.baseURL ?? model.config?.baseUrl ?? model.settings?.baseURL ?? model.settings?.baseUrl ?? model.baseURL ?? model.baseUrl;
+    if (baseUrl4 && typeof baseUrl4 === "string") {
+      info.baseUrl = baseUrl4;
+      if (!info.provider) {
+        if (baseUrl4.includes("openrouter.ai")) {
+          info.provider = "openrouter";
+        } else if (baseUrl4.includes("api.openai.com")) {
+          info.provider = "openai";
+        } else if (baseUrl4.includes("api.anthropic.com")) {
+          info.provider = "anthropic";
+        } else if (baseUrl4.includes("generativelanguage.googleapis.com")) {
+          info.provider = "google";
+        } else if (baseUrl4.includes("api.mistral.ai")) {
+          info.provider = "mistral";
+        } else if (baseUrl4.includes("api.together.xyz")) {
+          info.provider = "together";
+        } else if (baseUrl4.includes("api.groq.com")) {
+          info.provider = "groq";
+        } else if (baseUrl4.includes("localhost") || baseUrl4.includes("127.0.0.1")) {
+          info.provider = "local";
+        }
+      }
+    }
+    if (!info.provider && info.modelId) {
+      const modelStr = String(info.modelId).toLowerCase();
+      if (modelStr.includes("gpt-") || modelStr.includes("o1-") || modelStr.includes("text-embedding")) {
+        info.provider = info.provider ?? "openai";
+      } else if (modelStr.includes("claude-")) {
+        info.provider = info.provider ?? "anthropic";
+      } else if (modelStr.includes("gemini-") || modelStr.includes("gemma-")) {
+        info.provider = info.provider ?? "google";
+      } else if (modelStr.includes("mistral-") || modelStr.includes("mixtral-")) {
+        info.provider = info.provider ?? "mistral";
+      } else if (modelStr.includes("llama-") || modelStr.includes("meta-llama")) {
+        info.provider = info.provider ?? "meta";
+      } else if (modelStr.includes("/")) {
+        info.provider = info.provider ?? "openrouter";
+      }
+    }
+    if (result?.response) {
+      if (!info.modelId && result.response.modelId) {
+        info.modelId = result.response.modelId;
+      }
+    }
+    info.raw = {};
+    if (model.modelId) info.raw.modelId = model.modelId;
+    if (model.provider) {
+      info.raw.provider = typeof model.provider === "object" ? { id: model.provider.id, name: model.provider.name } : model.provider;
+    }
+    if (model.providerId) info.raw.providerId = model.providerId;
+    if (model.specificationVersion) info.raw.specificationVersion = model.specificationVersion;
+  } catch {
+  }
+  return info;
+}
+function providerInfoToAttributes(info) {
+  const attrs = {};
+  if (info.provider) {
+    attrs["fallom.provider"] = info.provider;
+  }
+  if (info.providerId) {
+    attrs["fallom.provider_id"] = info.providerId;
+  }
+  if (info.baseUrl) {
+    attrs["fallom.base_url"] = info.baseUrl;
+  }
+  if (info.aiSdkVersion) {
+    attrs["fallom.ai_sdk_version"] = info.aiSdkVersion;
+  }
+  if (info.raw && Object.keys(info.raw).length > 0) {
+    attrs["fallom.provider_raw"] = JSON.stringify(info.raw);
+  }
+  return attrs;
+}
+
 // src/trace/wrappers/vercel-ai/generate-text.ts
 function createGenerateTextWrapper(aiModule, sessionCtx, debug = false) {
   const ctx = sessionCtx;
@@ -2865,34 +2977,44 @@ function createGenerateTextWrapper(aiModule, sessionCtx, debug = false) {
       )) {
         if (tool && typeof tool.execute === "function") {
           const originalExecute = tool.execute;
-          wrappedTools[toolName] = {
-            ...tool,
-            execute: async (...executeArgs) => {
-              const toolStartTime = Date.now();
-              const toolCallId = `${toolName}-${toolStartTime}`;
-              try {
-                const result = await originalExecute(...executeArgs);
-                const toolEndTime = Date.now();
-                toolTimings.set(toolCallId, {
-                  name: toolName,
-                  startTime: toolStartTime - startTime,
-                  // Relative to request start
-                  endTime: toolEndTime - startTime,
-                  duration: toolEndTime - toolStartTime
-                });
-                return result;
-              } catch (error) {
-                const toolEndTime = Date.now();
-                toolTimings.set(toolCallId, {
-                  name: toolName,
-                  startTime: toolStartTime - startTime,
-                  endTime: toolEndTime - startTime,
-                  duration: toolEndTime - toolStartTime
-                });
-                throw error;
-              }
+          const wrappedTool = Object.create(Object.getPrototypeOf(tool) || {});
+          const allKeys = [
+            ...Object.getOwnPropertyNames(tool),
+            ...Object.getOwnPropertySymbols(tool)
+          ];
+          for (const key of allKeys) {
+            if (key === "execute") continue;
+            const descriptor = Object.getOwnPropertyDescriptor(tool, key);
+            if (descriptor) {
+              Object.defineProperty(wrappedTool, key, descriptor);
+            }
+          }
+          wrappedTool.execute = async (...executeArgs) => {
+            const toolStartTime = Date.now();
+            const toolCallId = `${toolName}-${toolStartTime}`;
+            try {
+              const result = await originalExecute(...executeArgs);
+              const toolEndTime = Date.now();
+              toolTimings.set(toolCallId, {
+                name: toolName,
+                startTime: toolStartTime - startTime,
+                // Relative to request start
+                endTime: toolEndTime - startTime,
+                duration: toolEndTime - toolStartTime
+              });
+              return result;
+            } catch (error) {
+              const toolEndTime = Date.now();
+              toolTimings.set(toolCallId, {
+                name: toolName,
+                startTime: toolStartTime - startTime,
+                endTime: toolEndTime - startTime,
+                duration: toolEndTime - toolStartTime
+              });
+              throw error;
             }
           };
+          wrappedTools[toolName] = wrappedTool;
         } else {
           wrappedTools[toolName] = tool;
         }
@@ -2908,9 +3030,12 @@ function createGenerateTextWrapper(aiModule, sessionCtx, debug = false) {
         console.log("   steps:", result?.steps?.length || 0);
       }
       const modelId = result?.response?.modelId || params?.model?.modelId || String(params?.model || "unknown");
+      const providerInfo = extractProviderInfo(params?.model, aiModule, result);
       const attributes = {
         "fallom.sdk_version": "2",
-        "fallom.method": "generateText"
+        "fallom.method": "generateText",
+        // Provider info for debugging
+        ...providerInfoToAttributes(providerInfo)
       };
       if (captureContent2) {
         attributes["fallom.raw.request"] = JSON.stringify({
@@ -2924,7 +3049,17 @@ function createGenerateTextWrapper(aiModule, sessionCtx, debug = false) {
         const mapToolCall = (tc) => {
           let args2 = tc?.args ?? tc?.input;
           if (args2 === void 0 && tc) {
-            const { type, toolCallId, toolName, providerExecuted, dynamic, invalid, error, providerMetadata, ...rest } = tc;
+            const {
+              type,
+              toolCallId,
+              toolName,
+              providerExecuted,
+              dynamic,
+              invalid,
+              error,
+              providerMetadata,
+              ...rest
+            } = tc;
             if (Object.keys(rest).length > 0) {
               args2 = rest;
             }
@@ -3168,33 +3303,43 @@ function createStreamTextWrapper(aiModule, sessionCtx, debug = false) {
       )) {
         if (tool && typeof tool.execute === "function") {
           const originalExecute = tool.execute;
-          wrappedTools[toolName] = {
-            ...tool,
-            execute: async (...executeArgs) => {
-              const toolStartTime = Date.now();
-              const toolCallId = `${toolName}-${toolStartTime}`;
-              try {
-                const result2 = await originalExecute(...executeArgs);
-                const toolEndTime = Date.now();
-                toolTimings.set(toolCallId, {
-                  name: toolName,
-                  startTime: toolStartTime - startTime,
-                  endTime: toolEndTime - startTime,
-                  duration: toolEndTime - toolStartTime
-                });
-                return result2;
-              } catch (error) {
-                const toolEndTime = Date.now();
-                toolTimings.set(toolCallId, {
-                  name: toolName,
-                  startTime: toolStartTime - startTime,
-                  endTime: toolEndTime - startTime,
-                  duration: toolEndTime - toolStartTime
-                });
-                throw error;
-              }
+          const wrappedTool = Object.create(Object.getPrototypeOf(tool) || {});
+          const allKeys = [
+            ...Object.getOwnPropertyNames(tool),
+            ...Object.getOwnPropertySymbols(tool)
+          ];
+          for (const key of allKeys) {
+            if (key === "execute") continue;
+            const descriptor = Object.getOwnPropertyDescriptor(tool, key);
+            if (descriptor) {
+              Object.defineProperty(wrappedTool, key, descriptor);
+            }
+          }
+          wrappedTool.execute = async (...executeArgs) => {
+            const toolStartTime = Date.now();
+            const toolCallId = `${toolName}-${toolStartTime}`;
+            try {
+              const result2 = await originalExecute(...executeArgs);
+              const toolEndTime = Date.now();
+              toolTimings.set(toolCallId, {
+                name: toolName,
+                startTime: toolStartTime - startTime,
+                endTime: toolEndTime - startTime,
+                duration: toolEndTime - toolStartTime
+              });
+              return result2;
+            } catch (error) {
+              const toolEndTime = Date.now();
+              toolTimings.set(toolCallId, {
+                name: toolName,
+                startTime: toolStartTime - startTime,
+                endTime: toolEndTime - startTime,
+                duration: toolEndTime - toolStartTime
+              });
+              throw error;
             }
           };
+          wrappedTools[toolName] = wrappedTool;
         } else {
           wrappedTools[toolName] = tool;
         }
@@ -3305,10 +3450,13 @@ function createStreamTextWrapper(aiModule, sessionCtx, debug = false) {
               providerMetadata = void 0;
             }
           }
+          const providerInfo = extractProviderInfo(params?.model, aiModule, result);
           const attributes = {
             "fallom.sdk_version": "2",
             "fallom.method": "streamText",
-            "fallom.is_streaming": true
+            "fallom.is_streaming": true,
+            // Provider info for debugging
+            ...providerInfoToAttributes(providerInfo)
           };
           if (captureContent2) {
             const mapToolCall = (tc) => {
@@ -3579,9 +3727,12 @@ function createGenerateObjectWrapper(aiModule, sessionCtx, debug = false) {
         );
       }
       const modelId = result?.response?.modelId || params?.model?.modelId || String(params?.model || "unknown");
+      const providerInfo = extractProviderInfo(params?.model, aiModule, result);
       const attributes = {
         "fallom.sdk_version": "2",
-        "fallom.method": "generateObject"
+        "fallom.method": "generateObject",
+        // Provider info for debugging
+        ...providerInfoToAttributes(providerInfo)
       };
       if (captureContent2) {
         attributes["fallom.raw.request"] = JSON.stringify({
@@ -3704,10 +3855,13 @@ function createStreamObjectWrapper(aiModule, sessionCtx, debug = false) {
             providerMetadata = void 0;
           }
         }
+        const providerInfo = extractProviderInfo(params?.model, aiModule, result);
         const attributes = {
           "fallom.sdk_version": "2",
           "fallom.method": "streamObject",
-          "fallom.is_streaming": true
+          "fallom.is_streaming": true,
+          // Provider info for debugging
+          ...providerInfoToAttributes(providerInfo)
         };
         if (captureContent2) {
           attributes["fallom.raw.request"] = JSON.stringify({