@fallom/trace 0.2.14 → 0.2.16

package/dist/index.js

@@ -590,9 +590,159 @@ async function datasetFromFallom(datasetKey, version, config) {
   );
   return items;
 }
+var EvaluationDataset;
 var init_helpers = __esm({
   "src/evals/helpers.ts"() {
     "use strict";
+    EvaluationDataset = class {
+      constructor() {
+        this._goldens = [];
+        this._testCases = [];
+        this._datasetKey = null;
+        this._datasetName = null;
+        this._version = null;
+      }
+      /** List of golden records (inputs with optional expected outputs). */
+      get goldens() {
+        return this._goldens;
+      }
+      /** List of test cases (inputs with actual outputs from your LLM). */
+      get testCases() {
+        return this._testCases;
+      }
+      /** The Fallom dataset key if pulled from Fallom. */
+      get datasetKey() {
+        return this._datasetKey;
+      }
+      /**
+       * Pull a dataset from Fallom.
+       *
+       * @param alias - The dataset key/alias in Fallom
+       * @param version - Specific version to pull (default: latest)
+       * @returns Self for chaining
+       */
+      async pull(alias, version) {
+        const { _apiKey: _apiKey2, _baseUrl: _baseUrl2, _initialized: _initialized2 } = await Promise.resolve().then(() => (init_core(), core_exports));
+        if (!_initialized2) {
+          throw new Error("Fallom evals not initialized. Call evals.init() first.");
+        }
+        const params = new URLSearchParams({ include_entries: "true" });
+        if (version !== void 0) {
+          params.set("version", String(version));
+        }
+        const url = `${_baseUrl2}/api/datasets/${encodeURIComponent(alias)}?${params}`;
+        const response = await fetch(url, {
+          headers: {
+            Authorization: `Bearer ${_apiKey2}`,
+            "Content-Type": "application/json"
+          }
+        });
+        if (response.status === 404) {
+          throw new Error(`Dataset '${alias}' not found`);
+        } else if (response.status === 403) {
+          throw new Error(`Access denied to dataset '${alias}'`);
+        }
+        if (!response.ok) {
+          throw new Error(`Failed to fetch dataset: ${response.statusText}`);
+        }
+        const data = await response.json();
+        this._datasetKey = alias;
+        this._datasetName = data.dataset?.name || alias;
+        this._version = data.version?.version || null;
+        this._goldens = [];
+        for (const entry of data.entries || []) {
+          this._goldens.push({
+            input: entry.input || "",
+            expectedOutput: entry.output,
+            systemMessage: entry.systemMessage,
+            metadata: entry.metadata
+          });
+        }
+        console.log(
+          `\u2713 Pulled dataset '${this._datasetName}' (version ${this._version}) with ${this._goldens.length} goldens`
+        );
+        return this;
+      }
+      /**
+       * Add a golden record manually.
+       * @param golden - A Golden object
+       * @returns Self for chaining
+       */
+      addGolden(golden) {
+        this._goldens.push(golden);
+        return this;
+      }
+      /**
+       * Add multiple golden records.
+       * @param goldens - Array of Golden objects
+       * @returns Self for chaining
+       */
+      addGoldens(goldens) {
+        this._goldens.push(...goldens);
+        return this;
+      }
+      /**
+       * Add a test case with actual LLM output.
+       * @param testCase - An LLMTestCase object
+       * @returns Self for chaining
+       */
+      addTestCase(testCase) {
+        this._testCases.push(testCase);
+        return this;
+      }
+      /**
+       * Add multiple test cases.
+       * @param testCases - Array of LLMTestCase objects
+       * @returns Self for chaining
+       */
+      addTestCases(testCases) {
+        this._testCases.push(...testCases);
+        return this;
+      }
+      /**
+       * Automatically generate test cases by running all goldens through your LLM app.
+       *
+       * @param llmApp - A callable that takes messages and returns response
+       * @param options - Configuration options
+       * @returns Self for chaining
+       */
+      async generateTestCases(llmApp, options = {}) {
+        const { includeContext = false } = options;
+        console.log(`Generating test cases for ${this._goldens.length} goldens...`);
+        for (let i = 0; i < this._goldens.length; i++) {
+          const golden = this._goldens[i];
+          const messages = [];
+          if (golden.systemMessage) {
+            messages.push({ role: "system", content: golden.systemMessage });
+          }
+          messages.push({ role: "user", content: golden.input });
+          const response = await llmApp(messages);
+          const testCase = {
+            input: golden.input,
+            actualOutput: response.content,
+            expectedOutput: golden.expectedOutput,
+            systemMessage: golden.systemMessage,
+            context: includeContext ? response.context : golden.context,
+            metadata: golden.metadata
+          };
+          this._testCases.push(testCase);
+          console.log(
+            `  [${i + 1}/${this._goldens.length}] Generated output for: ${golden.input.slice(0, 50)}...`
+          );
+        }
+        console.log(`\u2713 Generated ${this._testCases.length} test cases`);
+        return this;
+      }
+      /** Clear all test cases (useful for re-running with different LLM). */
+      clearTestCases() {
+        this._testCases = [];
+        return this;
+      }
+      /** Return the number of goldens. */
+      get length() {
+        return this._goldens.length;
+      }
+    };
   }
 });
 
@@ -707,9 +857,22 @@ async function evaluate(options) {
     name,
     description,
     verbose = true,
+    testCases,
     _skipUpload = false
   } = options;
-  const dataset = await resolveDataset(datasetInput);
+  let dataset;
+  if (testCases !== void 0 && testCases.length > 0) {
+    dataset = testCases.map((tc) => ({
+      input: tc.input,
+      output: tc.actualOutput,
+      systemMessage: tc.systemMessage,
+      metadata: tc.metadata
+    }));
+  } else if (datasetInput !== void 0) {
+    dataset = await resolveDataset(datasetInput);
+  } else {
+    throw new Error("Either 'dataset' or 'testCases' must be provided");
+  }
   for (const m of metrics) {
     if (typeof m === "string" && !AVAILABLE_METRICS.includes(m)) {
       throw new Error(
@@ -775,6 +938,9 @@ async function compareModels(options) {
     description,
     verbose = true
   } = options;
+  if (!datasetInput) {
+    throw new Error("'dataset' is required for compareModels()");
+  }
   const dataset = await resolveDataset(datasetInput);
   const results = {};
   if (includeProduction) {
@@ -1035,7 +1201,7 @@ var import_exporter_trace_otlp_http = require("@opentelemetry/exporter-trace-otl
 // node_modules/@opentelemetry/resources/build/esm/Resource.js
 var import_api = require("@opentelemetry/api");
 
-// node_modules/@opentelemetry/resources/node_modules/@opentelemetry/semantic-conventions/build/esm/resource/SemanticResourceAttributes.js
+// node_modules/@opentelemetry/semantic-conventions/build/esm/resource/SemanticResourceAttributes.js
 var SemanticResourceAttributes = {
   /**
    * Name of the cloud provider.
@@ -2474,10 +2640,9 @@ function createGenerateTextWrapper(aiModule, sessionCtx, debug = false) {
       const result = await aiModule.generateText(wrappedParams);
       const endTime = Date.now();
       if (debug || isDebugMode()) {
-        console.log(
-          "\n\u{1F50D} [Fallom Debug] generateText raw result:",
-          JSON.stringify(result, null, 2)
-        );
+        console.log("\n\u{1F50D} [Fallom Debug] generateText result:");
+        console.log("   toolCalls:", result?.toolCalls?.length || 0);
+        console.log("   steps:", result?.steps?.length || 0);
       }
       const modelId = result?.response?.modelId || params?.model?.modelId || String(params?.model || "unknown");
       const attributes = {
@@ -2496,15 +2661,15 @@ function createGenerateTextWrapper(aiModule, sessionCtx, debug = false) {
         const mapToolCall = (tc) => ({
           toolCallId: tc?.toolCallId,
           toolName: tc?.toolName,
-          args: tc?.args,
-          // The actual arguments passed to the tool!
+          args: tc?.args ?? tc?.input,
+          // v4: args, v5: input
           type: tc?.type
         });
         const mapToolResult = (tr) => ({
           toolCallId: tr?.toolCallId,
           toolName: tr?.toolName,
-          result: tr?.result,
-          // The actual result from the tool!
+          result: tr?.result ?? tr?.output,
+          // v4: result, v5: output
           type: tr?.type
         });
         attributes["fallom.raw.response"] = JSON.stringify({
@@ -2817,15 +2982,15 @@ function createStreamTextWrapper(aiModule, sessionCtx, debug = false) {
             const mapToolCall = (tc) => ({
               toolCallId: tc?.toolCallId,
               toolName: tc?.toolName,
-              args: tc?.args,
-              // The actual arguments passed to the tool!
+              args: tc?.args ?? tc?.input,
+              // v4: args, v5: input
               type: tc?.type
             });
             const mapToolResult = (tr) => ({
               toolCallId: tr?.toolCallId,
               toolName: tr?.toolName,
-              result: tr?.result,
-              // The actual result from the tool!
+              result: tr?.result ?? tr?.output,
+              // v4: result, v5: output
               type: tr?.type
             });
             attributes["fallom.raw.request"] = JSON.stringify({
@@ -3544,6 +3709,7 @@ var evals_exports = {};
 __export(evals_exports, {
   AVAILABLE_METRICS: () => AVAILABLE_METRICS,
   DEFAULT_JUDGE_MODEL: () => DEFAULT_JUDGE_MODEL,
+  EvaluationDataset: () => EvaluationDataset,
   METRIC_PROMPTS: () => METRIC_PROMPTS,
   compareModels: () => compareModels,
   createCustomModel: () => createCustomModel,

package/dist/index.mjs

@@ -5,6 +5,7 @@ import {
 import {
   AVAILABLE_METRICS,
   DEFAULT_JUDGE_MODEL,
+  EvaluationDataset,
   METRIC_PROMPTS,
   compareModels,
   createCustomModel,
@@ -18,7 +19,7 @@ import {
   init as init2,
   isCustomMetric,
   uploadResultsPublic
-} from "./chunk-2NGJF2JZ.mjs";
+} from "./chunk-3HBKT4HK.mjs";
 import {
   __export
 } from "./chunk-7P6ASYW6.mjs";
@@ -40,7 +41,7 @@ import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
 // node_modules/@opentelemetry/resources/build/esm/Resource.js
 import { diag } from "@opentelemetry/api";
 
-// node_modules/@opentelemetry/resources/node_modules/@opentelemetry/semantic-conventions/build/esm/resource/SemanticResourceAttributes.js
+// node_modules/@opentelemetry/semantic-conventions/build/esm/resource/SemanticResourceAttributes.js
 var SemanticResourceAttributes = {
   /**
    * Name of the cloud provider.
@@ -1479,10 +1480,9 @@ function createGenerateTextWrapper(aiModule, sessionCtx, debug = false) {
       const result = await aiModule.generateText(wrappedParams);
       const endTime = Date.now();
       if (debug || isDebugMode()) {
-        console.log(
-          "\n\u{1F50D} [Fallom Debug] generateText raw result:",
-          JSON.stringify(result, null, 2)
-        );
+        console.log("\n\u{1F50D} [Fallom Debug] generateText result:");
+        console.log("   toolCalls:", result?.toolCalls?.length || 0);
+        console.log("   steps:", result?.steps?.length || 0);
       }
       const modelId = result?.response?.modelId || params?.model?.modelId || String(params?.model || "unknown");
       const attributes = {
@@ -1501,15 +1501,15 @@ function createGenerateTextWrapper(aiModule, sessionCtx, debug = false) {
         const mapToolCall = (tc) => ({
           toolCallId: tc?.toolCallId,
           toolName: tc?.toolName,
-          args: tc?.args,
-          // The actual arguments passed to the tool!
+          args: tc?.args ?? tc?.input,
+          // v4: args, v5: input
           type: tc?.type
         });
         const mapToolResult = (tr) => ({
           toolCallId: tr?.toolCallId,
           toolName: tr?.toolName,
-          result: tr?.result,
-          // The actual result from the tool!
+          result: tr?.result ?? tr?.output,
+          // v4: result, v5: output
           type: tr?.type
         });
         attributes["fallom.raw.response"] = JSON.stringify({
@@ -1822,15 +1822,15 @@ function createStreamTextWrapper(aiModule, sessionCtx, debug = false) {
             const mapToolCall = (tc) => ({
               toolCallId: tc?.toolCallId,
               toolName: tc?.toolName,
-              args: tc?.args,
-              // The actual arguments passed to the tool!
+              args: tc?.args ?? tc?.input,
+              // v4: args, v5: input
               type: tc?.type
             });
             const mapToolResult = (tr) => ({
               toolCallId: tr?.toolCallId,
               toolName: tr?.toolName,
-              result: tr?.result,
-              // The actual result from the tool!
+              result: tr?.result ?? tr?.output,
+              // v4: result, v5: output
               type: tr?.type
             });
             attributes["fallom.raw.request"] = JSON.stringify({
@@ -2546,6 +2546,7 @@ var evals_exports = {};
 __export(evals_exports, {
   AVAILABLE_METRICS: () => AVAILABLE_METRICS,
   DEFAULT_JUDGE_MODEL: () => DEFAULT_JUDGE_MODEL,
+  EvaluationDataset: () => EvaluationDataset,
   METRIC_PROMPTS: () => METRIC_PROMPTS,
   compareModels: () => compareModels,
   createCustomModel: () => createCustomModel,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fallom/trace",
-  "version": "0.2.14",
+  "version": "0.2.16",
   "description": "Model A/B testing and tracing for LLM applications. Zero latency, production-ready.",
   "main": "./dist/index.js",
   "module": "./dist/index.mjs",

package/dist/chunk-KFD5AQ7V.mjs

@@ -1,308 +0,0 @@
-var __defProp = Object.defineProperty;
-var __export = (target, all) => {
-  for (var name in all)
-    __defProp(target, name, { get: all[name], enumerable: true });
-};
-
-// src/models.ts
-var models_exports = {};
-__export(models_exports, {
-  get: () => get,
-  init: () => init
-});
-import { createHash } from "crypto";
-var apiKey = null;
-var baseUrl = "https://configs.fallom.com";
-var initialized = false;
-var syncInterval = null;
-var debugMode = false;
-var configCache = /* @__PURE__ */ new Map();
-var SYNC_TIMEOUT = 2e3;
-var RECORD_TIMEOUT = 1e3;
-function log(msg) {
-  if (debugMode) {
-    console.log(`[Fallom] ${msg}`);
-  }
-}
-function evaluateTargeting(targeting, customerId, context) {
-  if (!targeting || targeting.enabled === false) {
-    return null;
-  }
-  const evalContext = {
-    ...context || {},
-    ...customerId ? { customerId } : {}
-  };
-  log(`Evaluating targeting with context: ${JSON.stringify(evalContext)}`);
-  if (targeting.individualTargets) {
-    for (const target of targeting.individualTargets) {
-      const fieldValue = evalContext[target.field];
-      if (fieldValue === target.value) {
-        log(`Individual target matched: ${target.field}=${target.value} -> variant ${target.variantIndex}`);
-        return target.variantIndex;
-      }
-    }
-  }
-  if (targeting.rules) {
-    for (const rule of targeting.rules) {
-      const allConditionsMatch = rule.conditions.every((condition) => {
-        const fieldValue = evalContext[condition.field];
-        if (fieldValue === void 0) return false;
-        switch (condition.operator) {
-          case "eq":
-            return fieldValue === condition.value;
-          case "neq":
-            return fieldValue !== condition.value;
-          case "in":
-            return Array.isArray(condition.value) && condition.value.includes(fieldValue);
-          case "nin":
-            return Array.isArray(condition.value) && !condition.value.includes(fieldValue);
-          case "contains":
-            return typeof condition.value === "string" && fieldValue.includes(condition.value);
-          case "startsWith":
-            return typeof condition.value === "string" && fieldValue.startsWith(condition.value);
-          case "endsWith":
-            return typeof condition.value === "string" && fieldValue.endsWith(condition.value);
-          default:
-            return false;
-        }
-      });
-      if (allConditionsMatch) {
-        log(`Rule matched: ${JSON.stringify(rule.conditions)} -> variant ${rule.variantIndex}`);
-        return rule.variantIndex;
-      }
-    }
-  }
-  log("No targeting rules matched, falling back to weighted random");
-  return null;
-}
-function init(options = {}) {
-  apiKey = options.apiKey || process.env.FALLOM_API_KEY || null;
-  baseUrl = options.baseUrl || process.env.FALLOM_CONFIGS_URL || process.env.FALLOM_BASE_URL || "https://configs.fallom.com";
-  initialized = true;
-  if (!apiKey) {
-    return;
-  }
-  fetchConfigs().catch(() => {
-  });
-  if (!syncInterval) {
-    syncInterval = setInterval(() => {
-      fetchConfigs().catch(() => {
-      });
-    }, 3e4);
-    syncInterval.unref();
-  }
-}
-function ensureInit() {
-  if (!initialized) {
-    try {
-      init();
-    } catch {
-    }
-  }
-}
-async function fetchConfigs(timeout = SYNC_TIMEOUT) {
-  if (!apiKey) {
-    log("_fetchConfigs: No API key, skipping");
-    return;
-  }
-  try {
-    log(`Fetching configs from ${baseUrl}/configs`);
-    const controller = new AbortController();
-    const timeoutId = setTimeout(() => controller.abort(), timeout);
-    const resp = await fetch(`${baseUrl}/configs`, {
-      headers: { Authorization: `Bearer ${apiKey}` },
-      signal: controller.signal
-    });
-    clearTimeout(timeoutId);
-    log(`Response status: ${resp.status}`);
-    if (resp.ok) {
-      const data = await resp.json();
-      const configs = data.configs || [];
-      log(`Got ${configs.length} configs: ${configs.map((c) => c.key)}`);
-      for (const c of configs) {
-        const key = c.key;
-        const version = c.version || 1;
-        log(`Config '${key}' v${version}: ${JSON.stringify(c.variants)}`);
-        if (!configCache.has(key)) {
-          configCache.set(key, { versions: /* @__PURE__ */ new Map(), latest: null });
-        }
-        const cached = configCache.get(key);
-        cached.versions.set(version, c);
-        cached.latest = version;
-      }
-    } else {
-      log(`Fetch failed: ${resp.statusText}`);
-    }
-  } catch (e) {
-    log(`Fetch exception: ${e}`);
-  }
-}
-async function fetchSpecificVersion(configKey, version, timeout = SYNC_TIMEOUT) {
-  if (!apiKey) return null;
-  try {
-    const controller = new AbortController();
-    const timeoutId = setTimeout(() => controller.abort(), timeout);
-    const resp = await fetch(
-      `${baseUrl}/configs/${configKey}/version/${version}`,
-      {
-        headers: { Authorization: `Bearer ${apiKey}` },
-        signal: controller.signal
-      }
-    );
-    clearTimeout(timeoutId);
-    if (resp.ok) {
-      const config = await resp.json();
-      if (!configCache.has(configKey)) {
-        configCache.set(configKey, { versions: /* @__PURE__ */ new Map(), latest: null });
-      }
-      configCache.get(configKey).versions.set(version, config);
-      return config;
-    }
-  } catch {
-  }
-  return null;
-}
-async function get(configKey, sessionId, options = {}) {
-  const { version, fallback, customerId, context, debug = false } = options;
-  debugMode = debug;
-  ensureInit();
-  log(
-    `get() called: configKey=${configKey}, sessionId=${sessionId}, fallback=${fallback}`
-  );
-  try {
-    let configData = configCache.get(configKey);
-    log(
-      `Cache lookup for '${configKey}': ${configData ? "found" : "not found"}`
-    );
-    if (!configData) {
-      log("Not in cache, fetching...");
-      await fetchConfigs(SYNC_TIMEOUT);
-      configData = configCache.get(configKey);
-      log(
-        `After fetch, cache lookup: ${configData ? "found" : "still not found"}`
-      );
-    }
-    if (!configData) {
-      log(`Config not found, using fallback: ${fallback}`);
-      if (fallback) {
-        console.warn(
-          `[Fallom WARNING] Config '${configKey}' not found, using fallback model: ${fallback}`
-        );
-        return returnModel(configKey, sessionId, fallback, 0);
-      }
-      throw new Error(
-        `Config '${configKey}' not found. Check that it exists in your Fallom dashboard.`
-      );
-    }
-    let config;
-    let targetVersion;
-    if (version !== void 0) {
-      config = configData.versions.get(version);
-      if (!config) {
-        config = await fetchSpecificVersion(configKey, version, SYNC_TIMEOUT) || void 0;
-      }
-      if (!config) {
-        if (fallback) {
-          console.warn(
-            `[Fallom WARNING] Config '${configKey}' version ${version} not found, using fallback: ${fallback}`
-          );
-          return returnModel(configKey, sessionId, fallback, 0);
-        }
-        throw new Error(`Config '${configKey}' version ${version} not found.`);
-      }
-      targetVersion = version;
-    } else {
-      targetVersion = configData.latest;
-      config = configData.versions.get(targetVersion);
-      if (!config) {
-        if (fallback) {
-          console.warn(
-            `[Fallom WARNING] Config '${configKey}' has no cached version, using fallback: ${fallback}`
-          );
-          return returnModel(configKey, sessionId, fallback, 0);
-        }
-        throw new Error(`Config '${configKey}' has no cached version.`);
-      }
-    }
-    const variantsRaw = config.variants;
-    const configVersion = config.version || targetVersion;
-    const variants = Array.isArray(variantsRaw) ? variantsRaw : Object.values(variantsRaw);
-    log(
-      `Config found! Version: ${configVersion}, Variants: ${JSON.stringify(
-        variants
-      )}`
-    );
-    const targetedVariantIndex = evaluateTargeting(config.targeting, customerId, context);
-    if (targetedVariantIndex !== null && variants[targetedVariantIndex]) {
-      const assignedModel2 = variants[targetedVariantIndex].model;
-      log(`\u2705 Assigned model via targeting: ${assignedModel2}`);
-      return returnModel(configKey, sessionId, assignedModel2, configVersion);
-    }
-    const hashBytes = createHash("md5").update(sessionId).digest();
-    const hashVal = hashBytes.readUInt32BE(0) % 1e6;
-    log(`Session hash: ${hashVal} (out of 1,000,000)`);
-    let cumulative = 0;
-    let assignedModel = variants[variants.length - 1].model;
-    for (const v of variants) {
-      const oldCumulative = cumulative;
-      cumulative += v.weight * 1e4;
-      log(
-        `Variant ${v.model}: weight=${v.weight}%, range=${oldCumulative}-${cumulative}, hash=${hashVal}, match=${hashVal < cumulative}`
-      );
-      if (hashVal < cumulative) {
-        assignedModel = v.model;
-        break;
-      }
-    }
-    log(`\u2705 Assigned model via weighted random: ${assignedModel}`);
-    return returnModel(configKey, sessionId, assignedModel, configVersion);
-  } catch (e) {
-    if (e instanceof Error && e.message.includes("not found")) {
-      throw e;
-    }
-    if (fallback) {
-      console.warn(
-        `[Fallom WARNING] Error getting model for '${configKey}': ${e}. Using fallback: ${fallback}`
-      );
-      return returnModel(configKey, sessionId, fallback, 0);
-    }
-    throw e;
-  }
-}
-function returnModel(configKey, sessionId, model, version) {
-  if (version > 0) {
-    recordSession(configKey, version, sessionId, model).catch(() => {
-    });
-  }
-  return model;
-}
-async function recordSession(configKey, version, sessionId, model) {
-  if (!apiKey) return;
-  try {
-    const controller = new AbortController();
-    const timeoutId = setTimeout(() => controller.abort(), RECORD_TIMEOUT);
-    await fetch(`${baseUrl}/sessions`, {
-      method: "POST",
-      headers: {
-        Authorization: `Bearer ${apiKey}`,
-        "Content-Type": "application/json"
-      },
-      body: JSON.stringify({
-        config_key: configKey,
-        config_version: version,
-        session_id: sessionId,
-        assigned_model: model
-      }),
-      signal: controller.signal
-    });
-    clearTimeout(timeoutId);
-  } catch {
-  }
-}
-
-export {
-  __export,
-  init,
-  get,
-  models_exports
-};

package/dist/models-SEFDGZU2.mjs

@@ -1,8 +0,0 @@
-import {
-  get,
-  init
-} from "./chunk-KFD5AQ7V.mjs";
-export {
-  get,
-  init
-};