@fallom/trace 0.2.22 → 0.2.23

package/dist/chunk-KFD5AQ7V.mjs

@@ -0,0 +1,308 @@
+var __defProp = Object.defineProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+
+// src/models.ts
+var models_exports = {};
+__export(models_exports, {
+  get: () => get,
+  init: () => init
+});
+import { createHash } from "crypto";
+var apiKey = null;
+var baseUrl = "https://configs.fallom.com";
+var initialized = false;
+var syncInterval = null;
+var debugMode = false;
+var configCache = /* @__PURE__ */ new Map();
+var SYNC_TIMEOUT = 2e3;
+var RECORD_TIMEOUT = 1e3;
+function log(msg) {
+  if (debugMode) {
+    console.log(`[Fallom] ${msg}`);
+  }
+}
+function evaluateTargeting(targeting, customerId, context) {
+  if (!targeting || targeting.enabled === false) {
+    return null;
+  }
+  const evalContext = {
+    ...context || {},
+    ...customerId ? { customerId } : {}
+  };
+  log(`Evaluating targeting with context: ${JSON.stringify(evalContext)}`);
+  if (targeting.individualTargets) {
+    for (const target of targeting.individualTargets) {
+      const fieldValue = evalContext[target.field];
+      if (fieldValue === target.value) {
+        log(`Individual target matched: ${target.field}=${target.value} -> variant ${target.variantIndex}`);
+        return target.variantIndex;
+      }
+    }
+  }
+  if (targeting.rules) {
+    for (const rule of targeting.rules) {
+      const allConditionsMatch = rule.conditions.every((condition) => {
+        const fieldValue = evalContext[condition.field];
+        if (fieldValue === void 0) return false;
+        switch (condition.operator) {
+          case "eq":
+            return fieldValue === condition.value;
+          case "neq":
+            return fieldValue !== condition.value;
+          case "in":
+            return Array.isArray(condition.value) && condition.value.includes(fieldValue);
+          case "nin":
+            return Array.isArray(condition.value) && !condition.value.includes(fieldValue);
+          case "contains":
+            return typeof condition.value === "string" && fieldValue.includes(condition.value);
+          case "startsWith":
+            return typeof condition.value === "string" && fieldValue.startsWith(condition.value);
+          case "endsWith":
+            return typeof condition.value === "string" && fieldValue.endsWith(condition.value);
+          default:
+            return false;
+        }
+      });
+      if (allConditionsMatch) {
+        log(`Rule matched: ${JSON.stringify(rule.conditions)} -> variant ${rule.variantIndex}`);
+        return rule.variantIndex;
+      }
+    }
+  }
+  log("No targeting rules matched, falling back to weighted random");
+  return null;
+}
+function init(options = {}) {
+  apiKey = options.apiKey || process.env.FALLOM_API_KEY || null;
+  baseUrl = options.baseUrl || process.env.FALLOM_CONFIGS_URL || process.env.FALLOM_BASE_URL || "https://configs.fallom.com";
+  initialized = true;
+  if (!apiKey) {
+    return;
+  }
+  fetchConfigs().catch(() => {
+  });
+  if (!syncInterval) {
+    syncInterval = setInterval(() => {
+      fetchConfigs().catch(() => {
+      });
+    }, 3e4);
+    syncInterval.unref();
+  }
+}
+function ensureInit() {
+  if (!initialized) {
+    try {
+      init();
+    } catch {
+    }
+  }
+}
+async function fetchConfigs(timeout = SYNC_TIMEOUT) {
+  if (!apiKey) {
+    log("_fetchConfigs: No API key, skipping");
+    return;
+  }
+  try {
+    log(`Fetching configs from ${baseUrl}/configs`);
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), timeout);
+    const resp = await fetch(`${baseUrl}/configs`, {
+      headers: { Authorization: `Bearer ${apiKey}` },
+      signal: controller.signal
+    });
+    clearTimeout(timeoutId);
+    log(`Response status: ${resp.status}`);
+    if (resp.ok) {
+      const data = await resp.json();
+      const configs = data.configs || [];
+      log(`Got ${configs.length} configs: ${configs.map((c) => c.key)}`);
+      for (const c of configs) {
+        const key = c.key;
+        const version = c.version || 1;
+        log(`Config '${key}' v${version}: ${JSON.stringify(c.variants)}`);
+        if (!configCache.has(key)) {
+          configCache.set(key, { versions: /* @__PURE__ */ new Map(), latest: null });
+        }
+        const cached = configCache.get(key);
+        cached.versions.set(version, c);
+        cached.latest = version;
+      }
+    } else {
+      log(`Fetch failed: ${resp.statusText}`);
+    }
+  } catch (e) {
+    log(`Fetch exception: ${e}`);
+  }
+}
+async function fetchSpecificVersion(configKey, version, timeout = SYNC_TIMEOUT) {
+  if (!apiKey) return null;
+  try {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), timeout);
+    const resp = await fetch(
+      `${baseUrl}/configs/${configKey}/version/${version}`,
+      {
+        headers: { Authorization: `Bearer ${apiKey}` },
+        signal: controller.signal
+      }
+    );
+    clearTimeout(timeoutId);
+    if (resp.ok) {
+      const config = await resp.json();
+      if (!configCache.has(configKey)) {
+        configCache.set(configKey, { versions: /* @__PURE__ */ new Map(), latest: null });
+      }
+      configCache.get(configKey).versions.set(version, config);
+      return config;
+    }
+  } catch {
+  }
+  return null;
+}
+async function get(configKey, sessionId, options = {}) {
+  const { version, fallback, customerId, context, debug = false } = options;
+  debugMode = debug;
+  ensureInit();
+  log(
+    `get() called: configKey=${configKey}, sessionId=${sessionId}, fallback=${fallback}`
+  );
+  try {
+    let configData = configCache.get(configKey);
+    log(
+      `Cache lookup for '${configKey}': ${configData ? "found" : "not found"}`
+    );
+    if (!configData) {
+      log("Not in cache, fetching...");
+      await fetchConfigs(SYNC_TIMEOUT);
+      configData = configCache.get(configKey);
+      log(
+        `After fetch, cache lookup: ${configData ? "found" : "still not found"}`
+      );
+    }
+    if (!configData) {
+      log(`Config not found, using fallback: ${fallback}`);
+      if (fallback) {
+        console.warn(
+          `[Fallom WARNING] Config '${configKey}' not found, using fallback model: ${fallback}`
+        );
+        return returnModel(configKey, sessionId, fallback, 0);
+      }
+      throw new Error(
+        `Config '${configKey}' not found. Check that it exists in your Fallom dashboard.`
+      );
+    }
+    let config;
+    let targetVersion;
+    if (version !== void 0) {
+      config = configData.versions.get(version);
+      if (!config) {
+        config = await fetchSpecificVersion(configKey, version, SYNC_TIMEOUT) || void 0;
+      }
+      if (!config) {
+        if (fallback) {
+          console.warn(
+            `[Fallom WARNING] Config '${configKey}' version ${version} not found, using fallback: ${fallback}`
+          );
+          return returnModel(configKey, sessionId, fallback, 0);
+        }
+        throw new Error(`Config '${configKey}' version ${version} not found.`);
+      }
+      targetVersion = version;
+    } else {
+      targetVersion = configData.latest;
+      config = configData.versions.get(targetVersion);
+      if (!config) {
+        if (fallback) {
+          console.warn(
+            `[Fallom WARNING] Config '${configKey}' has no cached version, using fallback: ${fallback}`
+          );
+          return returnModel(configKey, sessionId, fallback, 0);
+        }
+        throw new Error(`Config '${configKey}' has no cached version.`);
+      }
+    }
+    const variantsRaw = config.variants;
+    const configVersion = config.version || targetVersion;
+    const variants = Array.isArray(variantsRaw) ? variantsRaw : Object.values(variantsRaw);
+    log(
+      `Config found! Version: ${configVersion}, Variants: ${JSON.stringify(
+        variants
+      )}`
+    );
+    const targetedVariantIndex = evaluateTargeting(config.targeting, customerId, context);
+    if (targetedVariantIndex !== null && variants[targetedVariantIndex]) {
+      const assignedModel2 = variants[targetedVariantIndex].model;
+      log(`\u2705 Assigned model via targeting: ${assignedModel2}`);
+      return returnModel(configKey, sessionId, assignedModel2, configVersion);
+    }
+    const hashBytes = createHash("md5").update(sessionId).digest();
+    const hashVal = hashBytes.readUInt32BE(0) % 1e6;
+    log(`Session hash: ${hashVal} (out of 1,000,000)`);
+    let cumulative = 0;
+    let assignedModel = variants[variants.length - 1].model;
+    for (const v of variants) {
+      const oldCumulative = cumulative;
+      cumulative += v.weight * 1e4;
+      log(
+        `Variant ${v.model}: weight=${v.weight}%, range=${oldCumulative}-${cumulative}, hash=${hashVal}, match=${hashVal < cumulative}`
+      );
+      if (hashVal < cumulative) {
+        assignedModel = v.model;
+        break;
+      }
+    }
+    log(`\u2705 Assigned model via weighted random: ${assignedModel}`);
+    return returnModel(configKey, sessionId, assignedModel, configVersion);
+  } catch (e) {
+    if (e instanceof Error && e.message.includes("not found")) {
+      throw e;
+    }
+    if (fallback) {
+      console.warn(
+        `[Fallom WARNING] Error getting model for '${configKey}': ${e}. Using fallback: ${fallback}`
+      );
+      return returnModel(configKey, sessionId, fallback, 0);
+    }
+    throw e;
+  }
+}
+function returnModel(configKey, sessionId, model, version) {
+  if (version > 0) {
+    recordSession(configKey, version, sessionId, model).catch(() => {
+    });
+  }
+  return model;
+}
+async function recordSession(configKey, version, sessionId, model) {
+  if (!apiKey) return;
+  try {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), RECORD_TIMEOUT);
+    await fetch(`${baseUrl}/sessions`, {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${apiKey}`,
+        "Content-Type": "application/json"
+      },
+      body: JSON.stringify({
+        config_key: configKey,
+        config_version: version,
+        session_id: sessionId,
+        assigned_model: model
+      }),
+      signal: controller.signal
+    });
+    clearTimeout(timeoutId);
+  } catch {
+  }
+}
+
+export {
+  __export,
+  init,
+  get,
+  models_exports
+};

package/dist/{chunk-GZ6TE7G4.mjs → chunk-NNVWIZN5.mjs}

@@ -120,7 +120,16 @@ Respond in JSON format:
   "score": 0.85
 }`;
 }
-async function runGEval(metric, inputText, outputText, systemMessage, judgeModel, openrouterKey) {
+async function runGEval(options) {
+  const {
+    metric,
+    inputText,
+    outputText,
+    systemMessage,
+    judgeModel,
+    openrouterKey,
+    fallomApiKey
+  } = options;
   const apiKey = openrouterKey || process.env.OPENROUTER_API_KEY;
   if (!apiKey) {
     throw new Error(
@@ -131,6 +140,7 @@ async function runGEval(metric, inputText, outputText, systemMessage, judgeModel
   if (!config) {
     throw new Error(`Unknown metric: ${metric}`);
   }
+  const metricName = typeof metric === "object" ? metric.name : metric;
   const prompt = buildGEvalPrompt(
     config.criteria,
     config.steps,
@@ -138,6 +148,7 @@ async function runGEval(metric, inputText, outputText, systemMessage, judgeModel
     inputText,
     outputText
   );
+  const startTime = Date.now();
   const response = await fetch(
     "https://openrouter.ai/api/v1/chat/completions",
     {
@@ -158,17 +169,89 @@ async function runGEval(metric, inputText, outputText, systemMessage, judgeModel
     throw new Error(`G-Eval API error: ${response.statusText}`);
   }
   const data = await response.json();
+  const endTime = Date.now();
   try {
     const result = JSON.parse(data.choices[0].message.content);
-    return {
-      score: Math.max(0, Math.min(1, result.score)),
-      // Clamp to 0-1
-      reasoning: result.overall_reasoning || ""
-    };
+    const score = Math.max(0, Math.min(1, result.score));
+    const reasoning = result.overall_reasoning || "";
+    if (fallomApiKey) {
+      sendGEvalTrace({
+        fallomApiKey,
+        metricName,
+        judgeModel,
+        prompt,
+        response: data.choices[0].message.content,
+        score,
+        reasoning,
+        startTime,
+        endTime,
+        usage: data.usage
+      }).catch(() => {
+      });
+    }
+    return { score, reasoning };
   } catch {
     throw new Error("Failed to parse G-Eval response");
   }
 }
+async function sendGEvalTrace(options) {
+  const {
+    fallomApiKey,
+    metricName,
+    judgeModel,
+    prompt,
+    response,
+    score,
+    reasoning,
+    startTime,
+    endTime,
+    usage
+  } = options;
+  const traceUrl = process.env.FALLOM_TRACES_URL || "https://traces.fallom.com";
+  const traceData = {
+    config_key: "eval-worker",
+    session_id: `geval-${Date.now()}`,
+    trace_id: generateHexId(32),
+    span_id: generateHexId(16),
+    name: `geval.${metricName}`,
+    kind: "llm",
+    model: judgeModel,
+    start_time: new Date(startTime).toISOString(),
+    end_time: new Date(endTime).toISOString(),
+    duration_ms: endTime - startTime,
+    status: "OK",
+    metadata: {
+      metric: metricName,
+      score
+    },
+    tags: ["eval-worker", "geval", metricName],
+    attributes: {
+      "fallom.sdk_version": "2",
+      "fallom.method": "runGEval",
+      "geval.metric": metricName,
+      "geval.score": score,
+      "geval.reasoning": reasoning,
+      "gen_ai.prompt.0.role": "user",
+      "gen_ai.prompt.0.content": prompt,
+      "gen_ai.completion.0.content": response,
+      "gen_ai.usage.prompt_tokens": usage?.prompt_tokens,
+      "gen_ai.usage.completion_tokens": usage?.completion_tokens
+    }
+  };
+  await fetch(`${traceUrl}/v1/traces`, {
+    method: "POST",
+    headers: {
+      Authorization: `Bearer ${fallomApiKey}`,
+      "Content-Type": "application/json"
+    },
+    body: JSON.stringify(traceData)
+  });
+}
+function generateHexId(length) {
+  const bytes = new Uint8Array(length / 2);
+  crypto.getRandomValues(bytes);
+  return Array.from(bytes).map((b) => b.toString(16).padStart(2, "0")).join("");
+}
 function calculateAggregateScores(results) {
   const aggregates = {};
   for (const result of results) {
@@ -333,7 +416,7 @@ function datasetFromTraces(traces) {
   return items;
 }
 async function datasetFromFallom(datasetKey, version, config) {
-  const { _apiKey: _apiKey2, _baseUrl: _baseUrl2, _initialized: _initialized2 } = await import("./core-DUG2SP2V.mjs").then(
+  const { _apiKey: _apiKey2, _baseUrl: _baseUrl2, _initialized: _initialized2 } = await import("./core-3MHBKYBC.mjs").then(
     (m) => ({
       _apiKey: config?._apiKey ?? m._apiKey,
       _baseUrl: config?._baseUrl ?? m._baseUrl,
@@ -406,7 +489,7 @@ var EvaluationDataset = class {
    * @returns Self for chaining
    */
   async pull(alias, version) {
-    const { _apiKey: _apiKey2, _baseUrl: _baseUrl2, _initialized: _initialized2 } = await import("./core-DUG2SP2V.mjs");
+    const { _apiKey: _apiKey2, _baseUrl: _baseUrl2, _initialized: _initialized2 } = await import("./core-3MHBKYBC.mjs");
     if (!_initialized2) {
       throw new Error("Fallom evals not initialized. Call evals.init() first.");
     }
@@ -545,7 +628,13 @@ function init(options = {}) {
 }
 async function runGEval2(metric, inputText, outputText, systemMessage, judgeModel) {
   const metricArg = isCustomMetric(metric) ? { name: metric.name, criteria: metric.criteria, steps: metric.steps } : metric;
-  return runGEval(metricArg, inputText, outputText, systemMessage, judgeModel);
+  return runGEval({
+    metric: metricArg,
+    inputText,
+    outputText,
+    systemMessage,
+    judgeModel
+  });
 }
 async function resolveDataset(datasetInput) {
   if (typeof datasetInput === "string") {
@@ -617,7 +706,9 @@ async function evaluate(options) {
   for (const m of metrics) {
     if (typeof m === "string" && !AVAILABLE_METRICS.includes(m)) {
       throw new Error(
-        `Invalid metric: ${m}. Available: ${AVAILABLE_METRICS.join(", ")}. Or use CustomMetric for custom metrics.`
+        `Invalid metric: ${m}. Available: ${AVAILABLE_METRICS.join(
+          ", "
+        )}. Or use CustomMetric for custom metrics.`
       );
     }
   }

package/dist/{core-DUG2SP2V.mjs → core-3MHBKYBC.mjs}

@@ -7,7 +7,7 @@ import {
   evaluate,
   init,
   uploadResultsPublic
-} from "./chunk-GZ6TE7G4.mjs";
+} from "./chunk-NNVWIZN5.mjs";
 import "./chunk-7P6ASYW6.mjs";
 export {
   DEFAULT_JUDGE_MODEL,

package/dist/index.d.mts

@@ -568,22 +568,36 @@ interface GEvalScore {
     score: number;
     reasoning: string;
 }
+/**
+ * Options for runGEval function.
+ */
+interface RunGEvalOptions {
+    /** Built-in metric name or custom metric config */
+    metric: string | {
+        name: string;
+        criteria: string;
+        steps: string[];
+    };
+    /** The user's input/query */
+    inputText: string;
+    /** The LLM's response to evaluate */
+    outputText: string;
+    /** Optional system message for context */
+    systemMessage?: string;
+    /** The model to use as judge (OpenRouter format, e.g., "openai/gpt-4o-mini") */
+    judgeModel: string;
+    /** OpenRouter API key (defaults to OPENROUTER_API_KEY env var) */
+    openrouterKey?: string;
+    /** Optional Fallom API key to enable tracing of the judge LLM call */
+    fallomApiKey?: string;
+}
 /**
  * Run G-Eval for a single metric using OpenRouter.
  * This is the low-level function used by both the SDK and backend workers.
  *
- * @param metric - Built-in metric name or custom metric config
- * @param inputText - The user's input/query
- * @param outputText - The LLM's response
- * @param systemMessage - Optional system message
- * @param judgeModel - The model to use as judge (OpenRouter format)
- * @param openrouterKey - OpenRouter API key (defaults to env var)
+ * If `fallomApiKey` is provided, the judge LLM call will be traced to Fallom.
  */
-declare function runGEval(metric: string | {
-    name: string;
-    criteria: string;
-    steps: string[];
-}, inputText: string, outputText: string, systemMessage: string | undefined, judgeModel: string, openrouterKey?: string): Promise<GEvalScore>;
+declare function runGEval(options: RunGEvalOptions): Promise<GEvalScore>;
 /**
  * Calculate aggregate scores from a list of results.
  */
@@ -614,12 +628,22 @@ declare function detectRegression(currentScores: Record<string, {
 };
 
 /**
- * Core evaluation functions.
+ * Core evaluation functions for Fallom Evals.
+ *
+ * Provides the main API for running LLM evaluations using G-Eval methodology.
  */
 
+/** Default judge model (via OpenRouter) */
 declare const DEFAULT_JUDGE_MODEL = "openai/gpt-4o-mini";
 /**
  * Initialize Fallom evals.
+ *
+ * @example
+ * ```typescript
+ * import fallom from "@fallom/trace";
+ *
+ * fallom.evals.init({ apiKey: "your-api-key" });
+ * ```
  */
 declare function init$1(options?: InitOptions$1): void;
 /**
@@ -627,6 +651,13 @@ declare function init$1(options?: InitOptions$1): void;
  *
  * Results are automatically uploaded to Fallom dashboard.
  *
+ * @example
+ * ```typescript
+ * const results = await fallom.evals.evaluate({
+ *   dataset: [{ input: "What is 2+2?", output: "4" }],
+ *   metrics: ["answer_relevancy", "faithfulness"],
+ * });
+ * ```
  */
 declare function evaluate(options: EvaluateOptions): Promise<EvalResult[]>;
 /**
@@ -848,6 +879,7 @@ type evals_MetricName = MetricName;
 type evals_Model = Model;
 type evals_ModelCallable = ModelCallable;
 type evals_ModelResponse = ModelResponse;
+type evals_RunGEvalOptions = RunGEvalOptions;
 declare const evals_buildGEvalPrompt: typeof buildGEvalPrompt;
 declare const evals_calculateAggregateScores: typeof calculateAggregateScores;
 declare const evals_compareModels: typeof compareModels;
@@ -863,7 +895,7 @@ declare const evals_getMetricName: typeof getMetricName;
 declare const evals_isCustomMetric: typeof isCustomMetric;
 declare const evals_runGEval: typeof runGEval;
 declare namespace evals {
-  export { evals_AVAILABLE_METRICS as AVAILABLE_METRICS, type evals_CompareModelsOptions as CompareModelsOptions, type evals_CustomMetric as CustomMetric, evals_DEFAULT_JUDGE_MODEL as DEFAULT_JUDGE_MODEL, type evals_DatasetInput as DatasetInput, type evals_DatasetItem as DatasetItem, type evals_EvalResult as EvalResult, type evals_EvaluateOptions as EvaluateOptions, evals_EvaluationDataset as EvaluationDataset, type evals_GEvalScore as GEvalScore, type evals_Golden as Golden, type InitOptions$1 as InitOptions, type evals_LLMTestCase as LLMTestCase, evals_METRIC_PROMPTS as METRIC_PROMPTS, type evals_Message as Message, type evals_MetricInput as MetricInput, type evals_MetricName as MetricName, type evals_Model as Model, type evals_ModelCallable as ModelCallable, type evals_ModelResponse as ModelResponse, evals_buildGEvalPrompt as buildGEvalPrompt, evals_calculateAggregateScores as calculateAggregateScores, evals_compareModels as compareModels, evals_createCustomModel as createCustomModel, evals_createModelFromCallable as createModelFromCallable, evals_createOpenAIModel as createOpenAIModel, evals_customMetric as customMetric, evals_datasetFromFallom as datasetFromFallom, evals_datasetFromTraces as datasetFromTraces, evals_detectRegression as detectRegression, evals_evaluate as evaluate, evals_getMetricName as getMetricName, init$1 as init, evals_isCustomMetric as isCustomMetric, evals_runGEval as runGEval, uploadResultsPublic as uploadResults };
+  export { evals_AVAILABLE_METRICS as AVAILABLE_METRICS, type evals_CompareModelsOptions as CompareModelsOptions, type evals_CustomMetric as CustomMetric, evals_DEFAULT_JUDGE_MODEL as DEFAULT_JUDGE_MODEL, type evals_DatasetInput as DatasetInput, type evals_DatasetItem as DatasetItem, type evals_EvalResult as EvalResult, type evals_EvaluateOptions as EvaluateOptions, evals_EvaluationDataset as EvaluationDataset, type evals_GEvalScore as GEvalScore, type evals_Golden as Golden, type InitOptions$1 as InitOptions, type evals_LLMTestCase as LLMTestCase, evals_METRIC_PROMPTS as METRIC_PROMPTS, type evals_Message as Message, type evals_MetricInput as MetricInput, type evals_MetricName as MetricName, type evals_Model as Model, type evals_ModelCallable as ModelCallable, type evals_ModelResponse as ModelResponse, type evals_RunGEvalOptions as RunGEvalOptions, evals_buildGEvalPrompt as buildGEvalPrompt, evals_calculateAggregateScores as calculateAggregateScores, evals_compareModels as compareModels, evals_createCustomModel as createCustomModel, evals_createModelFromCallable as createModelFromCallable, evals_createOpenAIModel as createOpenAIModel, evals_customMetric as customMetric, evals_datasetFromFallom as datasetFromFallom, evals_datasetFromTraces as datasetFromTraces, evals_detectRegression as detectRegression, evals_evaluate as evaluate, evals_getMetricName as getMetricName, init$1 as init, evals_isCustomMetric as isCustomMetric, evals_runGEval as runGEval, uploadResultsPublic as uploadResults };
 }
 
 /**