@fallom/trace 0.2.25 → 0.2.28

package/dist/index.js

@@ -346,10 +346,15 @@ var init_types = __esm({
 });
 
 // src/evals/prompts.ts
-function buildGEvalPrompt(criteria, steps, systemMessage, inputText, outputText) {
+function buildGEvalPrompt(criteria, steps, systemMessage, inputText, outputText, judgeContext) {
   const stepsText = steps.map((s, i) => `${i + 1}. ${s}`).join("\n");
   return `You are an expert evaluator assessing LLM outputs using the G-Eval methodology.
+${judgeContext ? `
+## Important Context
+The following context provides background information about the product/domain being evaluated. Use this to inform your evaluation - for example, if the context mentions that certain features or capabilities exist, do not mark responses as hallucinations when they reference those features.
 
+${judgeContext}
+` : ""}
 ## Evaluation Criteria
 ${criteria}
 
@@ -388,7 +393,8 @@ async function runGEval(options) {
     openrouterKey,
     fallomApiKey,
     traceSessionId,
-    traceCustomerId
+    traceCustomerId,
+    judgeContext
   } = options;
   const apiKey4 = openrouterKey || process.env.OPENROUTER_API_KEY;
   if (!apiKey4) {
@@ -406,7 +412,8 @@ async function runGEval(options) {
     config.steps,
     systemMessage,
     inputText,
-    outputText
+    outputText,
+    judgeContext
   );
   const startTime = Date.now();
   const response = await fetch(
@@ -982,14 +989,15 @@ function init4(options = {}) {
   }
   _initialized = true;
 }
-async function runGEval2(metric, inputText, outputText, systemMessage, judgeModel) {
+async function runGEval2(metric, inputText, outputText, systemMessage, judgeModel, judgeContext) {
   const metricArg = isCustomMetric(metric) ? { name: metric.name, criteria: metric.criteria, steps: metric.steps } : metric;
   return runGEval({
     metric: metricArg,
     inputText,
     outputText,
     systemMessage,
-    judgeModel
+    judgeModel,
+    judgeContext
   });
 }
 async function resolveDataset(datasetInput) {
@@ -1040,6 +1048,7 @@ async function evaluate(options) {
     dataset: datasetInput,
     metrics = [...AVAILABLE_METRICS],
     judgeModel = DEFAULT_JUDGE_MODEL,
+    judgeContext,
     name,
     description,
     verbose = true,
@@ -1047,13 +1056,22 @@ async function evaluate(options) {
     _skipUpload = false
   } = options;
   let dataset;
+  let testCaseExtras = /* @__PURE__ */ new Map();
   if (testCases !== void 0 && testCases.length > 0) {
-    dataset = testCases.map((tc) => ({
-      input: tc.input,
-      output: tc.actualOutput,
-      systemMessage: tc.systemMessage,
-      metadata: tc.metadata
-    }));
+    dataset = testCases.map((tc, idx) => {
+      if (tc.expectedOutput || tc.context) {
+        testCaseExtras.set(idx, {
+          expectedOutput: tc.expectedOutput,
+          context: tc.context
+        });
+      }
+      return {
+        input: tc.input,
+        output: tc.actualOutput,
+        systemMessage: tc.systemMessage,
+        metadata: tc.metadata
+      };
+    });
   } else if (datasetInput !== void 0) {
     dataset = await resolveDataset(datasetInput);
   } else {
@@ -1072,10 +1090,14 @@ async function evaluate(options) {
   for (let i = 0; i < dataset.length; i++) {
     const item = dataset[i];
     if (verbose) console.log(`Evaluating item ${i + 1}/${dataset.length}...`);
+    const extras = testCaseExtras.get(i);
     const result = {
       input: item.input,
       output: item.output,
       systemMessage: item.systemMessage,
+      expectedOutput: extras?.expectedOutput,
+      context: extras?.context,
+      metadata: item.metadata,
       model: "production",
       isProduction: true,
       reasoning: {}
@@ -1089,7 +1111,8 @@ async function evaluate(options) {
           item.input,
           item.output,
           item.systemMessage,
-          judgeModel
+          judgeModel,
+          judgeContext
         );
         const key = isCustomMetric(metric) ? metricName : metricName.replace(/_([a-z])/g, (_, c) => c.toUpperCase());
         result[key] = score;
@@ -1120,6 +1143,7 @@ async function compareModels(options) {
     models,
     metrics = [...AVAILABLE_METRICS],
     judgeModel = DEFAULT_JUDGE_MODEL,
+    judgeContext,
     includeProduction = true,
     modelKwargs = {},
     name,
@@ -1137,6 +1161,7 @@ async function compareModels(options) {
       dataset,
       metrics,
       judgeModel,
+      judgeContext,
       verbose,
       _skipUpload: true
     });
@@ -1175,6 +1200,7 @@ async function compareModels(options) {
           input: item.input,
           output,
           systemMessage: item.systemMessage,
+          metadata: item.metadata,
           model: model.name,
           isProduction: false,
           reasoning: {},
@@ -1192,7 +1218,8 @@ async function compareModels(options) {
               item.input,
               output,
               item.systemMessage,
-              judgeModel
+              judgeModel,
+              judgeContext
             );
             const key = isCustomMetric(metric) ? metricName : metricName.replace(/_([a-z])/g, (_, c) => c.toUpperCase());
             result[key] = score;
@@ -1286,6 +1313,9 @@ async function uploadResults(results, name, description, judgeModel, verbose) {
     results: allResults.map((r) => ({
       input: r.input,
       system_message: r.systemMessage,
+      expected_output: r.expectedOutput,
+      context: r.context,
+      metadata: r.metadata,
       model: r.model,
       output: r.output,
       is_production: r.isProduction,
@@ -1361,6 +1391,7 @@ var index_exports = {};
 __export(index_exports, {
   FallomExporter: () => FallomExporter,
   FallomSession: () => FallomSession,
+  FallomSpan: () => FallomSpan,
   buildGEvalPrompt: () => buildGEvalPrompt,
   calculateAggregateScores: () => calculateAggregateScores,
   clearMastraPrompt: () => clearMastraPrompt,
@@ -1374,7 +1405,8 @@ __export(index_exports, {
   session: () => session,
   setMastraPrompt: () => setMastraPrompt,
   setMastraPromptAB: () => setMastraPromptAB,
-  trace: () => trace_exports
+  trace: () => trace_exports,
+  wrapTraced: () => wrapTraced
 });
 module.exports = __toCommonJS(index_exports);
 
@@ -1382,9 +1414,11 @@ module.exports = __toCommonJS(index_exports);
 var trace_exports = {};
 __export(trace_exports, {
   FallomSession: () => FallomSession,
+  FallomSpan: () => FallomSpan,
   init: () => init,
   session: () => session,
-  shutdown: () => shutdown
+  shutdown: () => shutdown,
+  wrapTraced: () => wrapTraced
 });
 
 // src/trace/core.ts
@@ -1395,7 +1429,7 @@ var import_exporter_trace_otlp_http = require("@opentelemetry/exporter-trace-otl
 // node_modules/@opentelemetry/resources/build/esm/Resource.js
 var import_api = require("@opentelemetry/api");
 
-// node_modules/@opentelemetry/resources/node_modules/@opentelemetry/semantic-conventions/build/esm/resource/SemanticResourceAttributes.js
+// node_modules/@opentelemetry/semantic-conventions/build/esm/resource/SemanticResourceAttributes.js
 var SemanticResourceAttributes = {
   /**
    * Name of the cloud provider.
@@ -2170,6 +2204,109 @@ function generateHexId(length) {
   return Array.from(bytes).map((b) => b.toString(16).padStart(2, "0")).join("");
 }
 
+// src/trace/span.ts
+var FallomSpan = class {
+  constructor(name, ctx, options = {}) {
+    this.name = name;
+    this.ctx = ctx;
+    this.attrs = {};
+    this.ended = false;
+    this._status = "OK";
+    this.spanId = generateHexId(16);
+    this.traceId = options.traceId || generateHexId(32);
+    this.parentSpanId = options.parentSpanId;
+    this.kind = options.kind || "custom";
+    this.startTime = Date.now();
+  }
+  /**
+   * Set attributes on the span.
+   * Can be called multiple times - attributes are merged.
+   */
+  set(attributes) {
+    if (this.ended) {
+      console.warn("[Fallom] Cannot set attributes on ended span");
+      return this;
+    }
+    Object.assign(this.attrs, attributes);
+    return this;
+  }
+  /**
+   * Mark the span as errored.
+   */
+  setError(error) {
+    this._status = "ERROR";
+    this._errorMessage = error instanceof Error ? error.message : error;
+    return this;
+  }
+  /**
+   * Get span context for creating child spans.
+   */
+  context() {
+    return {
+      traceId: this.traceId,
+      spanId: this.spanId
+    };
+  }
+  /**
+   * End the span and send it.
+   * Must be called for the span to be recorded.
+   */
+  end() {
+    if (this.ended) {
+      console.warn("[Fallom] Span already ended");
+      return;
+    }
+    this.ended = true;
+    if (!isInitialized()) {
+      return;
+    }
+    const endTime = Date.now();
+    sendTrace({
+      config_key: this.ctx.configKey,
+      session_id: this.ctx.sessionId,
+      customer_id: this.ctx.customerId,
+      metadata: this.ctx.metadata,
+      tags: this.ctx.tags,
+      trace_id: this.traceId,
+      span_id: this.spanId,
+      parent_span_id: this.parentSpanId,
+      name: this.name,
+      kind: this.kind,
+      start_time: new Date(this.startTime).toISOString(),
+      end_time: new Date(endTime).toISOString(),
+      duration_ms: endTime - this.startTime,
+      status: this._status,
+      error_message: this._errorMessage,
+      attributes: {
+        "fallom.sdk_version": "2",
+        "fallom.span_type": "manual",
+        ...this.attrs
+      }
+    }).catch(() => {
+    });
+  }
+};
+function wrapTraced(session2, name, fn, options = {}) {
+  return (async (...args) => {
+    const span = session2.span(name, options);
+    if (args.length === 1) {
+      span.set({ input: args[0] });
+    } else if (args.length > 1) {
+      span.set({ input: args });
+    }
+    try {
+      const result = await fn(...args);
+      span.set({ output: result });
+      span.end();
+      return result;
+    } catch (error) {
+      span.setError(error instanceof Error ? error : String(error));
+      span.end();
+      throw error;
+    }
+  });
+}
+
 // src/prompts.ts
 var prompts_exports = {};
 __export(prompts_exports, {
@@ -4039,6 +4176,26 @@ var FallomSession = class {
   getContext() {
     return { ...this.ctx };
   }
+  /**
+   * Create a manual span for custom operations.
+   *
+   * Use for non-LLM operations like RAG retrieval, preprocessing, tool execution, etc.
+   * The span uses the session's context (configKey, sessionId, etc.).
+   *
+   * @example
+   * ```typescript
+   * const span = session.span("rag.retrieve");
+   * span.set({ "rag.query": userQuery, "rag.topK": 5 });
+   *
+   * const docs = await retrieveDocuments(userQuery);
+   * span.set({ "rag.documents.count": docs.length });
+   *
+   * span.end(); // Must call to send the span
+   * ```
+   */
+  span(name, options) {
+    return new FallomSpan(name, this.ctx, options);
+  }
   /**
    * Get model assignment for this session (A/B testing).
    */
@@ -4277,7 +4434,7 @@ async function init5(options = {}) {
 }
 
 // src/mastra.ts
-var import_core13 = require("@opentelemetry/core");
+var import_core14 = require("@opentelemetry/core");
 var promptContext2 = {};
 function setMastraPrompt(promptKey, version) {
   promptContext2 = {
@@ -4327,7 +4484,7 @@ var FallomExporter = class {
    */
   export(spans, resultCallback) {
     if (spans.length === 0) {
-      resultCallback({ code: import_core13.ExportResultCode.SUCCESS });
+      resultCallback({ code: import_core14.ExportResultCode.SUCCESS });
       return;
     }
     this.log(`Exporting ${spans.length} spans...`);
@@ -4344,11 +4501,11 @@ var FallomExporter = class {
     }
     const exportPromise = this.sendSpans(spans).then(() => {
       this.log("Export successful");
-      resultCallback({ code: import_core13.ExportResultCode.SUCCESS });
+      resultCallback({ code: import_core14.ExportResultCode.SUCCESS });
     }).catch((error) => {
       console.error("[FallomExporter] Export failed:", error);
       resultCallback({
-        code: import_core13.ExportResultCode.FAILED,
+        code: import_core14.ExportResultCode.FAILED,
         error: error instanceof Error ? error : new Error(String(error))
       });
     });
@@ -4528,6 +4685,7 @@ var index_default = {
 0 && (module.exports = {
   FallomExporter,
   FallomSession,
+  FallomSpan,
   buildGEvalPrompt,
   calculateAggregateScores,
   clearMastraPrompt,
@@ -4540,5 +4698,6 @@ var index_default = {
   session,
   setMastraPrompt,
   setMastraPromptAB,
-  trace
+  trace,
+  wrapTraced
 });

package/dist/index.mjs

@@ -23,7 +23,7 @@ import {
   isCustomMetric,
   runGEval,
   uploadResultsPublic
-} from "./chunk-3VWF2OJX.mjs";
+} from "./chunk-MSI4HGK6.mjs";
 import {
   __export
 } from "./chunk-7P6ASYW6.mjs";
@@ -32,9 +32,11 @@ import {
 var trace_exports = {};
 __export(trace_exports, {
   FallomSession: () => FallomSession,
+  FallomSpan: () => FallomSpan,
   init: () => init3,
   session: () => session,
-  shutdown: () => shutdown
+  shutdown: () => shutdown,
+  wrapTraced: () => wrapTraced
 });
 
 // src/trace/core.ts
@@ -45,7 +47,7 @@ import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
 // node_modules/@opentelemetry/resources/build/esm/Resource.js
 import { diag } from "@opentelemetry/api";
 
-// node_modules/@opentelemetry/resources/node_modules/@opentelemetry/semantic-conventions/build/esm/resource/SemanticResourceAttributes.js
+// node_modules/@opentelemetry/semantic-conventions/build/esm/resource/SemanticResourceAttributes.js
 var SemanticResourceAttributes = {
   /**
    * Name of the cloud provider.
@@ -820,6 +822,109 @@ function generateHexId(length) {
   return Array.from(bytes).map((b) => b.toString(16).padStart(2, "0")).join("");
 }
 
+// src/trace/span.ts
+var FallomSpan = class {
+  constructor(name, ctx, options = {}) {
+    this.name = name;
+    this.ctx = ctx;
+    this.attrs = {};
+    this.ended = false;
+    this._status = "OK";
+    this.spanId = generateHexId(16);
+    this.traceId = options.traceId || generateHexId(32);
+    this.parentSpanId = options.parentSpanId;
+    this.kind = options.kind || "custom";
+    this.startTime = Date.now();
+  }
+  /**
+   * Set attributes on the span.
+   * Can be called multiple times - attributes are merged.
+   */
+  set(attributes) {
+    if (this.ended) {
+      console.warn("[Fallom] Cannot set attributes on ended span");
+      return this;
+    }
+    Object.assign(this.attrs, attributes);
+    return this;
+  }
+  /**
+   * Mark the span as errored.
+   */
+  setError(error) {
+    this._status = "ERROR";
+    this._errorMessage = error instanceof Error ? error.message : error;
+    return this;
+  }
+  /**
+   * Get span context for creating child spans.
+   */
+  context() {
+    return {
+      traceId: this.traceId,
+      spanId: this.spanId
+    };
+  }
+  /**
+   * End the span and send it.
+   * Must be called for the span to be recorded.
+   */
+  end() {
+    if (this.ended) {
+      console.warn("[Fallom] Span already ended");
+      return;
+    }
+    this.ended = true;
+    if (!isInitialized()) {
+      return;
+    }
+    const endTime = Date.now();
+    sendTrace({
+      config_key: this.ctx.configKey,
+      session_id: this.ctx.sessionId,
+      customer_id: this.ctx.customerId,
+      metadata: this.ctx.metadata,
+      tags: this.ctx.tags,
+      trace_id: this.traceId,
+      span_id: this.spanId,
+      parent_span_id: this.parentSpanId,
+      name: this.name,
+      kind: this.kind,
+      start_time: new Date(this.startTime).toISOString(),
+      end_time: new Date(endTime).toISOString(),
+      duration_ms: endTime - this.startTime,
+      status: this._status,
+      error_message: this._errorMessage,
+      attributes: {
+        "fallom.sdk_version": "2",
+        "fallom.span_type": "manual",
+        ...this.attrs
+      }
+    }).catch(() => {
+    });
+  }
+};
+function wrapTraced(session2, name, fn, options = {}) {
+  return (async (...args) => {
+    const span = session2.span(name, options);
+    if (args.length === 1) {
+      span.set({ input: args[0] });
+    } else if (args.length > 1) {
+      span.set({ input: args });
+    }
+    try {
+      const result = await fn(...args);
+      span.set({ output: result });
+      span.end();
+      return result;
+    } catch (error) {
+      span.setError(error instanceof Error ? error : String(error));
+      span.end();
+      throw error;
+    }
+  });
+}
+
 // src/prompts.ts
 var prompts_exports = {};
 __export(prompts_exports, {
@@ -2689,6 +2794,26 @@ var FallomSession = class {
   getContext() {
     return { ...this.ctx };
   }
+  /**
+   * Create a manual span for custom operations.
+   *
+   * Use for non-LLM operations like RAG retrieval, preprocessing, tool execution, etc.
+   * The span uses the session's context (configKey, sessionId, etc.).
+   *
+   * @example
+   * ```typescript
+   * const span = session.span("rag.retrieve");
+   * span.set({ "rag.query": userQuery, "rag.topK": 5 });
+   *
+   * const docs = await retrieveDocuments(userQuery);
+   * span.set({ "rag.documents.count": docs.length });
+   *
+   * span.end(); // Must call to send the span
+   * ```
+   */
+  span(name, options) {
+    return new FallomSpan(name, this.ctx, options);
+  }
   /**
    * Get model assignment for this session (A/B testing).
    */
@@ -3168,6 +3293,7 @@ var index_default = {
 export {
   FallomExporter,
   FallomSession,
+  FallomSpan,
   buildGEvalPrompt,
   calculateAggregateScores,
   clearMastraPrompt,
@@ -3181,5 +3307,6 @@ export {
   session,
   setMastraPrompt,
   setMastraPromptAB,
-  trace_exports as trace
+  trace_exports as trace,
+  wrapTraced
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fallom/trace",
-  "version": "0.2.25",
+  "version": "0.2.28",
   "description": "Model A/B testing and tracing for LLM applications. Zero latency, production-ready.",
   "main": "./dist/index.js",
   "module": "./dist/index.mjs",
@@ -40,7 +40,7 @@
   "license": "MIT",
   "repository": {
     "type": "git",
-    "url": "https://github.com/fallom/fallom-js"
+    "url": "https://github.com/Fallomai/fallom-typescript-sdk"
   },
   "dependencies": {
     "@opentelemetry/api": "^1.7.0",