@fallom/trace 0.2.5 → 0.2.10

package/dist/index.d.mts

@@ -101,7 +101,7 @@ interface WrapAISDKOptions {
  * await generateText({ model: openai("gpt-4o"), prompt: "Hello!" });
  * ```
  */
-declare function init$3(options?: {
+declare function init$4(options?: {
     apiKey?: string;
     baseUrl?: string;
     captureContent?: boolean;
@@ -223,7 +223,7 @@ type trace_WrapAISDKOptions = WrapAISDKOptions;
 declare const trace_session: typeof session;
 declare const trace_shutdown: typeof shutdown;
 declare namespace trace {
-  export { trace_FallomSession as FallomSession, type trace_SessionContext as SessionContext, type trace_SessionOptions as SessionOptions, type trace_TraceContext as TraceContext, type trace_TraceData as TraceData, type trace_WrapAISDKOptions as WrapAISDKOptions, init$3 as init, trace_session as session, trace_shutdown as shutdown };
+  export { trace_FallomSession as FallomSession, type trace_SessionContext as SessionContext, type trace_SessionOptions as SessionOptions, type trace_TraceContext as TraceContext, type trace_TraceData as TraceData, type trace_WrapAISDKOptions as WrapAISDKOptions, init$4 as init, trace_session as session, trace_shutdown as shutdown };
 }
 
 /**
@@ -244,7 +244,7 @@ declare namespace trace {
  * This is optional - get() will auto-init if needed.
  * Non-blocking: starts background config fetch immediately.
  */
-declare function init$2(options?: {
+declare function init$3(options?: {
     apiKey?: string;
     baseUrl?: string;
 }): void;
@@ -276,7 +276,7 @@ declare function get$1(configKey: string, sessionId: string, options?: {
 }): Promise<string>;
 
 declare namespace models {
-  export { get$1 as get, init$2 as init };
+  export { get$1 as get, init$3 as init };
 }
 
 /**
@@ -313,7 +313,7 @@ interface PromptResult {
  * Initialize Fallom prompts.
  * This is called automatically by fallom.init().
  */
-declare function init$1(options?: {
+declare function init$2(options?: {
     apiKey?: string;
     baseUrl?: string;
 }): void;
@@ -390,7 +390,193 @@ declare const prompts_get: typeof get;
 declare const prompts_getAB: typeof getAB;
 declare const prompts_getPromptContext: typeof getPromptContext;
 declare namespace prompts {
-  export { type prompts_PromptResult as PromptResult, prompts_clearPromptContext as clearPromptContext, prompts_get as get, prompts_getAB as getAB, prompts_getPromptContext as getPromptContext, init$1 as init };
+  export { type prompts_PromptResult as PromptResult, prompts_clearPromptContext as clearPromptContext, prompts_get as get, prompts_getAB as getAB, prompts_getPromptContext as getPromptContext, init$2 as init };
+}
+
+/**
+ * Fallom Evals - Run LLM evaluations locally using G-Eval with LLM as a Judge.
+ *
+ * Evaluate production outputs or compare different models on your dataset.
+ * Results are uploaded to Fallom dashboard for visualization.
+ *
+ */
+type MetricName = "answer_relevancy" | "hallucination" | "toxicity" | "faithfulness" | "completeness";
+declare const AVAILABLE_METRICS: MetricName[];
+/** Dataset can be a list of items OR a string (dataset key to fetch from Fallom) */
+type DatasetInput = DatasetItem[] | string;
+interface DatasetItem {
+    input: string;
+    output: string;
+    systemMessage?: string;
+    metadata?: Record<string, unknown>;
+}
+interface EvalResult {
+    input: string;
+    output: string;
+    systemMessage?: string;
+    model: string;
+    isProduction: boolean;
+    answerRelevancy?: number;
+    hallucination?: number;
+    toxicity?: number;
+    faithfulness?: number;
+    completeness?: number;
+    reasoning: Record<string, string>;
+    latencyMs?: number;
+    tokensIn?: number;
+    tokensOut?: number;
+    cost?: number;
+}
+/** Response format from model calls */
+interface ModelResponse {
+    content: string;
+    tokensIn?: number;
+    tokensOut?: number;
+    cost?: number;
+}
+/** Message format for model calls */
+interface Message {
+    role: "system" | "user" | "assistant";
+    content: string;
+}
+/** Callable type for custom models */
+type ModelCallable = (messages: Message[]) => Promise<ModelResponse>;
+/**
+ * A model configuration for use in compareModels().
+ * Can represent either an OpenRouter model or a custom model (fine-tuned, self-hosted)
+ */
+interface Model {
+    name: string;
+    callFn?: ModelCallable;
+}
+interface InitOptions$1 {
+    apiKey?: string;
+    baseUrl?: string;
+}
+interface EvaluateOptions {
+    dataset: DatasetInput;
+    metrics?: MetricName[];
+    judgeModel?: string;
+    name?: string;
+    description?: string;
+    verbose?: boolean;
+    _skipUpload?: boolean;
+}
+interface CompareModelsOptions extends EvaluateOptions {
+    /**
+     * List of models to test. Each can be:
+     * - A string (model slug for OpenRouter, e.g., "anthropic/claude-3-5-sonnet")
+     * - A Model object (for custom/fine-tuned models)
+     */
+    models: Array<string | Model>;
+    includeProduction?: boolean;
+    modelKwargs?: Record<string, unknown>;
+}
+/**
+ * Initialize Fallom evals.
+ */
+declare function init$1(options?: InitOptions$1): void;
+/**
+ * Evaluate production outputs against specified metrics using G-Eval.
+ * Results are automatically uploaded to Fallom dashboard.
+ */
+declare function evaluate(options: EvaluateOptions): Promise<EvalResult[]>;
+/**
+ * Create a Model using OpenAI directly (for fine-tuned models or direct API access).
+ *
+ * @param modelId - The OpenAI model ID (e.g., "gpt-4o" or "ft:gpt-4o-2024-08-06:org::id")
+ * @param options - Configuration options
+ * @returns A Model instance that can be used in compareModels()
+ */
+declare function createOpenAIModel(modelId: string, options?: {
+    name?: string;
+    apiKey?: string;
+    baseURL?: string;
+    temperature?: number;
+    maxTokens?: number;
+}): Model;
+/**
+ * Create a Model for any OpenAI-compatible API endpoint.
+ * Works with self-hosted models (vLLM, Ollama, LMStudio, etc.), custom endpoints,
+ * or any service that follows the OpenAI chat completions API format.
+ *
+ * @param name - Display name for the model
+ * @param options - Configuration options
+ * @returns A Model instance
+ */
+declare function createCustomModel(name: string, options: {
+    endpoint: string;
+    apiKey?: string;
+    headers?: Record<string, string>;
+    modelField?: string;
+    modelValue?: string;
+    temperature?: number;
+    maxTokens?: number;
+}): Model;
+/**
+ * Create a Model from any callable function.
+ * This is the most flexible option - you provide a function that handles the model call.
+ *
+ * @param name - Display name for the model
+ * @param callFn - Function that takes messages and returns a response
+ * @returns A Model instance
+ */
+declare function createModelFromCallable(name: string, callFn: ModelCallable): Model;
+/**
+ * Compare multiple models on the same dataset.
+ */
+declare function compareModels(options: CompareModelsOptions): Promise<Record<string, EvalResult[]>>;
+/**
+ * Manually upload evaluation results to Fallom dashboard.
+ * Note: Results are automatically uploaded after evaluate() and compareModels(),
+ * so this is only needed for custom scenarios.
+ */
+declare function uploadResults(results: EvalResult[] | Record<string, EvalResult[]>, name: string, description?: string, judgeModel?: string): Promise<string>;
+/**
+ * Create a dataset from Fallom trace data.
+ */
+declare function datasetFromTraces(traces: Array<{
+    attributes?: Record<string, unknown>;
+}>): DatasetItem[];
+/**
+ * Fetch a dataset stored in Fallom by its key.
+ *
+ * @param datasetKey - The unique key of the dataset (e.g., "customer-support-qa")
+ * @param version - Specific version number to fetch. If undefined, fetches the latest version.
+ * @returns List of DatasetItem ready for evaluation
+ */
+declare function datasetFromFallom(datasetKey: string, version?: number): Promise<DatasetItem[]>;
+declare const _default$1: {
+    init: typeof init$1;
+    evaluate: typeof evaluate;
+    compareModels: typeof compareModels;
+    uploadResults: typeof uploadResults;
+    datasetFromTraces: typeof datasetFromTraces;
+    datasetFromFallom: typeof datasetFromFallom;
+    AVAILABLE_METRICS: MetricName[];
+};
+
+declare const evals_AVAILABLE_METRICS: typeof AVAILABLE_METRICS;
+type evals_CompareModelsOptions = CompareModelsOptions;
+type evals_DatasetInput = DatasetInput;
+type evals_DatasetItem = DatasetItem;
+type evals_EvalResult = EvalResult;
+type evals_EvaluateOptions = EvaluateOptions;
+type evals_Message = Message;
+type evals_MetricName = MetricName;
+type evals_Model = Model;
+type evals_ModelCallable = ModelCallable;
+type evals_ModelResponse = ModelResponse;
+declare const evals_compareModels: typeof compareModels;
+declare const evals_createCustomModel: typeof createCustomModel;
+declare const evals_createModelFromCallable: typeof createModelFromCallable;
+declare const evals_createOpenAIModel: typeof createOpenAIModel;
+declare const evals_datasetFromFallom: typeof datasetFromFallom;
+declare const evals_datasetFromTraces: typeof datasetFromTraces;
+declare const evals_evaluate: typeof evaluate;
+declare const evals_uploadResults: typeof uploadResults;
+declare namespace evals {
+  export { evals_AVAILABLE_METRICS as AVAILABLE_METRICS, type evals_CompareModelsOptions as CompareModelsOptions, type evals_DatasetInput as DatasetInput, type evals_DatasetItem as DatasetItem, type evals_EvalResult as EvalResult, type evals_EvaluateOptions as EvaluateOptions, type InitOptions$1 as InitOptions, type evals_Message as Message, type evals_MetricName as MetricName, type evals_Model as Model, type evals_ModelCallable as ModelCallable, type evals_ModelResponse as ModelResponse, evals_compareModels as compareModels, evals_createCustomModel as createCustomModel, evals_createModelFromCallable as createModelFromCallable, evals_createOpenAIModel as createOpenAIModel, evals_datasetFromFallom as datasetFromFallom, evals_datasetFromTraces as datasetFromTraces, _default$1 as default, evals_evaluate as evaluate, init$1 as init, evals_uploadResults as uploadResults };
 }
 
 /**
@@ -551,7 +737,7 @@ declare class FallomExporter implements SpanExporter {
 }
 
 /**
- * Fallom - Model A/B testing, prompt management, and tracing for LLM applications.
+ * Fallom - Model A/B testing, prompt management, tracing, and evals for LLM applications.
  *
  * @example
  * ```typescript
@@ -579,6 +765,14 @@ declare class FallomExporter implements SpanExporter {
  *
  * // Get A/B tested model within session
  * const modelName = await session.getModel({ fallback: "gpt-4o-mini" });
+ *
+ * // Run evaluations
+ * fallom.evals.init({ apiKey: "your-api-key" });
+ * const results = await fallom.evals.evaluate({
+ *   dataset: [{ input: "...", output: "...", systemMessage: "..." }],
+ *   metrics: ["answer_relevancy", "faithfulness"]
+ * });
+ * await fallom.evals.uploadResults(results, "My Eval Run");
  * ```
  */
 
@@ -587,7 +781,8 @@ declare const _default: {
     trace: typeof trace;
     models: typeof models;
     prompts: typeof prompts;
+    evals: typeof evals;
     session: typeof session;
 };
 
-export { FallomExporter, type FallomExporterOptions, FallomSession, type InitOptions, type PromptResult, type SessionContext, type SessionOptions, clearMastraPrompt, _default as default, init, models, prompts, session, setMastraPrompt, setMastraPromptAB, trace };
+export { type CompareModelsOptions, type DatasetItem, type EvalResult, type EvaluateOptions, FallomExporter, type FallomExporterOptions, FallomSession, type InitOptions, type MetricName, type PromptResult, type SessionContext, type SessionOptions, clearMastraPrompt, _default as default, evals, init, models, prompts, session, setMastraPrompt, setMastraPromptAB, trace };

package/dist/index.d.ts

@@ -101,7 +101,7 @@ interface WrapAISDKOptions {
  * await generateText({ model: openai("gpt-4o"), prompt: "Hello!" });
  * ```
  */
-declare function init$3(options?: {
+declare function init$4(options?: {
     apiKey?: string;
     baseUrl?: string;
     captureContent?: boolean;
@@ -223,7 +223,7 @@ type trace_WrapAISDKOptions = WrapAISDKOptions;
 declare const trace_session: typeof session;
 declare const trace_shutdown: typeof shutdown;
 declare namespace trace {
-  export { trace_FallomSession as FallomSession, type trace_SessionContext as SessionContext, type trace_SessionOptions as SessionOptions, type trace_TraceContext as TraceContext, type trace_TraceData as TraceData, type trace_WrapAISDKOptions as WrapAISDKOptions, init$3 as init, trace_session as session, trace_shutdown as shutdown };
+  export { trace_FallomSession as FallomSession, type trace_SessionContext as SessionContext, type trace_SessionOptions as SessionOptions, type trace_TraceContext as TraceContext, type trace_TraceData as TraceData, type trace_WrapAISDKOptions as WrapAISDKOptions, init$4 as init, trace_session as session, trace_shutdown as shutdown };
 }
 
 /**
@@ -244,7 +244,7 @@ declare namespace trace {
  * This is optional - get() will auto-init if needed.
  * Non-blocking: starts background config fetch immediately.
  */
-declare function init$2(options?: {
+declare function init$3(options?: {
     apiKey?: string;
     baseUrl?: string;
 }): void;
@@ -276,7 +276,7 @@ declare function get$1(configKey: string, sessionId: string, options?: {
 }): Promise<string>;
 
 declare namespace models {
-  export { get$1 as get, init$2 as init };
+  export { get$1 as get, init$3 as init };
 }
 
 /**
@@ -313,7 +313,7 @@ interface PromptResult {
  * Initialize Fallom prompts.
  * This is called automatically by fallom.init().
  */
-declare function init$1(options?: {
+declare function init$2(options?: {
     apiKey?: string;
     baseUrl?: string;
 }): void;
@@ -390,7 +390,193 @@ declare const prompts_get: typeof get;
 declare const prompts_getAB: typeof getAB;
 declare const prompts_getPromptContext: typeof getPromptContext;
 declare namespace prompts {
-  export { type prompts_PromptResult as PromptResult, prompts_clearPromptContext as clearPromptContext, prompts_get as get, prompts_getAB as getAB, prompts_getPromptContext as getPromptContext, init$1 as init };
+  export { type prompts_PromptResult as PromptResult, prompts_clearPromptContext as clearPromptContext, prompts_get as get, prompts_getAB as getAB, prompts_getPromptContext as getPromptContext, init$2 as init };
+}
+
+/**
+ * Fallom Evals - Run LLM evaluations locally using G-Eval with LLM as a Judge.
+ *
+ * Evaluate production outputs or compare different models on your dataset.
+ * Results are uploaded to Fallom dashboard for visualization.
+ *
+ */
+type MetricName = "answer_relevancy" | "hallucination" | "toxicity" | "faithfulness" | "completeness";
+declare const AVAILABLE_METRICS: MetricName[];
+/** Dataset can be a list of items OR a string (dataset key to fetch from Fallom) */
+type DatasetInput = DatasetItem[] | string;
+interface DatasetItem {
+    input: string;
+    output: string;
+    systemMessage?: string;
+    metadata?: Record<string, unknown>;
+}
+interface EvalResult {
+    input: string;
+    output: string;
+    systemMessage?: string;
+    model: string;
+    isProduction: boolean;
+    answerRelevancy?: number;
+    hallucination?: number;
+    toxicity?: number;
+    faithfulness?: number;
+    completeness?: number;
+    reasoning: Record<string, string>;
+    latencyMs?: number;
+    tokensIn?: number;
+    tokensOut?: number;
+    cost?: number;
+}
+/** Response format from model calls */
+interface ModelResponse {
+    content: string;
+    tokensIn?: number;
+    tokensOut?: number;
+    cost?: number;
+}
+/** Message format for model calls */
+interface Message {
+    role: "system" | "user" | "assistant";
+    content: string;
+}
+/** Callable type for custom models */
+type ModelCallable = (messages: Message[]) => Promise<ModelResponse>;
+/**
+ * A model configuration for use in compareModels().
+ * Can represent either an OpenRouter model or a custom model (fine-tuned, self-hosted)
+ */
+interface Model {
+    name: string;
+    callFn?: ModelCallable;
+}
+interface InitOptions$1 {
+    apiKey?: string;
+    baseUrl?: string;
+}
+interface EvaluateOptions {
+    dataset: DatasetInput;
+    metrics?: MetricName[];
+    judgeModel?: string;
+    name?: string;
+    description?: string;
+    verbose?: boolean;
+    _skipUpload?: boolean;
+}
+interface CompareModelsOptions extends EvaluateOptions {
+    /**
+     * List of models to test. Each can be:
+     * - A string (model slug for OpenRouter, e.g., "anthropic/claude-3-5-sonnet")
+     * - A Model object (for custom/fine-tuned models)
+     */
+    models: Array<string | Model>;
+    includeProduction?: boolean;
+    modelKwargs?: Record<string, unknown>;
+}
+/**
+ * Initialize Fallom evals.
+ */
+declare function init$1(options?: InitOptions$1): void;
+/**
+ * Evaluate production outputs against specified metrics using G-Eval.
+ * Results are automatically uploaded to Fallom dashboard.
+ */
+declare function evaluate(options: EvaluateOptions): Promise<EvalResult[]>;
+/**
+ * Create a Model using OpenAI directly (for fine-tuned models or direct API access).
+ *
+ * @param modelId - The OpenAI model ID (e.g., "gpt-4o" or "ft:gpt-4o-2024-08-06:org::id")
+ * @param options - Configuration options
+ * @returns A Model instance that can be used in compareModels()
+ */
+declare function createOpenAIModel(modelId: string, options?: {
+    name?: string;
+    apiKey?: string;
+    baseURL?: string;
+    temperature?: number;
+    maxTokens?: number;
+}): Model;
+/**
+ * Create a Model for any OpenAI-compatible API endpoint.
+ * Works with self-hosted models (vLLM, Ollama, LMStudio, etc.), custom endpoints,
+ * or any service that follows the OpenAI chat completions API format.
+ *
+ * @param name - Display name for the model
+ * @param options - Configuration options
+ * @returns A Model instance
+ */
+declare function createCustomModel(name: string, options: {
+    endpoint: string;
+    apiKey?: string;
+    headers?: Record<string, string>;
+    modelField?: string;
+    modelValue?: string;
+    temperature?: number;
+    maxTokens?: number;
+}): Model;
+/**
+ * Create a Model from any callable function.
+ * This is the most flexible option - you provide a function that handles the model call.
+ *
+ * @param name - Display name for the model
+ * @param callFn - Function that takes messages and returns a response
+ * @returns A Model instance
+ */
+declare function createModelFromCallable(name: string, callFn: ModelCallable): Model;
+/**
+ * Compare multiple models on the same dataset.
+ */
+declare function compareModels(options: CompareModelsOptions): Promise<Record<string, EvalResult[]>>;
+/**
+ * Manually upload evaluation results to Fallom dashboard.
+ * Note: Results are automatically uploaded after evaluate() and compareModels(),
+ * so this is only needed for custom scenarios.
+ */
+declare function uploadResults(results: EvalResult[] | Record<string, EvalResult[]>, name: string, description?: string, judgeModel?: string): Promise<string>;
+/**
+ * Create a dataset from Fallom trace data.
+ */
+declare function datasetFromTraces(traces: Array<{
+    attributes?: Record<string, unknown>;
+}>): DatasetItem[];
+/**
+ * Fetch a dataset stored in Fallom by its key.
+ *
+ * @param datasetKey - The unique key of the dataset (e.g., "customer-support-qa")
+ * @param version - Specific version number to fetch. If undefined, fetches the latest version.
+ * @returns List of DatasetItem ready for evaluation
+ */
+declare function datasetFromFallom(datasetKey: string, version?: number): Promise<DatasetItem[]>;
+declare const _default$1: {
+    init: typeof init$1;
+    evaluate: typeof evaluate;
+    compareModels: typeof compareModels;
+    uploadResults: typeof uploadResults;
+    datasetFromTraces: typeof datasetFromTraces;
+    datasetFromFallom: typeof datasetFromFallom;
+    AVAILABLE_METRICS: MetricName[];
+};
+
+declare const evals_AVAILABLE_METRICS: typeof AVAILABLE_METRICS;
+type evals_CompareModelsOptions = CompareModelsOptions;
+type evals_DatasetInput = DatasetInput;
+type evals_DatasetItem = DatasetItem;
+type evals_EvalResult = EvalResult;
+type evals_EvaluateOptions = EvaluateOptions;
+type evals_Message = Message;
+type evals_MetricName = MetricName;
+type evals_Model = Model;
+type evals_ModelCallable = ModelCallable;
+type evals_ModelResponse = ModelResponse;
+declare const evals_compareModels: typeof compareModels;
+declare const evals_createCustomModel: typeof createCustomModel;
+declare const evals_createModelFromCallable: typeof createModelFromCallable;
+declare const evals_createOpenAIModel: typeof createOpenAIModel;
+declare const evals_datasetFromFallom: typeof datasetFromFallom;
+declare const evals_datasetFromTraces: typeof datasetFromTraces;
+declare const evals_evaluate: typeof evaluate;
+declare const evals_uploadResults: typeof uploadResults;
+declare namespace evals {
+  export { evals_AVAILABLE_METRICS as AVAILABLE_METRICS, type evals_CompareModelsOptions as CompareModelsOptions, type evals_DatasetInput as DatasetInput, type evals_DatasetItem as DatasetItem, type evals_EvalResult as EvalResult, type evals_EvaluateOptions as EvaluateOptions, type InitOptions$1 as InitOptions, type evals_Message as Message, type evals_MetricName as MetricName, type evals_Model as Model, type evals_ModelCallable as ModelCallable, type evals_ModelResponse as ModelResponse, evals_compareModels as compareModels, evals_createCustomModel as createCustomModel, evals_createModelFromCallable as createModelFromCallable, evals_createOpenAIModel as createOpenAIModel, evals_datasetFromFallom as datasetFromFallom, evals_datasetFromTraces as datasetFromTraces, _default$1 as default, evals_evaluate as evaluate, init$1 as init, evals_uploadResults as uploadResults };
 }
 
 /**
@@ -551,7 +737,7 @@ declare class FallomExporter implements SpanExporter {
 }
 
 /**
- * Fallom - Model A/B testing, prompt management, and tracing for LLM applications.
+ * Fallom - Model A/B testing, prompt management, tracing, and evals for LLM applications.
  *
  * @example
  * ```typescript
@@ -579,6 +765,14 @@ declare class FallomExporter implements SpanExporter {
  *
  * // Get A/B tested model within session
  * const modelName = await session.getModel({ fallback: "gpt-4o-mini" });
+ *
+ * // Run evaluations
+ * fallom.evals.init({ apiKey: "your-api-key" });
+ * const results = await fallom.evals.evaluate({
+ *   dataset: [{ input: "...", output: "...", systemMessage: "..." }],
+ *   metrics: ["answer_relevancy", "faithfulness"]
+ * });
+ * await fallom.evals.uploadResults(results, "My Eval Run");
  * ```
  */
 
@@ -587,7 +781,8 @@ declare const _default: {
     trace: typeof trace;
     models: typeof models;
     prompts: typeof prompts;
+    evals: typeof evals;
     session: typeof session;
 };
 
-export { FallomExporter, type FallomExporterOptions, FallomSession, type InitOptions, type PromptResult, type SessionContext, type SessionOptions, clearMastraPrompt, _default as default, init, models, prompts, session, setMastraPrompt, setMastraPromptAB, trace };
+export { type CompareModelsOptions, type DatasetItem, type EvalResult, type EvaluateOptions, FallomExporter, type FallomExporterOptions, FallomSession, type InitOptions, type MetricName, type PromptResult, type SessionContext, type SessionOptions, clearMastraPrompt, _default as default, evals, init, models, prompts, session, setMastraPrompt, setMastraPromptAB, trace };