@fallom/trace 0.2.17 → 0.2.21

package/dist/core-DUG2SP2V.mjs

@@ -0,0 +1,21 @@
+import {
+  DEFAULT_JUDGE_MODEL,
+  _apiKey,
+  _baseUrl,
+  _initialized,
+  compareModels,
+  evaluate,
+  init,
+  uploadResultsPublic
+} from "./chunk-GZ6TE7G4.mjs";
+import "./chunk-7P6ASYW6.mjs";
+export {
+  DEFAULT_JUDGE_MODEL,
+  _apiKey,
+  _baseUrl,
+  _initialized,
+  compareModels,
+  evaluate,
+  init,
+  uploadResultsPublic
+};

package/dist/core-JLHYFVYS.mjs

@@ -0,0 +1,21 @@
+import {
+  DEFAULT_JUDGE_MODEL,
+  _apiKey,
+  _baseUrl,
+  _initialized,
+  compareModels,
+  evaluate,
+  init,
+  uploadResultsPublic
+} from "./chunk-XBZ3ESNV.mjs";
+import "./chunk-7P6ASYW6.mjs";
+export {
+  DEFAULT_JUDGE_MODEL,
+  _apiKey,
+  _baseUrl,
+  _initialized,
+  compareModels,
+  evaluate,
+  init,
+  uploadResultsPublic
+};

package/dist/index.d.mts

@@ -415,7 +415,7 @@ declare namespace prompts {
  * Type definitions for Fallom Evals.
  */
 /** Built-in metric names */
-type MetricName = "answer_relevancy" | "hallucination" | "toxicity" | "faithfulness" | "completeness";
+type MetricName = "answer_relevancy" | "hallucination" | "toxicity" | "faithfulness" | "completeness" | "coherence" | "bias";
 /** List of all available built-in metrics */
 declare const AVAILABLE_METRICS: MetricName[];
 /**
@@ -452,6 +452,8 @@ interface EvalResult {
     toxicity?: number;
     faithfulness?: number;
     completeness?: number;
+    coherence?: number;
+    bias?: number;
     reasoning: Record<string, string>;
     latencyMs?: number;
     tokensIn?: number;
@@ -555,6 +557,61 @@ declare const METRIC_PROMPTS: Record<MetricName, {
     criteria: string;
     steps: string[];
 }>;
+/**
+ * Build the G-Eval prompt for the LLM judge.
+ */
+declare function buildGEvalPrompt(criteria: string, steps: string[], systemMessage: string | undefined, inputText: string, outputText: string): string;
+/**
+ * Result of running G-Eval on a single metric.
+ */
+interface GEvalScore {
+    score: number;
+    reasoning: string;
+}
+/**
+ * Run G-Eval for a single metric using OpenRouter.
+ * This is the low-level function used by both the SDK and backend workers.
+ *
+ * @param metric - Built-in metric name or custom metric config
+ * @param inputText - The user's input/query
+ * @param outputText - The LLM's response
+ * @param systemMessage - Optional system message
+ * @param judgeModel - The model to use as judge (OpenRouter format)
+ * @param openrouterKey - OpenRouter API key (defaults to env var)
+ */
+declare function runGEval(metric: string | {
+    name: string;
+    criteria: string;
+    steps: string[];
+}, inputText: string, outputText: string, systemMessage: string | undefined, judgeModel: string, openrouterKey?: string): Promise<GEvalScore>;
+/**
+ * Calculate aggregate scores from a list of results.
+ */
+declare function calculateAggregateScores(results: Array<{
+    scores: Record<string, {
+        score: number;
+    }>;
+}>): Record<string, {
+    avg: number;
+    min: number;
+    max: number;
+    count: number;
+}>;
+/**
+ * Detect regression by comparing current scores to previous scores.
+ */
+declare function detectRegression(currentScores: Record<string, {
+    avg: number;
+}>, previousScores: Record<string, {
+    avg: number;
+}>, threshold?: number): {
+    detected: boolean;
+    details: Record<string, {
+        current: number;
+        previous: number;
+        delta: number;
+    }>;
+};
 
 /**
  * Core evaluation functions.
@@ -781,6 +838,7 @@ type evals_EvalResult = EvalResult;
 type evals_EvaluateOptions = EvaluateOptions;
 type evals_EvaluationDataset = EvaluationDataset;
 declare const evals_EvaluationDataset: typeof EvaluationDataset;
+type evals_GEvalScore = GEvalScore;
 type evals_Golden = Golden;
 type evals_LLMTestCase = LLMTestCase;
 declare const evals_METRIC_PROMPTS: typeof METRIC_PROMPTS;
@@ -790,6 +848,8 @@ type evals_MetricName = MetricName;
 type evals_Model = Model;
 type evals_ModelCallable = ModelCallable;
 type evals_ModelResponse = ModelResponse;
+declare const evals_buildGEvalPrompt: typeof buildGEvalPrompt;
+declare const evals_calculateAggregateScores: typeof calculateAggregateScores;
 declare const evals_compareModels: typeof compareModels;
 declare const evals_createCustomModel: typeof createCustomModel;
 declare const evals_createModelFromCallable: typeof createModelFromCallable;
@@ -797,11 +857,13 @@ declare const evals_createOpenAIModel: typeof createOpenAIModel;
 declare const evals_customMetric: typeof customMetric;
 declare const evals_datasetFromFallom: typeof datasetFromFallom;
 declare const evals_datasetFromTraces: typeof datasetFromTraces;
+declare const evals_detectRegression: typeof detectRegression;
 declare const evals_evaluate: typeof evaluate;
 declare const evals_getMetricName: typeof getMetricName;
 declare const evals_isCustomMetric: typeof isCustomMetric;
+declare const evals_runGEval: typeof runGEval;
 declare namespace evals {
-  export { evals_AVAILABLE_METRICS as AVAILABLE_METRICS, type evals_CompareModelsOptions as CompareModelsOptions, type evals_CustomMetric as CustomMetric, evals_DEFAULT_JUDGE_MODEL as DEFAULT_JUDGE_MODEL, type evals_DatasetInput as DatasetInput, type evals_DatasetItem as DatasetItem, type evals_EvalResult as EvalResult, type evals_EvaluateOptions as EvaluateOptions, evals_EvaluationDataset as EvaluationDataset, type evals_Golden as Golden, type InitOptions$1 as InitOptions, type evals_LLMTestCase as LLMTestCase, evals_METRIC_PROMPTS as METRIC_PROMPTS, type evals_Message as Message, type evals_MetricInput as MetricInput, type evals_MetricName as MetricName, type evals_Model as Model, type evals_ModelCallable as ModelCallable, type evals_ModelResponse as ModelResponse, evals_compareModels as compareModels, evals_createCustomModel as createCustomModel, evals_createModelFromCallable as createModelFromCallable, evals_createOpenAIModel as createOpenAIModel, evals_customMetric as customMetric, evals_datasetFromFallom as datasetFromFallom, evals_datasetFromTraces as datasetFromTraces, evals_evaluate as evaluate, evals_getMetricName as getMetricName, init$1 as init, evals_isCustomMetric as isCustomMetric, uploadResultsPublic as uploadResults };
+  export { evals_AVAILABLE_METRICS as AVAILABLE_METRICS, type evals_CompareModelsOptions as CompareModelsOptions, type evals_CustomMetric as CustomMetric, evals_DEFAULT_JUDGE_MODEL as DEFAULT_JUDGE_MODEL, type evals_DatasetInput as DatasetInput, type evals_DatasetItem as DatasetItem, type evals_EvalResult as EvalResult, type evals_EvaluateOptions as EvaluateOptions, evals_EvaluationDataset as EvaluationDataset, type evals_GEvalScore as GEvalScore, type evals_Golden as Golden, type InitOptions$1 as InitOptions, type evals_LLMTestCase as LLMTestCase, evals_METRIC_PROMPTS as METRIC_PROMPTS, type evals_Message as Message, type evals_MetricInput as MetricInput, type evals_MetricName as MetricName, type evals_Model as Model, type evals_ModelCallable as ModelCallable, type evals_ModelResponse as ModelResponse, evals_buildGEvalPrompt as buildGEvalPrompt, evals_calculateAggregateScores as calculateAggregateScores, evals_compareModels as compareModels, evals_createCustomModel as createCustomModel, evals_createModelFromCallable as createModelFromCallable, evals_createOpenAIModel as createOpenAIModel, evals_customMetric as customMetric, evals_datasetFromFallom as datasetFromFallom, evals_datasetFromTraces as datasetFromTraces, evals_detectRegression as detectRegression, evals_evaluate as evaluate, evals_getMetricName as getMetricName, init$1 as init, evals_isCustomMetric as isCustomMetric, evals_runGEval as runGEval, uploadResultsPublic as uploadResults };
 }
 
 /**

package/dist/index.d.ts

@@ -415,7 +415,7 @@ declare namespace prompts {
  * Type definitions for Fallom Evals.
  */
 /** Built-in metric names */
-type MetricName = "answer_relevancy" | "hallucination" | "toxicity" | "faithfulness" | "completeness";
+type MetricName = "answer_relevancy" | "hallucination" | "toxicity" | "faithfulness" | "completeness" | "coherence" | "bias";
 /** List of all available built-in metrics */
 declare const AVAILABLE_METRICS: MetricName[];
 /**
@@ -452,6 +452,8 @@ interface EvalResult {
     toxicity?: number;
     faithfulness?: number;
     completeness?: number;
+    coherence?: number;
+    bias?: number;
     reasoning: Record<string, string>;
     latencyMs?: number;
     tokensIn?: number;
@@ -555,6 +557,61 @@ declare const METRIC_PROMPTS: Record<MetricName, {
     criteria: string;
     steps: string[];
 }>;
+/**
+ * Build the G-Eval prompt for the LLM judge.
+ */
+declare function buildGEvalPrompt(criteria: string, steps: string[], systemMessage: string | undefined, inputText: string, outputText: string): string;
+/**
+ * Result of running G-Eval on a single metric.
+ */
+interface GEvalScore {
+    score: number;
+    reasoning: string;
+}
+/**
+ * Run G-Eval for a single metric using OpenRouter.
+ * This is the low-level function used by both the SDK and backend workers.
+ *
+ * @param metric - Built-in metric name or custom metric config
+ * @param inputText - The user's input/query
+ * @param outputText - The LLM's response
+ * @param systemMessage - Optional system message
+ * @param judgeModel - The model to use as judge (OpenRouter format)
+ * @param openrouterKey - OpenRouter API key (defaults to env var)
+ */
+declare function runGEval(metric: string | {
+    name: string;
+    criteria: string;
+    steps: string[];
+}, inputText: string, outputText: string, systemMessage: string | undefined, judgeModel: string, openrouterKey?: string): Promise<GEvalScore>;
+/**
+ * Calculate aggregate scores from a list of results.
+ */
+declare function calculateAggregateScores(results: Array<{
+    scores: Record<string, {
+        score: number;
+    }>;
+}>): Record<string, {
+    avg: number;
+    min: number;
+    max: number;
+    count: number;
+}>;
+/**
+ * Detect regression by comparing current scores to previous scores.
+ */
+declare function detectRegression(currentScores: Record<string, {
+    avg: number;
+}>, previousScores: Record<string, {
+    avg: number;
+}>, threshold?: number): {
+    detected: boolean;
+    details: Record<string, {
+        current: number;
+        previous: number;
+        delta: number;
+    }>;
+};
 
 /**
  * Core evaluation functions.
@@ -781,6 +838,7 @@ type evals_EvalResult = EvalResult;
 type evals_EvaluateOptions = EvaluateOptions;
 type evals_EvaluationDataset = EvaluationDataset;
 declare const evals_EvaluationDataset: typeof EvaluationDataset;
+type evals_GEvalScore = GEvalScore;
 type evals_Golden = Golden;
 type evals_LLMTestCase = LLMTestCase;
 declare const evals_METRIC_PROMPTS: typeof METRIC_PROMPTS;
@@ -790,6 +848,8 @@ type evals_MetricName = MetricName;
 type evals_Model = Model;
 type evals_ModelCallable = ModelCallable;
 type evals_ModelResponse = ModelResponse;
+declare const evals_buildGEvalPrompt: typeof buildGEvalPrompt;
+declare const evals_calculateAggregateScores: typeof calculateAggregateScores;
 declare const evals_compareModels: typeof compareModels;
 declare const evals_createCustomModel: typeof createCustomModel;
 declare const evals_createModelFromCallable: typeof createModelFromCallable;
@@ -797,11 +857,13 @@ declare const evals_createOpenAIModel: typeof createOpenAIModel;
 declare const evals_customMetric: typeof customMetric;
 declare const evals_datasetFromFallom: typeof datasetFromFallom;
 declare const evals_datasetFromTraces: typeof datasetFromTraces;
+declare const evals_detectRegression: typeof detectRegression;
 declare const evals_evaluate: typeof evaluate;
 declare const evals_getMetricName: typeof getMetricName;
 declare const evals_isCustomMetric: typeof isCustomMetric;
+declare const evals_runGEval: typeof runGEval;
 declare namespace evals {
-  export { evals_AVAILABLE_METRICS as AVAILABLE_METRICS, type evals_CompareModelsOptions as CompareModelsOptions, type evals_CustomMetric as CustomMetric, evals_DEFAULT_JUDGE_MODEL as DEFAULT_JUDGE_MODEL, type evals_DatasetInput as DatasetInput, type evals_DatasetItem as DatasetItem, type evals_EvalResult as EvalResult, type evals_EvaluateOptions as EvaluateOptions, evals_EvaluationDataset as EvaluationDataset, type evals_Golden as Golden, type InitOptions$1 as InitOptions, type evals_LLMTestCase as LLMTestCase, evals_METRIC_PROMPTS as METRIC_PROMPTS, type evals_Message as Message, type evals_MetricInput as MetricInput, type evals_MetricName as MetricName, type evals_Model as Model, type evals_ModelCallable as ModelCallable, type evals_ModelResponse as ModelResponse, evals_compareModels as compareModels, evals_createCustomModel as createCustomModel, evals_createModelFromCallable as createModelFromCallable, evals_createOpenAIModel as createOpenAIModel, evals_customMetric as customMetric, evals_datasetFromFallom as datasetFromFallom, evals_datasetFromTraces as datasetFromTraces, evals_evaluate as evaluate, evals_getMetricName as getMetricName, init$1 as init, evals_isCustomMetric as isCustomMetric, uploadResultsPublic as uploadResults };
+  export { evals_AVAILABLE_METRICS as AVAILABLE_METRICS, type evals_CompareModelsOptions as CompareModelsOptions, type evals_CustomMetric as CustomMetric, evals_DEFAULT_JUDGE_MODEL as DEFAULT_JUDGE_MODEL, type evals_DatasetInput as DatasetInput, type evals_DatasetItem as DatasetItem, type evals_EvalResult as EvalResult, type evals_EvaluateOptions as EvaluateOptions, evals_EvaluationDataset as EvaluationDataset, type evals_GEvalScore as GEvalScore, type evals_Golden as Golden, type InitOptions$1 as InitOptions, type evals_LLMTestCase as LLMTestCase, evals_METRIC_PROMPTS as METRIC_PROMPTS, type evals_Message as Message, type evals_MetricInput as MetricInput, type evals_MetricName as MetricName, type evals_Model as Model, type evals_ModelCallable as ModelCallable, type evals_ModelResponse as ModelResponse, evals_buildGEvalPrompt as buildGEvalPrompt, evals_calculateAggregateScores as calculateAggregateScores, evals_compareModels as compareModels, evals_createCustomModel as createCustomModel, evals_createModelFromCallable as createModelFromCallable, evals_createOpenAIModel as createOpenAIModel, evals_customMetric as customMetric, evals_datasetFromFallom as datasetFromFallom, evals_datasetFromTraces as datasetFromTraces, evals_detectRegression as detectRegression, evals_evaluate as evaluate, evals_getMetricName as getMetricName, init$1 as init, evals_isCustomMetric as isCustomMetric, evals_runGEval as runGEval, uploadResultsPublic as uploadResults };
 }
 
 /**