@fallom/trace 0.2.15 → 0.2.17

package/dist/core-4L56QWI7.mjs

@@ -0,0 +1,21 @@
+import {
+  DEFAULT_JUDGE_MODEL,
+  _apiKey,
+  _baseUrl,
+  _initialized,
+  compareModels,
+  evaluate,
+  init,
+  uploadResultsPublic
+} from "./chunk-3HBKT4HK.mjs";
+import "./chunk-7P6ASYW6.mjs";
+export {
+  DEFAULT_JUDGE_MODEL,
+  _apiKey,
+  _baseUrl,
+  _initialized,
+  compareModels,
+  evaluate,
+  init,
+  uploadResultsPublic
+};

package/dist/index.d.mts

@@ -458,6 +458,37 @@ interface EvalResult {
     tokensOut?: number;
     cost?: number;
 }
+/**
+ * A golden record from a dataset - contains input and optionally expected output.
+ * This represents a "golden" test case that you can use to generate actual outputs
+ * from your LLM pipeline and then evaluate.
+ */
+interface Golden {
+    input: string;
+    expectedOutput?: string;
+    systemMessage?: string;
+    /** Retrieved documents for RAG evaluation */
+    context?: string[];
+    metadata?: Record<string, unknown>;
+}
+/**
+ * A test case for evaluation - contains input and actual output from your LLM.
+ *
+ */
+interface LLMTestCase {
+    /** The user input/query */
+    input: string;
+    /** The output generated by your LLM pipeline */
+    actualOutput: string;
+    /** (Optional) The expected/golden output for comparison */
+    expectedOutput?: string;
+    /** (Optional) System prompt used */
+    systemMessage?: string;
+    /** (Optional) Retrieved documents for RAG faithfulness evaluation */
+    context?: string[];
+    /** (Optional) Additional metadata */
+    metadata?: Record<string, unknown>;
+}
 /** Response format from model calls */
 interface ModelResponse {
     content: string;
@@ -487,13 +518,16 @@ interface InitOptions$1 {
 }
 /** Options for evaluate() */
 interface EvaluateOptions {
-    dataset: DatasetInput;
+    /** Dataset to evaluate (list of DatasetItem or Fallom dataset key) */
+    dataset?: DatasetInput;
     /** List of metrics to run (built-in or custom). Default: all built-in metrics */
     metrics?: MetricInput[];
     judgeModel?: string;
     name?: string;
     description?: string;
     verbose?: boolean;
+    /** Alternative to dataset - provide test cases from EvaluationDataset */
+    testCases?: LLMTestCase[];
     _skipUpload?: boolean;
 }
 /** Options for compareModels() */
@@ -532,9 +566,10 @@ declare const DEFAULT_JUDGE_MODEL = "openai/gpt-4o-mini";
  */
 declare function init$1(options?: InitOptions$1): void;
 /**
- * Evaluate production outputs against specified metrics using G-Eval.
+ * Evaluate outputs against specified metrics using G-Eval.
  *
  * Results are automatically uploaded to Fallom dashboard.
+ *
  */
 declare function evaluate(options: EvaluateOptions): Promise<EvalResult[]>;
 /**
@@ -630,12 +665,110 @@ declare function datasetFromFallom(datasetKey: string, version?: number, config?
     _baseUrl?: string;
     _initialized?: boolean;
 }): Promise<DatasetItem[]>;
+/**
+ * A dataset for evaluation that supports pulling from Fallom and adding test cases.
+ *
+ * This provides a workflow where you:
+ * 1. Pull a dataset (goldens) from Fallom
+ * 2. Run your own LLM pipeline on each golden to generate outputs
+ * 3. Add the results as test cases
+ * 4. Evaluate the test cases
+ *
+ */
+declare class EvaluationDataset {
+    private _goldens;
+    private _testCases;
+    private _datasetKey;
+    private _datasetName;
+    private _version;
+    /** List of golden records (inputs with optional expected outputs). */
+    get goldens(): Golden[];
+    /** List of test cases (inputs with actual outputs from your LLM). */
+    get testCases(): LLMTestCase[];
+    /** The Fallom dataset key if pulled from Fallom. */
+    get datasetKey(): string | null;
+    /**
+     * Pull a dataset from Fallom.
+     *
+     * @param alias - The dataset key/alias in Fallom
+     * @param version - Specific version to pull (default: latest)
+     * @returns Self for chaining
+     */
+    pull(alias: string, version?: number): Promise<EvaluationDataset>;
+    /**
+     * Add a golden record manually.
+     * @param golden - A Golden object
+     * @returns Self for chaining
+     */
+    addGolden(golden: Golden): EvaluationDataset;
+    /**
+     * Add multiple golden records.
+     * @param goldens - Array of Golden objects
+     * @returns Self for chaining
+     */
+    addGoldens(goldens: Golden[]): EvaluationDataset;
+    /**
+     * Add a test case with actual LLM output.
+     * @param testCase - An LLMTestCase object
+     * @returns Self for chaining
+     */
+    addTestCase(testCase: LLMTestCase): EvaluationDataset;
+    /**
+     * Add multiple test cases.
+     * @param testCases - Array of LLMTestCase objects
+     * @returns Self for chaining
+     */
+    addTestCases(testCases: LLMTestCase[]): EvaluationDataset;
+    /**
+     * Automatically generate test cases by running all goldens through your LLM app.
+     *
+     * @param llmApp - A callable that takes messages and returns response
+     * @param options - Configuration options
+     * @returns Self for chaining
+     */
+    generateTestCases(llmApp: (messages: Message[]) => Promise<ModelResponse>, options?: {
+        includeContext?: boolean;
+    }): Promise<EvaluationDataset>;
+    /** Clear all test cases (useful for re-running with different LLM). */
+    clearTestCases(): EvaluationDataset;
+    /** Return the number of goldens. */
+    get length(): number;
+}
 
 /**
  * Fallom Evals - Run LLM evaluations locally using G-Eval with LLM as a Judge.
  *
  * Evaluate production outputs or compare different models on your dataset.
  * Results are uploaded to Fallom dashboard for visualization.
+ *
+ * @example
+ * import fallom from "@fallom/trace";
+ *
+ * // Initialize
+ * fallom.evals.init({ apiKey: "flm_xxx" });
+ *
+ * // Method 1: Direct dataset evaluation
+ * const results = await fallom.evals.evaluate({
+ *   dataset: [...],
+ *   metrics: ["answer_relevancy", "faithfulness"],
+ * });
+ *
+ * // Method 2: Use EvaluationDataset with your own LLM pipeline
+ * const dataset = new fallom.evals.EvaluationDataset();
+ * await dataset.pull("my-dataset-key");
+ *
+ * for (const golden of dataset.goldens) {
+ *   const actualOutput = await myLLMApp(golden.input);
+ *   dataset.addTestCase({
+ *     input: golden.input,
+ *     actualOutput,
+ *   });
+ * }
+ *
+ * const results = await fallom.evals.evaluate({
+ *   testCases: dataset.testCases,
+ *   metrics: ["answer_relevancy", "faithfulness"],
+ * });
  */
 
 declare const evals_AVAILABLE_METRICS: typeof AVAILABLE_METRICS;
@@ -646,6 +779,10 @@ type evals_DatasetInput = DatasetInput;
 type evals_DatasetItem = DatasetItem;
 type evals_EvalResult = EvalResult;
 type evals_EvaluateOptions = EvaluateOptions;
+type evals_EvaluationDataset = EvaluationDataset;
+declare const evals_EvaluationDataset: typeof EvaluationDataset;
+type evals_Golden = Golden;
+type evals_LLMTestCase = LLMTestCase;
 declare const evals_METRIC_PROMPTS: typeof METRIC_PROMPTS;
 type evals_Message = Message;
 type evals_MetricInput = MetricInput;
@@ -664,7 +801,7 @@ declare const evals_evaluate: typeof evaluate;
 declare const evals_getMetricName: typeof getMetricName;
 declare const evals_isCustomMetric: typeof isCustomMetric;
 declare namespace evals {
-  export { evals_AVAILABLE_METRICS as AVAILABLE_METRICS, type evals_CompareModelsOptions as CompareModelsOptions, type evals_CustomMetric as CustomMetric, evals_DEFAULT_JUDGE_MODEL as DEFAULT_JUDGE_MODEL, type evals_DatasetInput as DatasetInput, type evals_DatasetItem as DatasetItem, type evals_EvalResult as EvalResult, type evals_EvaluateOptions as EvaluateOptions, type InitOptions$1 as InitOptions, evals_METRIC_PROMPTS as METRIC_PROMPTS, type evals_Message as Message, type evals_MetricInput as MetricInput, type evals_MetricName as MetricName, type evals_Model as Model, type evals_ModelCallable as ModelCallable, type evals_ModelResponse as ModelResponse, evals_compareModels as compareModels, evals_createCustomModel as createCustomModel, evals_createModelFromCallable as createModelFromCallable, evals_createOpenAIModel as createOpenAIModel, evals_customMetric as customMetric, evals_datasetFromFallom as datasetFromFallom, evals_datasetFromTraces as datasetFromTraces, evals_evaluate as evaluate, evals_getMetricName as getMetricName, init$1 as init, evals_isCustomMetric as isCustomMetric, uploadResultsPublic as uploadResults };
+  export { evals_AVAILABLE_METRICS as AVAILABLE_METRICS, type evals_CompareModelsOptions as CompareModelsOptions, type evals_CustomMetric as CustomMetric, evals_DEFAULT_JUDGE_MODEL as DEFAULT_JUDGE_MODEL, type evals_DatasetInput as DatasetInput, type evals_DatasetItem as DatasetItem, type evals_EvalResult as EvalResult, type evals_EvaluateOptions as EvaluateOptions, evals_EvaluationDataset as EvaluationDataset, type evals_Golden as Golden, type InitOptions$1 as InitOptions, type evals_LLMTestCase as LLMTestCase, evals_METRIC_PROMPTS as METRIC_PROMPTS, type evals_Message as Message, type evals_MetricInput as MetricInput, type evals_MetricName as MetricName, type evals_Model as Model, type evals_ModelCallable as ModelCallable, type evals_ModelResponse as ModelResponse, evals_compareModels as compareModels, evals_createCustomModel as createCustomModel, evals_createModelFromCallable as createModelFromCallable, evals_createOpenAIModel as createOpenAIModel, evals_customMetric as customMetric, evals_datasetFromFallom as datasetFromFallom, evals_datasetFromTraces as datasetFromTraces, evals_evaluate as evaluate, evals_getMetricName as getMetricName, init$1 as init, evals_isCustomMetric as isCustomMetric, uploadResultsPublic as uploadResults };
 }
 
 /**

package/dist/index.d.ts

@@ -458,6 +458,37 @@ interface EvalResult {
     tokensOut?: number;
     cost?: number;
 }
+/**
+ * A golden record from a dataset - contains input and optionally expected output.
+ * This represents a "golden" test case that you can use to generate actual outputs
+ * from your LLM pipeline and then evaluate.
+ */
+interface Golden {
+    input: string;
+    expectedOutput?: string;
+    systemMessage?: string;
+    /** Retrieved documents for RAG evaluation */
+    context?: string[];
+    metadata?: Record<string, unknown>;
+}
+/**
+ * A test case for evaluation - contains input and actual output from your LLM.
+ *
+ */
+interface LLMTestCase {
+    /** The user input/query */
+    input: string;
+    /** The output generated by your LLM pipeline */
+    actualOutput: string;
+    /** (Optional) The expected/golden output for comparison */
+    expectedOutput?: string;
+    /** (Optional) System prompt used */
+    systemMessage?: string;
+    /** (Optional) Retrieved documents for RAG faithfulness evaluation */
+    context?: string[];
+    /** (Optional) Additional metadata */
+    metadata?: Record<string, unknown>;
+}
 /** Response format from model calls */
 interface ModelResponse {
     content: string;
@@ -487,13 +518,16 @@ interface InitOptions$1 {
 }
 /** Options for evaluate() */
 interface EvaluateOptions {
-    dataset: DatasetInput;
+    /** Dataset to evaluate (list of DatasetItem or Fallom dataset key) */
+    dataset?: DatasetInput;
     /** List of metrics to run (built-in or custom). Default: all built-in metrics */
     metrics?: MetricInput[];
     judgeModel?: string;
     name?: string;
     description?: string;
     verbose?: boolean;
+    /** Alternative to dataset - provide test cases from EvaluationDataset */
+    testCases?: LLMTestCase[];
     _skipUpload?: boolean;
 }
 /** Options for compareModels() */
@@ -532,9 +566,10 @@ declare const DEFAULT_JUDGE_MODEL = "openai/gpt-4o-mini";
  */
 declare function init$1(options?: InitOptions$1): void;
 /**
- * Evaluate production outputs against specified metrics using G-Eval.
+ * Evaluate outputs against specified metrics using G-Eval.
  *
  * Results are automatically uploaded to Fallom dashboard.
+ *
  */
 declare function evaluate(options: EvaluateOptions): Promise<EvalResult[]>;
 /**
@@ -630,12 +665,110 @@ declare function datasetFromFallom(datasetKey: string, version?: number, config?
     _baseUrl?: string;
     _initialized?: boolean;
 }): Promise<DatasetItem[]>;
+/**
+ * A dataset for evaluation that supports pulling from Fallom and adding test cases.
+ *
+ * This provides a workflow where you:
+ * 1. Pull a dataset (goldens) from Fallom
+ * 2. Run your own LLM pipeline on each golden to generate outputs
+ * 3. Add the results as test cases
+ * 4. Evaluate the test cases
+ *
+ */
+declare class EvaluationDataset {
+    private _goldens;
+    private _testCases;
+    private _datasetKey;
+    private _datasetName;
+    private _version;
+    /** List of golden records (inputs with optional expected outputs). */
+    get goldens(): Golden[];
+    /** List of test cases (inputs with actual outputs from your LLM). */
+    get testCases(): LLMTestCase[];
+    /** The Fallom dataset key if pulled from Fallom. */
+    get datasetKey(): string | null;
+    /**
+     * Pull a dataset from Fallom.
+     *
+     * @param alias - The dataset key/alias in Fallom
+     * @param version - Specific version to pull (default: latest)
+     * @returns Self for chaining
+     */
+    pull(alias: string, version?: number): Promise<EvaluationDataset>;
+    /**
+     * Add a golden record manually.
+     * @param golden - A Golden object
+     * @returns Self for chaining
+     */
+    addGolden(golden: Golden): EvaluationDataset;
+    /**
+     * Add multiple golden records.
+     * @param goldens - Array of Golden objects
+     * @returns Self for chaining
+     */
+    addGoldens(goldens: Golden[]): EvaluationDataset;
+    /**
+     * Add a test case with actual LLM output.
+     * @param testCase - An LLMTestCase object
+     * @returns Self for chaining
+     */
+    addTestCase(testCase: LLMTestCase): EvaluationDataset;
+    /**
+     * Add multiple test cases.
+     * @param testCases - Array of LLMTestCase objects
+     * @returns Self for chaining
+     */
+    addTestCases(testCases: LLMTestCase[]): EvaluationDataset;
+    /**
+     * Automatically generate test cases by running all goldens through your LLM app.
+     *
+     * @param llmApp - A callable that takes messages and returns response
+     * @param options - Configuration options
+     * @returns Self for chaining
+     */
+    generateTestCases(llmApp: (messages: Message[]) => Promise<ModelResponse>, options?: {
+        includeContext?: boolean;
+    }): Promise<EvaluationDataset>;
+    /** Clear all test cases (useful for re-running with different LLM). */
+    clearTestCases(): EvaluationDataset;
+    /** Return the number of goldens. */
+    get length(): number;
+}
 
 /**
  * Fallom Evals - Run LLM evaluations locally using G-Eval with LLM as a Judge.
  *
  * Evaluate production outputs or compare different models on your dataset.
  * Results are uploaded to Fallom dashboard for visualization.
+ *
+ * @example
+ * import fallom from "@fallom/trace";
+ *
+ * // Initialize
+ * fallom.evals.init({ apiKey: "flm_xxx" });
+ *
+ * // Method 1: Direct dataset evaluation
+ * const results = await fallom.evals.evaluate({
+ *   dataset: [...],
+ *   metrics: ["answer_relevancy", "faithfulness"],
+ * });
+ *
+ * // Method 2: Use EvaluationDataset with your own LLM pipeline
+ * const dataset = new fallom.evals.EvaluationDataset();
+ * await dataset.pull("my-dataset-key");
+ *
+ * for (const golden of dataset.goldens) {
+ *   const actualOutput = await myLLMApp(golden.input);
+ *   dataset.addTestCase({
+ *     input: golden.input,
+ *     actualOutput,
+ *   });
+ * }
+ *
+ * const results = await fallom.evals.evaluate({
+ *   testCases: dataset.testCases,
+ *   metrics: ["answer_relevancy", "faithfulness"],
+ * });
  */
 
 declare const evals_AVAILABLE_METRICS: typeof AVAILABLE_METRICS;
@@ -646,6 +779,10 @@ type evals_DatasetInput = DatasetInput;
 type evals_DatasetItem = DatasetItem;
 type evals_EvalResult = EvalResult;
 type evals_EvaluateOptions = EvaluateOptions;
+type evals_EvaluationDataset = EvaluationDataset;
+declare const evals_EvaluationDataset: typeof EvaluationDataset;
+type evals_Golden = Golden;
+type evals_LLMTestCase = LLMTestCase;
 declare const evals_METRIC_PROMPTS: typeof METRIC_PROMPTS;
 type evals_Message = Message;
 type evals_MetricInput = MetricInput;
@@ -664,7 +801,7 @@ declare const evals_evaluate: typeof evaluate;
 declare const evals_getMetricName: typeof getMetricName;
 declare const evals_isCustomMetric: typeof isCustomMetric;
 declare namespace evals {
-  export { evals_AVAILABLE_METRICS as AVAILABLE_METRICS, type evals_CompareModelsOptions as CompareModelsOptions, type evals_CustomMetric as CustomMetric, evals_DEFAULT_JUDGE_MODEL as DEFAULT_JUDGE_MODEL, type evals_DatasetInput as DatasetInput, type evals_DatasetItem as DatasetItem, type evals_EvalResult as EvalResult, type evals_EvaluateOptions as EvaluateOptions, type InitOptions$1 as InitOptions, evals_METRIC_PROMPTS as METRIC_PROMPTS, type evals_Message as Message, type evals_MetricInput as MetricInput, type evals_MetricName as MetricName, type evals_Model as Model, type evals_ModelCallable as ModelCallable, type evals_ModelResponse as ModelResponse, evals_compareModels as compareModels, evals_createCustomModel as createCustomModel, evals_createModelFromCallable as createModelFromCallable, evals_createOpenAIModel as createOpenAIModel, evals_customMetric as customMetric, evals_datasetFromFallom as datasetFromFallom, evals_datasetFromTraces as datasetFromTraces, evals_evaluate as evaluate, evals_getMetricName as getMetricName, init$1 as init, evals_isCustomMetric as isCustomMetric, uploadResultsPublic as uploadResults };
+  export { evals_AVAILABLE_METRICS as AVAILABLE_METRICS, type evals_CompareModelsOptions as CompareModelsOptions, type evals_CustomMetric as CustomMetric, evals_DEFAULT_JUDGE_MODEL as DEFAULT_JUDGE_MODEL, type evals_DatasetInput as DatasetInput, type evals_DatasetItem as DatasetItem, type evals_EvalResult as EvalResult, type evals_EvaluateOptions as EvaluateOptions, evals_EvaluationDataset as EvaluationDataset, type evals_Golden as Golden, type InitOptions$1 as InitOptions, type evals_LLMTestCase as LLMTestCase, evals_METRIC_PROMPTS as METRIC_PROMPTS, type evals_Message as Message, type evals_MetricInput as MetricInput, type evals_MetricName as MetricName, type evals_Model as Model, type evals_ModelCallable as ModelCallable, type evals_ModelResponse as ModelResponse, evals_compareModels as compareModels, evals_createCustomModel as createCustomModel, evals_createModelFromCallable as createModelFromCallable, evals_createOpenAIModel as createOpenAIModel, evals_customMetric as customMetric, evals_datasetFromFallom as datasetFromFallom, evals_datasetFromTraces as datasetFromTraces, evals_evaluate as evaluate, evals_getMetricName as getMetricName, init$1 as init, evals_isCustomMetric as isCustomMetric, uploadResultsPublic as uploadResults };
 }
 
 /**