@langfuse/client 4.0.0 → 4.1.0-alpha.1

package/dist/index.d.ts

@@ -1,17 +1,404 @@
 import * as _langfuse_core from '@langfuse/core';
-import { LangfuseAPIClient, Dataset, DatasetItem, DatasetRunItem, ParsedMediaReference, CreatePromptRequest, ChatMessage, ChatMessageWithPlaceholders, PlaceholderMessage, BasePrompt, Prompt, ScoreBody } from '@langfuse/core';
+import { DatasetItem, ScoreBody, Dataset, DatasetRunItem, LangfuseAPIClient, ParsedMediaReference, CreatePromptRequest, ChatMessage, ChatMessageWithPlaceholders, PlaceholderMessage, BasePrompt, Prompt } from '@langfuse/core';
 import { Span } from '@opentelemetry/api';
 
+type ExperimentItem<Input = any, ExpectedOutput = any, Metadata extends Record<string, any> = Record<string, any>> = {
+    /**
+     * The input data to pass to the task function.
+     *
+     * Can be any type - string, object, array, etc. This data will be passed
+     * to your task function as the `input` parameter. Structure it according
+     * to your task's requirements.
+     */
+    input?: Input;
+    /**
+     * The expected output for evaluation purposes.
+     *
+     * Optional ground truth or reference output for this input.
+     * Used by evaluators to assess task performance. If not provided,
+     * only evaluators that don't require expected output can be used.
+     */
+    expectedOutput?: ExpectedOutput;
+    /**
+     * Optional metadata to attach to the experiment item.
+     *
+     * Store additional context, tags, or custom data related to this specific item.
+     * This metadata will be available in traces and can be used for filtering,
+     * analysis, or custom evaluator logic.
+     */
+    metadata?: Metadata;
+} | DatasetItem;
+/**
+ * Parameters passed to an experiment task function.
+ *
+ * Can be either an ExperimentItem (for custom datasets) or a DatasetItem
+ * (for Langfuse datasets). The task function should handle both types.
+ *
+ * @public
+ * @since 4.1.0
+ */
+type ExperimentTaskParams<Input = any, ExpectedOutput = any, Metadata extends Record<string, any> = Record<string, any>> = ExperimentItem<Input, ExpectedOutput, Metadata>;
+/**
+ * Function type for experiment tasks that process input data and return output.
+ *
+ * The task function is the core component being tested in an experiment.
+ * It receives either an ExperimentItem or DatasetItem and produces output
+ * that will be evaluated.
+ *
+ * @param params - Either an ExperimentItem or DatasetItem containing input and metadata
+ * @returns Promise resolving to the task's output (any type)
+ *
+ * @example Task handling both item types
+ * ```typescript
+ * const universalTask: ExperimentTask = async (item) => {
+ *   // Works with both ExperimentItem and DatasetItem
+ *   const input = item.input;
+ *   const metadata = item.metadata;
+ *
+ *   const response = await openai.chat.completions.create({
+ *     model: "gpt-4",
+ *     messages: [{ role: "user", content: input }]
+ *   });
+ *
+ *   return response.choices[0].message.content;
+ * };
+ * ```
+ *
+ * @public
+ * @since 4.1.0
+ */
+type ExperimentTask<Input = any, ExpectedOutput = any, Metadata extends Record<string, any> = Record<string, any>> = (params: ExperimentTaskParams<Input, ExpectedOutput, Metadata>) => Promise<any>;
+type Evaluation = Pick<ScoreBody, "name" | "value" | "comment" | "metadata" | "dataType">;
+type EvaluatorParams<Input = any, ExpectedOutput = any, Metadata extends Record<string, any> = Record<string, any>> = {
+    /**
+     * The original input data passed to the task.
+     *
+     * This is the same input that was provided to the task function.
+     * Use this for context-aware evaluations or input-output relationship analysis.
+     */
+    input: Input;
+    /**
+     * The output produced by the task.
+     *
+     * This is the actual result returned by your task function.
+     * This is the primary value to evaluate against expectations.
+     */
+    output: any;
+    /**
+     * The expected output for comparison (optional).
+     *
+     * This is the ground truth or expected result for the given input.
+     * May not be available for all evaluation scenarios.
+     */
+    expectedOutput?: ExpectedOutput;
+    /**
+     * Optional metadata about the evaluation context.
+     *
+     * Contains additional information from the experiment item or dataset item
+     * that may be useful for evaluation logic, such as tags, categories,
+     * or other contextual data.
+     */
+    metadata?: Metadata;
+};
+type Evaluator<Input = any, ExpectedOutput = any, Metadata extends Record<string, any> = Record<string, any>> = (params: EvaluatorParams<Input, ExpectedOutput, Metadata>) => Promise<Evaluation[] | Evaluation>;
+type RunEvaluatorParams<Input = any, ExpectedOutput = any, Metadata extends Record<string, any> = Record<string, any>> = {
+    /**
+     * Results from all processed experiment items.
+     *
+     * Each item contains the input, output, evaluations, and metadata from
+     * processing a single data item. Use this for aggregate analysis,
+     * statistical calculations, and cross-item comparisons.
+     */
+    itemResults: ExperimentItemResult<Input, ExpectedOutput, Metadata>[];
+};
+type RunEvaluator<Input = any, ExpectedOutput = any, Metadata extends Record<string, any> = Record<string, any>> = (params: RunEvaluatorParams<Input, ExpectedOutput, Metadata>) => Promise<Evaluation[] | Evaluation>;
+type ExperimentParams<Input = any, ExpectedOutput = any, Metadata extends Record<string, any> = Record<string, any>> = {
+    /**
+     * Human-readable name for the experiment.
+     *
+     * This name will appear in Langfuse UI and experiment results.
+     * Choose a descriptive name that identifies the experiment's purpose.
+     */
+    name: string;
+    /**
+     * Optional description explaining the experiment's purpose.
+     *
+     * Provide context about what you're testing, methodology, or goals.
+     * This helps with experiment tracking and result interpretation.
+     */
+    description?: string;
+    /**
+     * Optional metadata to attach to the experiment run.
+     *
+     * Store additional context like model versions, hyperparameters,
+     * or any other relevant information for analysis and comparison.
+     */
+    metadata?: Record<string, any>;
+    /**
+     * Array of data items to process.
+     *
+     * Can be either custom ExperimentItem[] or DatasetItem[] from Langfuse.
+     * Each item should contain input data and optionally expected output.
+     */
+    data: ExperimentItem<Input, ExpectedOutput, Metadata>[];
+    /**
+     * The task function to execute on each data item.
+     *
+     * This function receives input data and produces output that will be evaluated.
+     * It should encapsulate the model or system being tested.
+     */
+    task: ExperimentTask<Input, ExpectedOutput, Metadata>;
+    /**
+     * Optional array of evaluator functions to assess each item's output.
+     *
+     * Each evaluator receives input, output, and expected output (if available)
+     * and returns evaluation results. Multiple evaluators enable comprehensive assessment.
+     */
+    evaluators?: Evaluator<Input, ExpectedOutput, Metadata>[];
+    /**
+     * Optional array of run-level evaluators to assess the entire experiment.
+     *
+     * These evaluators receive all item results and can perform aggregate analysis
+     * like calculating averages, detecting patterns, or statistical analysis.
+     */
+    runEvaluators?: RunEvaluator<Input, ExpectedOutput, Metadata>[];
+    /**
+     * Maximum number of concurrent task executions (default: Infinity).
+     *
+     * Controls parallelism to manage resource usage and API rate limits.
+     * Set lower values for expensive operations or rate-limited services.
+     */
+    maxConcurrency?: number;
+};
+type ExperimentItemResult<Input = any, ExpectedOutput = any, Metadata extends Record<string, any> = Record<string, any>> = Pick<ExperimentItem<Input, ExpectedOutput, Metadata>, "input" | "expectedOutput"> & {
+    /**
+     * The original experiment or dataset item that was processed.
+     *
+     * Contains the complete original item data including input, expected output,
+     * metadata, and any additional fields. Useful for accessing item-specific
+     * context or metadata in result analysis.
+     */
+    item: ExperimentItem<Input, ExpectedOutput, Metadata>;
+    /**
+     * The actual output produced by the task.
+     *
+     * This is the result returned by your task function for this specific input.
+     * It will be passed to evaluators for assessment against expected outputs.
+     */
+    output: any;
+    /**
+     * Results from all evaluators that ran on this item.
+     *
+     * Contains evaluation scores, comments, and metadata from each evaluator
+     * that successfully processed this item. Failed evaluators are excluded.
+     */
+    evaluations: Evaluation[];
+    /**
+     * Langfuse trace ID for this item's execution (for debugging and analysis).
+     *
+     * Use this ID to view detailed execution traces in the Langfuse UI,
+     * including timing, inputs, outputs, and any nested observations.
+     */
+    traceId?: string;
+    /**
+     * Dataset run ID if this item was part of a Langfuse dataset.
+     *
+     * Present only when running experiments on Langfuse datasets.
+     * Links this item result to a specific dataset run for tracking and comparison.
+     */
+    datasetRunId?: string;
+};
+/**
+ * Complete result of an experiment execution.
+ *
+ * Contains all results from processing the experiment data,
+ * including individual item results, run-level evaluations,
+ * and utilities for result visualization.
+ *
+ * @example Using experiment results
+ * ```typescript
+ * const result = await langfuse.experiment.run(config);
+ *
+ * // Access individual results
+ * console.log(`Processed ${result.itemResults.length} items`);
+ *
+ * // Check run-level evaluations
+ * const avgScore = result.runEvaluations.find(e => e.name === 'average_score');
+ * console.log(`Average score: ${avgScore?.value}`);
+ *
+ * // Print formatted results
+ * console.log(await result.prettyPrint());
+ *
+ * // Print summary with individual item results
+ * console.log(await result.prettyPrint({ includeItemResults: true }));
+ *
+ * // Link to dataset run (if available)
+ * if (result.datasetRunUrl) {
+ *   console.log(`View in Langfuse: dataset run ${result.datasetRunUrl}`);
+ * }
+ * ```
+ *
+ * @public
+ */
+type ExperimentResult<Input = any, ExpectedOutput = any, Metadata extends Record<string, any> = Record<string, any>> = {
+    /**
+     * ID of the dataset run in Langfuse (only for experiments on Langfuse datasets).
+     *
+     * Present only when running experiments on Langfuse datasets.
+     * Use this ID to access the dataset run via the Langfuse API or UI
+     * for detailed analysis and comparison with other runs.
+     */
+    datasetRunId?: string;
+    /**
+     * URL to the dataset run in the Langfuse UI (only for experiments on Langfuse datasets).
+     *
+     * Direct link to view the complete dataset run in the Langfuse web interface,
+     * including all experiment results, traces, and analytics. Provides easy access
+     * to detailed analysis and visualization of the experiment.
+     */
+    datasetRunUrl?: string;
+    /**
+     * Results from processing each individual data item.
+     *
+     * Contains the complete results for every item in your experiment data,
+     * including inputs, outputs, evaluations, and trace information.
+     * Use this for detailed analysis of individual item performance.
+     */
+    itemResults: ExperimentItemResult<Input, ExpectedOutput, Metadata>[];
+    /**
+     * Results from run-level evaluators that assessed the entire experiment.
+     *
+     * Contains aggregate evaluations that analyze the complete experiment,
+     * such as average scores, statistical measures, or overall quality assessments.
+     */
+    runEvaluations: Evaluation[];
+    /**
+     * Function to format and display experiment results in a human-readable format.
+     *
+     * Generates a comprehensive, nicely formatted summary including individual results,
+     * aggregate statistics, evaluation scores, and links to traces and dataset runs.
+     *
+     * @param options - Formatting options
+     * @param options.includeItemResults - Whether to include individual item details (default: false)
+     * @returns Promise resolving to formatted string representation
+     */
+    prettyPrint: (options?: {
+        includeItemResults?: boolean;
+    }) => Promise<string>;
+};
+
+/**
+ * Function type for running experiments on Langfuse datasets.
+ *
+ * This function type is attached to fetched datasets to enable convenient
+ * experiment execution directly on dataset objects.
+ *
+ * @param params - Experiment parameters excluding data (since data comes from the dataset)
+ * @returns Promise resolving to experiment results
+ *
+ * @example
+ * ```typescript
+ * const dataset = await langfuse.dataset.get("my-dataset");
+ * const result = await dataset.runExperiment({
+ *   name: "Model Evaluation",
+ *   task: myTask,
+ *   evaluators: [myEvaluator]
+ * });
+ * ```
+ *
+ * @public
+ * @since 4.0.0
+ */
+type RunExperimentOnDataset = (params: Omit<ExperimentParams<any, any, Record<string, any>>, "data">) => Promise<ExperimentResult<any, any, Record<string, any>>>;
+/**
+ * Enhanced dataset object with additional methods for linking and experiments.
+ *
+ * This type extends the base Dataset with functionality for:
+ * - Linking dataset items to traces/observations
+ * - Running experiments directly on the dataset
+ *
+ * @example Working with a fetched dataset
+ * ```typescript
+ * const dataset = await langfuse.dataset.get("my-evaluation-dataset");
+ *
+ * // Access dataset metadata
+ * console.log(dataset.name, dataset.description);
+ *
+ * // Work with individual items
+ * for (const item of dataset.items) {
+ *   console.log(item.input, item.expectedOutput);
+ *
+ *   // Link item to a trace
+ *   await item.link(myObservation, "experiment-run-1");
+ * }
+ *
+ * // Run experiments on the entire dataset
+ * const result = await dataset.runExperiment({
+ *   name: "Model Comparison",
+ *   task: myTask,
+ *   evaluators: [accuracyEvaluator]
+ * });
+ * ```
+ *
+ * @public
+ * @since 4.0.0
+ */
+type FetchedDataset = Dataset & {
+    /** Dataset items with additional linking functionality */
+    items: (DatasetItem & {
+        link: LinkDatasetItemFunction;
+    })[];
+    /** Function to run experiments directly on this dataset */
+    runExperiment: RunExperimentOnDataset;
+};
 /**
  * Function type for linking dataset items to OpenTelemetry spans.
- * This allows dataset items to be associated with specific traces for experiment tracking.
  *
- * @param obj - Object containing the OpenTelemetry span
- * @param runName - Name of the dataset run
- * @param runArgs - Optional arguments for the dataset run
+ * This function creates a connection between a dataset item and a trace/observation,
+ * enabling tracking of which dataset items were used in which experiments or runs.
+ * This is essential for creating dataset runs and tracking experiment lineage.
+ *
+ * @param obj - Object containing the OpenTelemetry span to link to
+ * @param obj.otelSpan - The OpenTelemetry span from a Langfuse observation
+ * @param runName - Name of the experiment run for grouping related items
+ * @param runArgs - Optional configuration for the dataset run
+ * @param runArgs.description - Description of the experiment run
+ * @param runArgs.metadata - Additional metadata to attach to the run
  * @returns Promise that resolves to the created dataset run item
  *
+ * @example Basic linking
+ * ```typescript
+ * const dataset = await langfuse.dataset.get("my-dataset");
+ * const span = startObservation("my-task", { input: "test" });
+ * span.update({ output: "result" });
+ * span.end();
+ *
+ * // Link the dataset item to this execution
+ * await dataset.items[0].link(
+ *   { otelSpan: span.otelSpan },
+ *   "experiment-run-1"
+ * );
+ * ```
+ *
+ * @example Linking with metadata
+ * ```typescript
+ * await dataset.items[0].link(
+ *   { otelSpan: span.otelSpan },
+ *   "model-comparison-v2",
+ *   {
+ *     description: "Comparing GPT-4 vs Claude performance",
+ *     metadata: {
+ *       modelVersion: "gpt-4-1106-preview",
+ *       temperature: 0.7,
+ *       timestamp: new Date().toISOString()
+ *     }
+ *   }
+ * );
+ * ```
+ *
+ * @see {@link https://langfuse.com/docs/datasets} Langfuse datasets documentation
  * @public
+ * @since 4.0.0
  */
 type LinkDatasetItemFunction = (obj: {
     otelSpan: Span;
@@ -30,7 +417,7 @@ type LinkDatasetItemFunction = (obj: {
  * @public
  */
 declare class DatasetManager {
-    private apiClient;
+    private langfuseClient;
     /**
      * Creates a new DatasetManager instance.
      *
@@ -38,44 +425,87 @@ declare class DatasetManager {
      * @internal
      */
     constructor(params: {
-        apiClient: LangfuseAPIClient;
+        langfuseClient: LangfuseClient;
     });
     /**
-     * Retrieves a dataset by name along with all its items.
+     * Retrieves a dataset by name with all its items and experiment functionality.
      *
-     * This method automatically handles pagination to fetch all dataset items
-     * and enhances each item with a `link` function for easy experiment tracking.
+     * This method fetches a dataset and all its associated items, with support
+     * for automatic pagination to handle large datasets efficiently. The returned
+     * dataset object includes enhanced functionality for linking items to traces
+     * and running experiments directly on the dataset.
      *
      * @param name - The name of the dataset to retrieve
-     * @param options - Optional configuration for fetching
+     * @param options - Optional configuration for data fetching
      * @param options.fetchItemsPageSize - Number of items to fetch per page (default: 50)
+     * @returns Promise resolving to enhanced dataset with items, linking, and experiment capabilities
      *
-     * @returns Promise that resolves to the dataset with enhanced items
+     * @example Basic dataset retrieval
+     * ```typescript
+     * const dataset = await langfuse.dataset.get("my-evaluation-dataset");
+     * console.log(`Dataset ${dataset.name} has ${dataset.items.length} items`);
      *
-     * @example
+     * // Access dataset properties
+     * console.log(dataset.description);
+     * console.log(dataset.metadata);
+     * ```
+     *
+     * @example Working with dataset items
      * ```typescript
-     * const dataset = await langfuse.dataset.get("my-dataset");
+     * const dataset = await langfuse.dataset.get("qa-dataset");
      *
      * for (const item of dataset.items) {
-     *   // Use the item data for your experiment
-     *   const result = await processItem(item.input);
-     *
-     *   // Link the result to the dataset item
-     *   await item.link(
-     *     { otelSpan: currentSpan },
-     *     "experiment-run-1",
-     *     { description: "Testing new model" }
-     *   );
+     *   console.log("Question:", item.input);
+     *   console.log("Expected Answer:", item.expectedOutput);
+     *
+     *   // Each item has a link function for connecting to traces
+     *   // await item.link(span, "experiment-name");
      * }
      * ```
+     *
+     * @example Running experiments on datasets
+     * ```typescript
+     * const dataset = await langfuse.dataset.get("benchmark-dataset");
+     *
+     * const result = await dataset.runExperiment({
+     *   name: "GPT-4 Benchmark",
+     *   description: "Evaluating GPT-4 on our benchmark tasks",
+     *   task: async ({ input }) => {
+     *     const response = await openai.chat.completions.create({
+     *       model: "gpt-4",
+     *       messages: [{ role: "user", content: input }]
+     *     });
+     *     return response.choices[0].message.content;
+     *   },
+     *   evaluators: [
+     *     async ({ output, expectedOutput }) => ({
+     *       name: "exact_match",
+     *       value: output === expectedOutput ? 1 : 0
+     *     })
+     *   ]
+     * });
+     *
+     * console.log(await result.prettyPrint());
+     * ```
+     *
+     * @example Handling large datasets
+     * ```typescript
+     * // For very large datasets, use smaller page sizes
+     * const largeDataset = await langfuse.dataset.get(
+     *   "large-dataset",
+     *   { fetchItemsPageSize: 100 }
+     * );
+     * ```
+     *
+     * @throws {Error} If the dataset does not exist or cannot be accessed
+     * @see {@link FetchedDataset} for the complete return type specification
+     * @see {@link RunExperimentOnDataset} for experiment execution details
+     * @public
+     * @since 4.0.0
      */
     get(name: string, options?: {
         fetchItemsPageSize: number;
-    }): Promise<Dataset & {
-        items: (DatasetItem & {
-            link: LinkDatasetItemFunction;
-        })[];
-    }>;
+    }): Promise<FetchedDataset>;
     /**
      * Creates a link function for a specific dataset item.
      *
@@ -86,6 +516,250 @@ declare class DatasetManager {
     private createDatasetItemLinkFunction;
 }
 
+/**
+ * Manages the execution and evaluation of experiments on datasets.
+ *
+ * The ExperimentManager provides a comprehensive framework for running experiments
+ * that test models or tasks against datasets, with support for automatic evaluation,
+ * scoring.
+ *
+ * @example Basic experiment usage
+ * ```typescript
+ * const langfuse = new LangfuseClient();
+ *
+ * const result = await langfuse.experiment.run({
+ *   name: "Capital Cities Test",
+ *   description: "Testing model knowledge of world capitals",
+ *   data: [
+ *     { input: "France", expectedOutput: "Paris" },
+ *     { input: "Germany", expectedOutput: "Berlin" }
+ *   ],
+ *   task: async ({ input }) => {
+ *     const response = await openai.chat.completions.create({
+ *       model: "gpt-4",
+ *       messages: [{ role: "user", content: `What is the capital of ${input}?` }]
+ *     });
+ *     return response.choices[0].message.content;
+ *   },
+ *   evaluators: [
+ *     async ({ input, output, expectedOutput }) => ({
+ *       name: "exact_match",
+ *       value: output === expectedOutput ? 1 : 0
+ *     })
+ *   ]
+ * });
+ *
+ * console.log(await result.prettyPrint());
+ * ```
+ *
+ * @example Using with Langfuse datasets
+ * ```typescript
+ * const dataset = await langfuse.dataset.get("my-dataset");
+ *
+ * const result = await dataset.runExperiment({
+ *   name: "Model Comparison",
+ *   task: myTask,
+ *   evaluators: [myEvaluator],
+ *   runEvaluators: [averageScoreEvaluator]
+ * });
+ * ```
+ *
+ * @public
+ */
+declare class ExperimentManager {
+    private langfuseClient;
+    /**
+     * Creates a new ExperimentManager instance.
+     *
+     * @param params - Configuration object
+     * @param params.langfuseClient - The Langfuse client instance for API communication
+     * @internal
+     */
+    constructor(params: {
+        langfuseClient: LangfuseClient;
+    });
+    /**
+     * Gets the global logger instance for experiment-related logging.
+     *
+     * @returns The global logger instance
+     * @internal
+     */
+    get logger(): _langfuse_core.Logger;
+    /**
+     * Executes an experiment by running a task on each data item and evaluating the results.
+     *
+     * This method orchestrates the complete experiment lifecycle:
+     * 1. Executes the task function on each data item with proper tracing
+     * 2. Runs item-level evaluators on each task output
+     * 3. Executes run-level evaluators on the complete result set
+     * 4. Links results to dataset runs (for Langfuse datasets)
+     * 5. Stores all scores and traces in Langfuse
+     *
+     * @param config - The experiment configuration
+     * @param config.name - Human-readable name for the experiment
+     * @param config.description - Optional description of the experiment's purpose
+     * @param config.metadata - Optional metadata to attach to the experiment run
+     * @param config.data - Array of data items to process (ExperimentItem[] or DatasetItem[])
+     * @param config.task - Function that processes each data item and returns output
+     * @param config.evaluators - Optional array of functions to evaluate each item's output
+     * @param config.runEvaluators - Optional array of functions to evaluate the entire run
+     * @param config.maxConcurrency - Maximum number of concurrent task executions (default: Infinity)
+     *
+     * @returns Promise that resolves to experiment results including:
+     *   - itemResults: Results for each processed data item
+     *   - runEvaluations: Results from run-level evaluators
+     *   - datasetRunId: ID of the dataset run (if using Langfuse datasets)
+     *   - prettyPrint: Function to format and display results
+     *
+     * @throws {Error} When task execution fails and cannot be handled gracefully
+     * @throws {Error} When required evaluators fail critically
+     *
+     * @example Simple experiment
+     * ```typescript
+     * const result = await langfuse.experiment.run({
+     *   name: "Translation Quality Test",
+     *   data: [
+     *     { input: "Hello world", expectedOutput: "Hola mundo" },
+     *     { input: "Good morning", expectedOutput: "Buenos días" }
+     *   ],
+     *   task: async ({ input }) => translateText(input, 'es'),
+     *   evaluators: [
+     *     async ({ output, expectedOutput }) => ({
+     *       name: "bleu_score",
+     *       value: calculateBleuScore(output, expectedOutput)
+     *     })
+     *   ]
+     * });
+     * ```
+     *
+     * @example Experiment with concurrency control
+     * ```typescript
+     * const result = await langfuse.experiment.run({
+     *   name: "Large Scale Evaluation",
+     *   data: largeBatchOfItems,
+     *   task: expensiveModelCall,
+     *   maxConcurrency: 5, // Process max 5 items simultaneously
+     *   evaluators: [myEvaluator],
+     *   runEvaluators: [
+     *     async ({ itemResults }) => ({
+     *       name: "average_score",
+     *       value: itemResults.reduce((acc, r) => acc + r.evaluations[0].value, 0) / itemResults.length
+     *     })
+     *   ]
+     * });
+     * ```
+     *
+     * @see {@link ExperimentParams} for detailed parameter documentation
+     * @see {@link ExperimentResult} for detailed return value documentation
+     * @see {@link Evaluator} for evaluator function specifications
+     * @see {@link RunEvaluator} for run evaluator function specifications
+     *
+     * @public
+     */
+    run<Input = any, ExpectedOutput = any, Metadata extends Record<string, any> = Record<string, any>>(config: ExperimentParams<Input, ExpectedOutput, Metadata>): Promise<ExperimentResult<Input, ExpectedOutput, Metadata>>;
+    /**
+     * Executes the task and evaluators for a single data item.
+     *
+     * This method handles the complete processing pipeline for one data item:
+     * 1. Executes the task within a traced observation span
+     * 2. Links the result to a dataset run (if applicable)
+     * 3. Runs all item-level evaluators on the output
+     * 4. Stores evaluation scores in Langfuse
+     * 5. Handles errors gracefully by continuing with remaining evaluators
+     *
+     * @param params - Parameters for item execution
+     * @param params.experimentName - Name of the parent experiment
+     * @param params.experimentDescription - Description of the parent experiment
+     * @param params.experimentMetadata - Metadata for the parent experiment
+     * @param params.item - The data item to process
+     * @param params.task - The task function to execute
+     * @param params.evaluators - Optional evaluators to run on the output
+     *
+     * @returns Promise resolving to the item result with output, evaluations, and trace info
+     *
+     * @throws {Error} When task execution fails (propagated from task function)
+     *
+     * @internal
+     */
+    private runItem;
+    /**
+     * Formats experiment results into a human-readable string representation.
+     *
+     * Creates a comprehensive, nicely formatted summary of the experiment including:
+     * - Individual item results with inputs, outputs, expected values, and scores
+     * - Dataset item and trace links (when available)
+     * - Experiment overview with aggregate statistics
+     * - Average scores across all evaluations
+     * - Run-level evaluation results
+     * - Links to dataset runs in the Langfuse UI
+     *
+     * @param params - Formatting parameters
+     * @param params.datasetRunUrl - Optional URL to the dataset run in Langfuse UI
+     * @param params.itemResults - Results from processing each data item
+     * @param params.originalData - The original input data items
+     * @param params.runEvaluations - Results from run-level evaluators
+     * @param params.name - Name of the experiment
+     * @param params.description - Optional description of the experiment
+     * @param params.includeItemResults - Whether to include individual item details (default: false)
+     *
+     * @returns Promise resolving to formatted string representation
+     *
+     * @example Output format
+     * ```
+     * 1. Item 1:
+     *    Input:    What is the capital of France?
+     *    Expected: Paris
+     *    Actual:   Paris
+     *    Scores:
+     *      • exact_match: 1.000
+     *      • similarity: 0.95
+     *        💭 Very close match with expected output
+     *
+     *    Dataset Item:
+     *    https://cloud.langfuse.com/project/123/datasets/456/items/789
+     *
+     *    Trace:
+     *    https://cloud.langfuse.com/project/123/traces/abc123
+     *
+     * ──────────────────────────────────────────────────
+     * 📊 Translation Quality Test - Testing model accuracy
+     * 2 items
+     * Evaluations:
+     *   • exact_match
+     *   • similarity
+     *
+     * Average Scores:
+     *   • exact_match: 0.850
+     *   • similarity: 0.923
+     *
+     * Run Evaluations:
+     *   • overall_quality: 0.887
+     *     💭 Good performance with room for improvement
+     *
+     * 🔗 Dataset Run:
+     *    https://cloud.langfuse.com/project/123/datasets/456/runs/def456
+     * ```
+     *
+     * @internal
+     */
+    private prettyPrintResults;
+    /**
+     * Formats a value for display in pretty-printed output.
+     *
+     * Handles different value types appropriately:
+     * - Strings: Truncates long strings to 50 characters with "..."
+     * - Objects/Arrays: Converts to JSON string representation
+     * - Primitives: Uses toString() representation
+     *
+     * @param value - The value to format
+     * @returns Formatted string representation suitable for display
+     *
+     * @internal
+     */
+    private formatValue;
+    private isOtelRegistered;
+}
+
 /**
  * Parameters for resolving media references in objects.
  *
@@ -793,6 +1467,61 @@ declare class LangfuseClient {
      * Manager for media upload and reference resolution.
      */
     media: MediaManager;
+    /**
+     * Manager for running experiments on datasets and data items.
+     *
+     * The experiment manager provides comprehensive functionality for:
+     * - Running tasks on datasets or custom data arrays
+     * - Evaluating outputs with custom or pre-built evaluators
+     * - Tracking experiment runs with automatic tracing
+     * - Generating formatted result summaries
+     * - Integrating with AutoEvals library evaluators
+     *
+     * @example Basic experiment execution
+     * ```typescript
+     * const langfuse = new LangfuseClient();
+     *
+     * const result = await langfuse.experiment.run({
+     *   name: "Model Evaluation",
+     *   description: "Testing model performance on Q&A tasks",
+     *   data: [
+     *     { input: "What is 2+2?", expectedOutput: "4" },
+     *     { input: "What is the capital of France?", expectedOutput: "Paris" }
+     *   ],
+     *   task: async ({ input }) => {
+     *     // Your model/task implementation
+     *     const response = await myModel.generate(input);
+     *     return response;
+     *   },
+     *   evaluators: [
+     *     async ({ output, expectedOutput }) => ({
+     *       name: "exact_match",
+     *       value: output.trim().toLowerCase() === expectedOutput.toLowerCase() ? 1 : 0
+     *     })
+     *   ]
+     * });
+     *
+     * console.log(await result.prettyPrint());
+     * ```
+     *
+     * @example Using with datasets
+     * ```typescript
+     * const dataset = await langfuse.dataset.get("my-test-dataset");
+     * const result = await dataset.runExperiment({
+     *   name: "Production Readiness Test",
+     *   task: myTask,
+     *   evaluators: [accuracyEvaluator, latencyEvaluator],
+     *   runEvaluators: [overallQualityEvaluator]
+     * });
+     * ```
+     *
+     * @see {@link ExperimentManager} for detailed API documentation
+     * @see {@link ExperimentParams} for configuration options
+     * @see {@link ExperimentResult} for result structure
+     * @public
+     * @since 4.0.0
+     */
+    experiment: ExperimentManager;
     private baseUrl;
     private projectId;
     /**
@@ -926,4 +1655,72 @@ declare class LangfuseClient {
     getTraceUrl(traceId: string): Promise<string>;
 }
 
-export { type ChatMessageOrPlaceholder, ChatMessageType, ChatPromptClient, type CreateChatPromptBodyWithPlaceholders, DatasetManager, type LangchainMessagesPlaceholder, LangfuseClient, type LangfuseClientParams, type LangfuseMediaResolveMediaReferencesParams, type LinkDatasetItemFunction, MediaManager, PromptManager, ScoreManager, TextPromptClient };
+/**
+ * Converts an AutoEvals evaluator to a Langfuse-compatible evaluator function.
+ *
+ * This adapter function bridges the gap between AutoEvals library evaluators
+ * and Langfuse experiment evaluators, handling parameter mapping and result
+ * formatting automatically.
+ *
+ * AutoEvals evaluators expect `input`, `output`, and `expected` parameters,
+ * while Langfuse evaluators use `input`, `output`, and `expectedOutput`.
+ * This function handles the parameter name mapping.
+ *
+ * @template E - Type of the AutoEvals evaluator function
+ * @param autoevalEvaluator - The AutoEvals evaluator function to convert
+ * @param params - Optional additional parameters to pass to the AutoEvals evaluator
+ * @returns A Langfuse-compatible evaluator function
+ *
+ * @example Basic usage with AutoEvals
+ * ```typescript
+ * import { Factuality, Levenshtein } from 'autoevals';
+ * import { autoevalsToLangfuseEvaluator } from '@langfuse/client';
+ *
+ * const factualityEvaluator = autoevalsToLangfuseEvaluator(Factuality);
+ * const levenshteinEvaluator = autoevalsToLangfuseEvaluator(Levenshtein);
+ *
+ * await langfuse.experiment.run({
+ *   name: "AutoEvals Integration Test",
+ *   data: myDataset,
+ *   task: myTask,
+ *   evaluators: [factualityEvaluator, levenshteinEvaluator]
+ * });
+ * ```
+ *
+ * @example Using with additional parameters
+ * ```typescript
+ * import { Factuality } from 'autoevals';
+ *
+ * const factualityEvaluator = autoevalsToLangfuseEvaluator(
+ *   Factuality,
+ *   { model: 'gpt-4o' } // Additional params for AutoEvals
+ * );
+ *
+ * await langfuse.experiment.run({
+ *   name: "Factuality Test",
+ *   data: myDataset,
+ *   task: myTask,
+ *   evaluators: [factualityEvaluator]
+ * });
+ * ```
+ *
+ * @see {@link https://github.com/braintrustdata/autoevals} AutoEvals library documentation
+ * @see {@link Evaluator} for Langfuse evaluator specifications
+ *
+ * @public
+ * @since 4.0.0
+ */
+declare function autoevalsToLangfuseEvaluator<E extends CallableFunction>(autoevalEvaluator: E, params?: Params<E>): Evaluator;
+/**
+ * Utility type to extract parameter types from AutoEvals evaluator functions.
+ *
+ * This type helper extracts the parameter type from an AutoEvals evaluator
+ * and omits the standard parameters (input, output, expected) that are
+ * handled by the adapter, leaving only the additional configuration parameters.
+ *
+ * @template E - The AutoEvals evaluator function type
+ * @internal
+ */
+type Params<E> = Parameters<E extends (...args: any[]) => any ? E : never>[0] extends infer P ? Omit<P, "input" | "output" | "expected"> : never;
+
+export { type ChatMessageOrPlaceholder, ChatMessageType, ChatPromptClient, type CreateChatPromptBodyWithPlaceholders, DatasetManager, type Evaluation, type Evaluator, type EvaluatorParams, type ExperimentItem, type ExperimentItemResult, ExperimentManager, type ExperimentParams, type ExperimentResult, type ExperimentTask, type ExperimentTaskParams, type FetchedDataset, type LangchainMessagesPlaceholder, LangfuseClient, type LangfuseClientParams, type LangfuseMediaResolveMediaReferencesParams, type LinkDatasetItemFunction, MediaManager, PromptManager, type RunEvaluator, type RunEvaluatorParams, type RunExperimentOnDataset, ScoreManager, TextPromptClient, autoevalsToLangfuseEvaluator };