langsmith 0.1.21 → 0.1.22

package/dist/evaluation/_runner.d.ts

@@ -0,0 +1,158 @@
+import { Client } from "../index.js";
+import { Example, KVMap, Run, TracerSession } from "../schemas.js";
+import { EvaluationResult, EvaluationResults, RunEvaluator } from "./evaluator.js";
+type TargetT = ((input: KVMap, config?: KVMap) => Promise<KVMap>) | ((input: KVMap, config?: KVMap) => KVMap) | {
+    invoke: (input: KVMap, config?: KVMap) => KVMap;
+} | {
+    invoke: (input: KVMap, config?: KVMap) => Promise<KVMap>;
+};
+type TargetNoInvoke = ((input: KVMap, config?: KVMap) => Promise<KVMap>) | ((input: KVMap, config?: KVMap) => KVMap);
+type DataT = string | AsyncIterable<Example> | Example[];
+type SummaryEvaluatorT = ((runs: Array<Run>, examples: Array<Example>) => Promise<EvaluationResult | EvaluationResults>) | ((runs: Array<Run>, examples: Array<Example>) => EvaluationResult | EvaluationResults);
+type EvaluatorT = RunEvaluator | ((run: Run, example?: Example) => EvaluationResult) | ((run: Run, example?: Example) => Promise<EvaluationResult>);
+interface _ForwardResults {
+    run: Run;
+    example: Example;
+}
+interface _ExperimentManagerArgs {
+    data?: DataT;
+    experiment?: TracerSession | string;
+    metadata?: KVMap;
+    client?: Client;
+    runs?: AsyncGenerator<Run>;
+    evaluationResults?: AsyncGenerator<EvaluationResults>;
+    summaryResults?: AsyncGenerator<(runsArray: Run[]) => AsyncGenerator<EvaluationResults, any, unknown>, any, unknown>;
+    examples?: Example[];
+    _runsArray?: Run[];
+}
+export interface EvaluateOptions {
+    /**
+     * The dataset to evaluate on. Can be a dataset name, a list of
+     * examples, or a generator of examples.
+     */
+    data: DataT;
+    /**
+     * A list of evaluators to run on each example.
+     * @default undefined
+     */
+    evaluators?: Array<EvaluatorT>;
+    /**
+     * A list of summary evaluators to run on the entire dataset.
+     * @default undefined
+     */
+    summaryEvaluators?: Array<SummaryEvaluatorT>;
+    /**
+     * Metadata to attach to the experiment.
+     * @default undefined
+     */
+    metadata?: KVMap;
+    /**
+     * A prefix to provide for your experiment name.
+     * @default undefined
+     */
+    experimentPrefix?: string;
+    /**
+     * The maximum number of concurrent evaluations to run.
+     * @default undefined
+     */
+    maxConcurrency?: number;
+    /**
+     * The LangSmith client to use.
+     * @default undefined
+     */
+    client?: Client;
+}
+export declare function evaluate(
+/**
+ * The target system or function to evaluate.
+ */
+target: TargetT, options: EvaluateOptions): Promise<ExperimentResults>;
+interface ExperimentResultRow {
+    run: Run;
+    example: Example;
+    evaluationResults: EvaluationResults;
+}
+/**
+ * Manage the execution of experiments.
+ *
+ * Supports lazily running predictions and evaluations in parallel to facilitate
+ * result streaming and early debugging.
+ */
+declare class _ExperimentManager {
+    _data?: DataT;
+    _runs?: AsyncGenerator<Run>;
+    _evaluationResults?: AsyncGenerator<EvaluationResults>;
+    _summaryResults?: AsyncGenerator<(runsArray: Run[]) => AsyncGenerator<EvaluationResults, any, unknown>, any, unknown>;
+    _examples?: Example[];
+    _runsArray?: Run[];
+    client: Client;
+    _experiment?: TracerSession;
+    _experimentName: string;
+    _metadata: KVMap;
+    get experimentName(): string;
+    getExamples(): Promise<Array<Example>>;
+    setExamples(examples: Example[]): void;
+    get datasetId(): Promise<string>;
+    get evaluationResults(): AsyncGenerator<EvaluationResults>;
+    get runs(): AsyncGenerator<Run>;
+    constructor(args: _ExperimentManagerArgs);
+    _getExperiment(): TracerSession;
+    _getExperimentMetadata(): Promise<KVMap>;
+    _getProject(firstExample: Example): Promise<TracerSession>;
+    _printExperimentStart(): void;
+    start(): Promise<_ExperimentManager>;
+    withPredictions(target: TargetNoInvoke, options?: {
+        maxConcurrency?: number;
+    }): Promise<_ExperimentManager>;
+    withEvaluators(evaluators: Array<EvaluatorT | RunEvaluator>, options?: {
+        maxConcurrency?: number;
+    }): Promise<_ExperimentManager>;
+    withSummaryEvaluators(summaryEvaluators: Array<SummaryEvaluatorT>): Promise<_ExperimentManager>;
+    getResults(): AsyncGenerator<ExperimentResultRow>;
+    getSummaryScores(): Promise<EvaluationResults>;
+    /**
+     * Run the target function on the examples.
+     * @param {TargetNoInvoke} target The target function to evaluate.
+     * @param options
+     * @returns {AsyncGenerator<_ForwardResults>} An async generator of the results.
+     */
+    _predict(target: TargetNoInvoke, options?: {
+        maxConcurrency?: number;
+    }): AsyncGenerator<_ForwardResults>;
+    _runEvaluators(evaluators: Array<RunEvaluator>, currentResults: ExperimentResultRow, fields: {
+        experimentName: string;
+        client: Client;
+    }): Promise<ExperimentResultRow>;
+    /**
+     * Run the evaluators on the prediction stream.
+     * Expects runs to be available in the manager.
+     * (e.g. from a previous prediction step)
+     * @param {Array<RunEvaluator>} evaluators
+     * @param {number} maxConcurrency
+     */
+    _score(evaluators: Array<RunEvaluator>, options?: {
+        maxConcurrency?: number;
+    }): AsyncGenerator<ExperimentResultRow>;
+    _applySummaryEvaluators(summaryEvaluators: Array<SummaryEvaluatorT>): AsyncGenerator<(runsArray: Run[]) => AsyncGenerator<EvaluationResults>>;
+    _getDatasetVersion(): Promise<string | undefined>;
+    _end(): Promise<void>;
+}
+/**
+ * Represents the results of an evaluate() call.
+ * This class provides an iterator interface to iterate over the experiment results
+ * as they become available. It also provides methods to access the experiment name,
+ * the number of results, and to wait for the results to be processed.
+ */
+declare class ExperimentResults implements AsyncIterableIterator<ExperimentResultRow> {
+    private manager;
+    results: ExperimentResultRow[];
+    processedCount: number;
+    summaryResults: EvaluationResults;
+    constructor(experimentManager: _ExperimentManager);
+    get experimentName(): string;
+    [Symbol.asyncIterator](): AsyncIterableIterator<ExperimentResultRow>;
+    next(): Promise<IteratorResult<ExperimentResultRow>>;
+    processData(manager: _ExperimentManager): Promise<void>;
+    get length(): number;
+}
+export {};