langwatch 0.10.0 → 0.12.0

package/dist/index.d.ts

@@ -1,6 +1,6 @@
 import { L as Logger, C as ConsoleLogger, N as NoOpLogger } from './index-D7rKIGrO.js';
-export { F as FilterableBatchSpanProcessor, L as LangWatchExporter, S as SpanProcessingExcludeRule, g as getLangWatchLogger, d as getLangWatchTracer } from './implementation-CPxv2BdW.js';
-import { p as paths, P as PromptResponse, g as CreatePromptBody, U as UpdatePromptBody, h as PromptData, i as Prompt, F as FetchPolicy } from './types-usU5mTCX.js';
+export { F as FilterableBatchSpanProcessor, L as LangWatchExporter, S as SpanProcessingExcludeRule, g as getLangWatchLogger, d as getLangWatchTracer } from './implementation-Bnc8Aymq.js';
+import { p as paths, P as PromptResponse, g as CreatePromptBody, U as UpdatePromptBody, h as PromptData, i as Prompt, F as FetchPolicy, L as LangWatchSpan } from './types-fo-Ij9pl.js';
 import openApiCreateClient from 'openapi-fetch';
 import { z } from 'zod';
 export { l as attributes } from './types-DRiQaKFG.js';
@@ -353,6 +353,667 @@ declare class PromptsFacade implements Pick<PromptsApiService, "sync" | "delete"
     }) => Promise<SyncResult>;
 }
 
+/**
+ * Types for the Dataset API
+ */
+/**
+ * A single entry in a dataset
+ */
+type DatasetEntry<T extends Record<string, unknown> = Record<string, unknown>> = {
+    /** Unique identifier for this entry */
+    id: string;
+    /** The dataset this entry belongs to */
+    datasetId: string;
+    /** The project this entry belongs to */
+    projectId: string;
+    /** The actual data for this entry */
+    entry: T;
+    /** When this entry was created */
+    createdAt: string;
+    /** When this entry was last updated */
+    updatedAt: string;
+};
+/**
+ * A dataset containing multiple entries
+ */
+type Dataset<T extends Record<string, unknown> = Record<string, unknown>> = {
+    /** Array of dataset entries */
+    entries: DatasetEntry<T>[];
+};
+/**
+ * Options for getting a dataset
+ */
+type GetDatasetOptions = {
+    /** Skip tracing for this operation */
+    ignoreTracing?: boolean;
+};
+
+type DatasetsFacadeConfig = {
+    langwatchApiClient: LangwatchApiClient;
+    logger: Logger;
+};
+/**
+ * Facade for dataset operations
+ *
+ * Provides a simple interface for fetching datasets from LangWatch.
+ *
+ * @example
+ * ```typescript
+ * const langwatch = new LangWatch({ apiKey: "your-api-key" });
+ *
+ * // Get a dataset by slug or ID
+ * const dataset = await langwatch.datasets.get("my-dataset");
+ *
+ * // Use with evaluation
+ * const evaluation = langwatch.evaluation.init("my-experiment");
+ * await evaluation.run(dataset.entries.map(e => e.entry), async ({ item, index }) => {
+ *   const output = await myLLM(item.input);
+ *   await evaluation.evaluate("my-evaluator", {
+ *     data: { input: item.input, output, expected_output: item.expected_output },
+ *     settings: {}
+ *   });
+ * });
+ * ```
+ */
+declare class DatasetsFacade {
+    #private;
+    constructor(config: DatasetsFacadeConfig);
+    /**
+     * Fetches a dataset by its slug or ID
+     *
+     * @param slugOrId - The slug or ID of the dataset to fetch
+     * @param options - Optional configuration
+     * @returns The dataset with all entries
+     *
+     * @example
+     * ```typescript
+     * // Get dataset by slug
+     * const dataset = await langwatch.datasets.get("product-qa");
+     *
+     * // Get dataset by ID
+     * const dataset = await langwatch.datasets.get("ds_abc123");
+     *
+     * // Typed dataset
+     * type MyDatasetEntry = { input: string; expected_output: string; };
+     * const dataset = await langwatch.datasets.get<MyDatasetEntry>("my-dataset");
+     *
+     * // Iterate over entries
+     * for (const entry of dataset.entries) {
+     *   console.log(entry.entry.input);  // typed as string
+     * }
+     * ```
+     */
+    get: <T extends Record<string, unknown> = Record<string, unknown>>(slugOrId: string, options?: GetDatasetOptions) => Promise<Dataset<T>>;
+}
+
+/**
+ * Types for the Evaluation API
+ *
+ * These types define the structure for batch evaluations, including
+ * logging metrics, running evaluators, and managing targets.
+ */
+
+/**
+ * Status of an evaluation result
+ */
+type EvaluationStatus = "processed" | "error" | "skipped";
+/**
+ * Target types for batch evaluations
+ */
+type TargetType = "prompt" | "agent" | "custom";
+/**
+ * Metadata for targets - used for comparison charts
+ */
+type TargetMetadata = Record<string, string | number | boolean>;
+declare const targetInfoSchema: z.ZodObject<{
+    id: z.ZodString;
+    name: z.ZodString;
+    type: z.ZodDefault<z.ZodEnum<{
+        agent: "agent";
+        custom: "custom";
+        prompt: "prompt";
+    }>>;
+    metadata: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>>>;
+}, z.core.$strip>;
+declare const evaluationResultSchema: z.ZodObject<{
+    name: z.ZodString;
+    evaluator: z.ZodString;
+    trace_id: z.ZodString;
+    status: z.ZodEnum<{
+        error: "error";
+        processed: "processed";
+        skipped: "skipped";
+    }>;
+    data: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+    score: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
+    passed: z.ZodOptional<z.ZodNullable<z.ZodBoolean>>;
+    details: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+    index: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
+    label: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+    cost: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
+    duration: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
+    error_type: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+    traceback: z.ZodOptional<z.ZodNullable<z.ZodArray<z.ZodString>>>;
+    target_id: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+}, z.core.$strip>;
+/**
+ * Information about a registered target
+ */
+type TargetInfo = z.infer<typeof targetInfoSchema>;
+/**
+ * Result of an evaluation
+ */
+type EvaluationResult = z.infer<typeof evaluationResultSchema>;
+/**
+ * Options for initializing an evaluation
+ */
+type EvaluationInitOptions = {
+    /** Custom run ID (auto-generated if not provided) */
+    runId?: string;
+    /** Number of parallel threads for submit() */
+    threads?: number;
+};
+/**
+ * Options for the log() method
+ */
+type LogOptions = {
+    /**
+     * Row index in the dataset.
+     * Optional when called inside withTarget() - will be auto-inferred from context.
+     */
+    index?: number;
+    /** Additional data/inputs for the evaluation */
+    data?: Record<string, unknown>;
+    /** Numeric score (typically 0-1) */
+    score?: number;
+    /** Whether the evaluation passed */
+    passed?: boolean;
+    /** Label/category for the result */
+    label?: string;
+    /** Human-readable description of the result */
+    details?: string;
+    /** Status of the evaluation */
+    status?: EvaluationStatus;
+    /** Duration in milliseconds */
+    duration?: number;
+    /** Cost amount in USD */
+    cost?: number;
+    /** Error if one occurred */
+    error?: Error;
+    /**
+     * Target name for multi-target comparisons.
+     * Optional when called inside withTarget() - will be auto-inferred from context.
+     */
+    target?: string;
+    /** Metadata for the target (only used on first call per target) */
+    metadata?: TargetMetadata;
+};
+/**
+ * Options for the evaluate() method (built-in evaluators)
+ */
+type EvaluateOptions = {
+    /**
+     * Row index in the dataset.
+     * Optional when called inside withTarget() - will be auto-inferred from context.
+     */
+    index?: number;
+    /** Data to pass to the evaluator */
+    data: Record<string, unknown>;
+    /** Evaluator settings */
+    settings?: Record<string, unknown>;
+    /** Human-readable name for the evaluation */
+    name?: string;
+    /** Whether to run as a guardrail */
+    asGuardrail?: boolean;
+    /**
+     * Target name for multi-target comparisons.
+     * Optional when called inside withTarget() - will be auto-inferred from context.
+     */
+    target?: string;
+    /** Metadata for the target */
+    metadata?: TargetMetadata;
+};
+/**
+ * Context passed to the run() callback
+ */
+type RunContext<T> = {
+    /** Current index in the dataset */
+    index: number;
+    /** The dataset item */
+    item: T;
+    /** The span for this iteration (for custom instrumentation) */
+    span: LangWatchSpan;
+};
+/**
+ * Options for the run() method
+ */
+type RunOptions = {
+    /** Number of concurrent executions (default: 4) */
+    concurrency?: number;
+};
+/**
+ * Callback function for run()
+ */
+type RunCallback<T> = (context: RunContext<T>) => Promise<void> | void;
+/**
+ * Context passed to the withTarget() callback
+ */
+type TargetContext = {
+    /** The LangWatch span for this target execution */
+    span: LangWatchSpan;
+    /** The trace ID for this target execution */
+    traceId: string;
+    /** The span ID for this target execution */
+    spanId: string;
+};
+/**
+ * Callback function for withTarget()
+ */
+type TargetCallback<R> = (context: TargetContext) => Promise<R> | R;
+/**
+ * Result from withTarget() including captured metrics
+ */
+type TargetResult<R> = {
+    /** The return value from the callback */
+    result: R;
+    /** Duration in milliseconds (automatically captured) */
+    duration: number;
+    /** Cost in USD (captured from span if available) */
+    cost?: number;
+    /** The trace ID for this execution */
+    traceId: string;
+    /** The span ID for this execution */
+    spanId: string;
+};
+
+/**
+ * Evaluation - Main class for running batch evaluations
+ *
+ * Provides a clean API for running evaluations over datasets with:
+ * - Automatic tracing per iteration
+ * - Parallel execution with concurrency control
+ * - Batched result sending
+ * - Built-in evaluator support
+ * - Multi-target comparison with withTarget() context isolation
+ */
+
+/**
+ * Evaluation session for running batch evaluations
+ */
+declare class Evaluation {
+    readonly name: string;
+    readonly runId: string;
+    readonly experimentSlug: string;
+    private readonly apiClient;
+    private readonly endpoint;
+    private readonly apiKey;
+    private readonly logger;
+    private readonly concurrency;
+    private initialized;
+    private createdAtMs;
+    private total;
+    private progress;
+    private batch;
+    private lastSentMs;
+    private pendingFlush;
+    private flushTimeout;
+    private targets;
+    private currentTraceId;
+    private currentIndex;
+    private iterationUsedWithTarget;
+    private evaluationUsesTargets;
+    private constructor();
+    /**
+     * Initialize an evaluation session
+     */
+    static init(name: string, options: {
+        apiClient: LangwatchApiClient;
+        endpoint: string;
+        apiKey: string;
+        logger: Logger;
+    } & EvaluationInitOptions): Promise<Evaluation>;
+    /**
+     * Initialize the evaluation by creating/getting the experiment
+     */
+    private initialize;
+    /**
+     * Run evaluation over a dataset with a callback
+     *
+     * @param dataset - Array of items to evaluate
+     * @param callback - Function called for each item with { item, index, span }
+     * @param options - Concurrency options
+     *
+     * @example
+     * ```typescript
+     * await evaluation.run(dataset, async ({ item, index, span }) => {
+     *   const response = await myAgent(item.question);
+     *   evaluation.log('accuracy', { index, score: 0.95 });
+     * }, { concurrency: 4 });
+     * ```
+     */
+    run<T>(dataset: T[], callback: RunCallback<T>, options?: RunOptions): Promise<void>;
+    /**
+     * Execute a single item in the dataset
+     */
+    private executeItem;
+    /**
+     * Log a custom metric result
+     *
+     * @param metric - Name of the metric
+     * @param options - Metric options including index, score, passed, etc.
+     *
+     * If called inside a withTarget() block, the target and index are automatically
+     * inferred from the context and don't need to be specified.
+     *
+     * @example
+     * ```typescript
+     * // Explicit target (outside withTarget)
+     * evaluation.log('accuracy', { index, score: 0.95, target: 'gpt-4' });
+     *
+     * // Implicit target (inside withTarget)
+     * await evaluation.withTarget('gpt-4', { model: 'openai/gpt-4' }, async () => {
+     *   evaluation.log('accuracy', { score: 0.95 }); // target and index auto-inferred
+     * });
+     * ```
+     */
+    log(metric: string, options: LogOptions): void;
+    /**
+     * Run a built-in evaluator
+     *
+     * @param evaluatorSlug - The evaluator identifier (e.g., 'ragas/faithfulness')
+     * @param options - Evaluator options including data and settings
+     *
+     * If called inside a withTarget() block, the target and index are automatically
+     * inferred from the context and don't need to be specified.
+     *
+     * @example
+     * ```typescript
+     * // Inside withTarget() - target and index auto-inferred
+     * await evaluation.withTarget('gpt-4', { model: 'openai/gpt-4' }, async () => {
+     *   await evaluation.evaluate('ragas/faithfulness', {
+     *     data: { input, output, contexts },
+     *   });
+     * });
+     *
+     * // Or explicit index/target
+     * await evaluation.evaluate('ragas/faithfulness', {
+     *   index,
+     *   data: { input, output, contexts },
+     *   target: 'gpt-4',
+     * });
+     * ```
+     */
+    evaluate(evaluatorSlug: string, options: EvaluateOptions): Promise<void>;
+    /**
+     * Execute code within a target context with automatic tracing
+     *
+     * Creates a new span for this target execution and sets up context
+     * so that log() calls inside the callback automatically use this target.
+     * Duration and output are captured automatically.
+     *
+     * This creates a dataset entry per target (like Evaluations V3), enabling
+     * proper per-target latency and cost tracking.
+     *
+     * @param targetName - Unique identifier for the target
+     * @param metadata - Optional metadata for comparison (e.g., { model: 'gpt-4' })
+     * @param callback - Function to execute within the target context
+     * @returns The callback result along with captured metrics
+     *
+     * @example
+     * ```typescript
+     * await evaluation.run(dataset, async ({ item, index }) => {
+     *   // Compare GPT-4 and Claude on the same input
+     *   const [gpt4Result, claudeResult] = await Promise.all([
+     *     evaluation.withTarget('gpt-4', { model: 'openai/gpt-4' }, async () => {
+     *       const response = await openai.chat(item.question);
+     *       evaluation.log('quality', { score: 0.95 }); // target auto-inferred
+     *       return response;
+     *     }),
+     *     evaluation.withTarget('claude-3', { model: 'anthropic/claude-3' }, async () => {
+     *       const response = await anthropic.messages(item.question);
+     *       evaluation.log('quality', { score: 0.85 }); // target auto-inferred
+     *       return response;
+     *     }),
+     *   ]);
+     * });
+     * ```
+     */
+    withTarget<R>(targetName: string, metadata: TargetMetadata | null, callback: TargetCallback<R>): Promise<TargetResult<R>>;
+    withTarget<R>(targetName: string, callback: TargetCallback<R>): Promise<TargetResult<R>>;
+    /**
+     * Register a target for multi-target comparison
+     */
+    private registerTarget;
+    /**
+     * Schedule a debounced send
+     */
+    private scheduleSend;
+    /**
+     * Send current batch to the API
+     */
+    private sendBatch;
+    /**
+     * Flush all pending data
+     */
+    private flush;
+    /**
+     * Serialize a dataset item for the API
+     */
+    private serializeItem;
+    /**
+     * Get trace ID from current OpenTelemetry context
+     */
+    private getTraceIdFromContext;
+    /**
+     * Get span ID from current OpenTelemetry context
+     */
+    private getSpanIdFromContext;
+}
+
+/**
+ * Types for platform-configured evaluations (Evaluations V3)
+ */
+/**
+ * Summary of a completed evaluation run
+ */
+type EvaluationRunSummary = {
+    runId?: string;
+    totalCells?: number;
+    completedCells?: number;
+    failedCells?: number;
+    duration?: number;
+    runUrl?: string;
+    timestamps?: {
+        startedAt: number;
+        finishedAt?: number;
+        stoppedAt?: number;
+    };
+    targets?: Array<{
+        targetId: string;
+        name: string;
+        passed: number;
+        failed: number;
+        avgLatency: number;
+        totalCost: number;
+    }>;
+    evaluators?: Array<{
+        evaluatorId: string;
+        name: string;
+        passed: number;
+        failed: number;
+        passRate: number;
+        avgScore?: number;
+    }>;
+    totalPassed?: number;
+    totalFailed?: number;
+    passRate?: number;
+    totalCost?: number;
+};
+/**
+ * Options for running a platform evaluation
+ */
+type RunEvaluationOptions = {
+    /**
+     * Polling interval in milliseconds (default: 2000)
+     */
+    pollInterval?: number;
+    /**
+     * Maximum time to wait for completion in milliseconds (default: 600000 = 10 minutes)
+     */
+    timeout?: number;
+    /**
+     * Callback for progress updates
+     */
+    onProgress?: (progress: number, total: number) => void;
+};
+/**
+ * Final result of a platform evaluation run
+ */
+type EvaluationRunResult = {
+    runId: string;
+    status: "completed" | "failed" | "stopped";
+    passed: number;
+    failed: number;
+    passRate: number;
+    duration: number;
+    runUrl: string;
+    summary: EvaluationRunSummary;
+    /**
+     * Print a CI-friendly summary of the results
+     * @param exitOnFailure - If true (default), calls process.exit(1) when there are failures
+     */
+    printSummary: (exitOnFailure?: boolean) => void;
+};
+
+/**
+ * EvaluationFacade - Entry point for the evaluation API
+ *
+ * Provides:
+ * - `init()` method to create evaluation sessions (SDK-defined evaluations)
+ * - `run()` method to execute platform-configured evaluations (Evaluations V3)
+ */
+
+type EvaluationFacadeConfig = {
+    langwatchApiClient: LangwatchApiClient;
+    endpoint: string;
+    apiKey: string;
+    logger: Logger;
+};
+/**
+ * Facade for creating evaluation sessions and running platform-configured evaluations
+ */
+declare class EvaluationFacade {
+    private readonly config;
+    constructor(config: EvaluationFacadeConfig);
+    /**
+     * Initialize a new evaluation session (SDK-defined)
+     *
+     * @param name - Name of the experiment (used as slug)
+     * @param options - Optional configuration
+     * @returns An initialized Evaluation instance
+     *
+     * @example
+     * ```typescript
+     * const evaluation = await langwatch.evaluation.init('my-experiment');
+     *
+     * await evaluation.run(dataset, async ({ item, index }) => {
+     *   const response = await myAgent(item.question);
+     *   evaluation.log('accuracy', { index, score: 0.95 });
+     * });
+     * ```
+     */
+    init(name: string, options?: EvaluationInitOptions): Promise<Evaluation>;
+    /**
+     * Run a platform-configured evaluation (Evaluations V3)
+     *
+     * This runs an evaluation that was configured in the LangWatch platform.
+     * The method automatically prints a summary and exits with code 1 on failure
+     * (unless `exitOnFailure: false` is passed).
+     *
+     * @param slug - The slug of the evaluation (found in the evaluation URL)
+     * @param options - Optional configuration
+     * @returns The evaluation results including pass rate and summary
+     *
+     * @example
+     * ```typescript
+     * import { LangWatch } from "langwatch";
+     *
+     * const langwatch = new LangWatch();
+     *
+     * const result = await langwatch.evaluation.run("my-evaluation-slug");
+     * result.printSummary();
+     * ```
+     */
+    run(slug: string, options?: RunEvaluationOptions): Promise<EvaluationRunResult>;
+    /**
+     * Run an evaluation and wait for completion using polling
+     */
+    private runWithPolling;
+    /**
+     * Start an evaluation run
+     */
+    private startRun;
+    /**
+     * Get the status of a run
+     */
+    private getRunStatus;
+    /**
+     * Build the result object from API response
+     */
+    private buildResult;
+    /**
+     * Print a CI-friendly summary of the evaluation results
+     */
+    private printSummary;
+    private sleep;
+    /**
+     * Replace the domain of a URL with a new base URL, preserving the path
+     */
+    private replaceUrlDomain;
+}
+
+/**
+ * Errors for the Evaluation API
+ */
+/**
+ * Base error for evaluation-related issues
+ */
+declare class EvaluationError extends Error {
+    constructor(message: string);
+}
+/**
+ * Thrown when initialization fails
+ */
+declare class EvaluationInitError extends EvaluationError {
+    readonly cause?: Error | undefined;
+    constructor(message: string, cause?: Error | undefined);
+}
+/**
+ * Thrown when API calls fail
+ */
+declare class EvaluationApiError extends EvaluationError {
+    readonly statusCode?: number | undefined;
+    readonly cause?: Error | undefined;
+    constructor(message: string, statusCode?: number | undefined, cause?: Error | undefined);
+}
+/**
+ * Thrown when target metadata conflicts
+ */
+declare class TargetMetadataConflictError extends EvaluationError {
+    readonly targetName: string;
+    readonly existingMetadata: Record<string, unknown>;
+    readonly newMetadata: Record<string, unknown>;
+    constructor(targetName: string, existingMetadata: Record<string, unknown>, newMetadata: Record<string, unknown>);
+}
+/**
+ * Thrown when an evaluator call fails
+ */
+declare class EvaluatorError extends EvaluationError {
+    readonly evaluatorSlug: string;
+    readonly cause?: Error | undefined;
+    constructor(evaluatorSlug: string, message: string, cause?: Error | undefined);
+}
+
 interface GetTraceParams {
     includeSpans?: boolean;
 }
@@ -376,6 +1037,8 @@ declare class LangWatch {
     private readonly config;
     readonly prompts: PromptsFacade;
     readonly traces: TracesFacade;
+    readonly evaluation: EvaluationFacade;
+    readonly datasets: DatasetsFacade;
     constructor(options?: LangWatchConstructorOptions);
     get apiClient(): LangwatchApiClient;
 }
@@ -385,4 +1048,4 @@ declare const logger: {
     NoOpLogger: typeof NoOpLogger;
 };
 
-export { FetchPolicy, type GetPromptOptions, LangWatch, logger };
+export { type EvaluateOptions, Evaluation, EvaluationApiError, EvaluationError, EvaluationFacade, EvaluationInitError, type EvaluationInitOptions, type EvaluationResult, type EvaluationStatus, EvaluatorError, FetchPolicy, type GetPromptOptions, LangWatch, type LogOptions, type RunCallback, type RunContext, type RunOptions, type TargetInfo, type TargetMetadata, TargetMetadataConflictError, type TargetType, logger };