langsmith 0.7.7 → 0.7.9

package/dist/_openapi_client/resources/datasets/group.d.ts

@@ -0,0 +1,160 @@
+import { APIResource } from '../../core/resource.js';
+import * as RunsAPI from './runs.js';
+import { APIPromise } from '../../core/api-promise.js';
+import { RequestOptions } from '../../internal/request-options.js';
+export declare class Group extends APIResource {
+    /**
+     * Fetch examples for a dataset, and fetch the runs for each example if they are
+     * associated with the given session_ids.
+     */
+    runs(datasetID: string, body: GroupRunsParams, options?: RequestOptions): APIPromise<GroupRunsResponse>;
+}
+/**
+ * Response for grouped comparison view of dataset examples.
+ *
+ * Returns dataset examples grouped by a run metadata value (e.g., model='gpt-4').
+ * Optional filters are applied to all runs before grouping.
+ *
+ * Shows:
+ *
+ * - Which examples were executed with each metadata value
+ * - Per-session aggregate statistics for runs on those examples
+ * - The actual example data with their associated runs
+ *
+ * Used for comparing how different sessions performed on the same set of examples.
+ */
+export interface GroupRunsResponse {
+    groups: Array<GroupRunsResponse.Group>;
+}
+export declare namespace GroupRunsResponse {
+    /**
+     * Group of examples with a specific metadata value across multiple sessions.
+     *
+     * Extends RunGroupBase with:
+     *
+     * - group_key: metadata value that defines this group
+     * - sessions: per-session stats for runs matching this metadata value
+     * - examples: shared examples across all sessions (intersection logic) with flat
+     *   array of runs (each run has session_id field for frontend to determine column)
+     * - example_count: unique example count (pagination-aware, same across all
+     *   sessions due to intersection)
+     *
+     * Inherited from RunGroupBase:
+     *
+     * - filter: metadata filter for this group (e.g., "and(eq(is_root, true),
+     *   and(eq(metadata_key, 'model'), eq(metadata_value, 'gpt-4')))")
+     * - count: total run count across all sessions (includes duplicate runs)
+     * - total_tokens, total_cost: aggregate across sessions
+     * - min_start_time, max_start_time: time range across sessions
+     * - latency_p50, latency_p99: aggregate latency stats across sessions
+     * - feedback_stats: weighted average feedback across sessions
+     *
+     * Additional aggregate stats:
+     *
+     * - prompt_tokens, completion_tokens: separate token counts
+     * - prompt_cost, completion_cost: separate costs
+     * - error_rate: average error rate
+     */
+    interface Group {
+        example_count: number;
+        examples: Array<RunsAPI.ExampleWithRunsCh>;
+        filter: string;
+        group_key: string | number;
+        sessions: Array<Group.Session>;
+        completion_cost?: string | null;
+        completion_tokens?: number | null;
+        count?: number | null;
+        error_rate?: number | null;
+        feedback_stats?: {
+            [key: string]: unknown;
+        } | null;
+        latency_p50?: number | null;
+        latency_p99?: number | null;
+        max_start_time?: string | null;
+        min_start_time?: string | null;
+        prompt_cost?: string | null;
+        prompt_tokens?: number | null;
+        total_cost?: string | null;
+        total_tokens?: number | null;
+    }
+    namespace Group {
+        /**
+         * TracerSession stats filtered to runs matching a specific metadata value.
+         *
+         * Extends TracerSession with:
+         *
+         * - example_count: unique examples (vs run_count = total runs including
+         *   duplicates)
+         * - filter: ClickHouse filter for fetching runs in this session/group
+         * - min/max_start_time: time range for runs in this session/group
+         */
+        interface Session {
+            id: string;
+            filter: string;
+            tenant_id: string;
+            completion_cost?: string | null;
+            completion_tokens?: number | null;
+            default_dataset_id?: string | null;
+            description?: string | null;
+            end_time?: string | null;
+            error_rate?: number | null;
+            example_count?: number | null;
+            experiment_progress?: Session.ExperimentProgress | null;
+            extra?: {
+                [key: string]: unknown;
+            } | null;
+            feedback_stats?: {
+                [key: string]: unknown;
+            } | null;
+            first_token_p50?: number | null;
+            first_token_p99?: number | null;
+            last_run_start_time?: string | null;
+            last_run_start_time_live?: string | null;
+            latency_p50?: number | null;
+            latency_p99?: number | null;
+            max_start_time?: string | null;
+            min_start_time?: string | null;
+            name?: string;
+            prompt_cost?: string | null;
+            prompt_tokens?: number | null;
+            reference_dataset_id?: string | null;
+            run_count?: number | null;
+            run_facets?: Array<{
+                [key: string]: unknown;
+            }> | null;
+            session_feedback_stats?: {
+                [key: string]: unknown;
+            } | null;
+            start_time?: string;
+            streaming_rate?: number | null;
+            test_run_number?: number | null;
+            total_cost?: string | null;
+            total_tokens?: number | null;
+            trace_tier?: 'longlived' | 'shortlived' | null;
+        }
+        namespace Session {
+            interface ExperimentProgress {
+                evaluator_progress: {
+                    [key: string]: number;
+                };
+                expected_run_count: number;
+                run_progress: number;
+            }
+        }
+    }
+}
+export interface GroupRunsParams {
+    group_by: 'run_metadata' | 'example_metadata';
+    metadata_key: string;
+    session_ids: Array<string>;
+    filters?: {
+        [key: string]: Array<string>;
+    } | null;
+    limit?: number;
+    offset?: number;
+    per_group_limit?: number;
+    preview?: boolean;
+}
+export declare namespace Group {
+    export { type GroupRunsResponse as GroupRunsResponse, type GroupRunsParams as GroupRunsParams };
+}

package/dist/_openapi_client/resources/datasets/group.js

@@ -0,0 +1,13 @@
+// @ts-nocheck
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+import { APIResource } from '../../core/resource.js';
+import { path } from '../../internal/utils/path.js';
+export class Group extends APIResource {
+    /**
+     * Fetch examples for a dataset, and fetch the runs for each example if they are
+     * associated with the given session_ids.
+     */
+    runs(datasetID, body, options) {
+        return this._client.post(path `/api/v1/datasets/${datasetID}/group/runs`, { body, ...options });
+    }
+}

package/dist/_openapi_client/resources/datasets/runs.cjs

@@ -0,0 +1,29 @@
+"use strict";
+// @ts-nocheck
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Runs = void 0;
+const resource_js_1 = require("../../core/resource.cjs");
+const path_js_1 = require("../../internal/utils/path.cjs");
+class Runs extends resource_js_1.APIResource {
+    /**
+     * Fetch examples for a dataset, and fetch the runs for each example if they are
+     * associated with the given session_ids.
+     */
+    create(datasetID, params, options) {
+        const { format, ...body } = params;
+        return this._client.post((0, path_js_1.path) `/api/v1/datasets/${datasetID}/runs`, {
+            query: { format },
+            body,
+            ...options,
+        });
+    }
+    /**
+     * Fetch the number of regressions/improvements for each example in a dataset,
+     * between sessions[0] and sessions[1].
+     */
+    delta(datasetID, body, options) {
+        return this._client.post((0, path_js_1.path) `/api/v1/datasets/${datasetID}/runs/delta`, { body, ...options });
+    }
+}
+exports.Runs = Runs;

package/dist/_openapi_client/resources/datasets/runs.d.ts

@@ -0,0 +1,216 @@
+import { APIResource } from '../../core/resource.js';
+import * as RunsAPI_ from '../runs/runs.js';
+import { APIPromise } from '../../core/api-promise.js';
+import { RequestOptions } from '../../internal/request-options.js';
+export declare class Runs extends APIResource {
+    /**
+     * Fetch examples for a dataset, and fetch the runs for each example if they are
+     * associated with the given session_ids.
+     */
+    create(datasetID: string, params: RunCreateParams, options?: RequestOptions): APIPromise<RunCreateResponse | null>;
+    /**
+     * Fetch the number of regressions/improvements for each example in a dataset,
+     * between sessions[0] and sessions[1].
+     */
+    delta(datasetID: string, body: RunDeltaParams, options?: RequestOptions): APIPromise<SessionFeedbackDelta>;
+}
+/**
+ * Example schema with list of runs from ClickHouse.
+ *
+ * For non-grouped endpoint (/datasets/{dataset_id}/runs): runs from single
+ * session. For grouped endpoint (/datasets/{dataset_id}/group/runs): flat array of
+ * runs from all sessions, where each run has a session_id field for frontend to
+ * determine column placement.
+ */
+export interface ExampleWithRunsCh {
+    id: string;
+    dataset_id: string;
+    inputs: {
+        [key: string]: unknown;
+    };
+    name: string;
+    runs: Array<ExampleWithRunsCh.Run>;
+    attachment_urls?: {
+        [key: string]: unknown;
+    } | null;
+    created_at?: string;
+    metadata?: {
+        [key: string]: unknown;
+    } | null;
+    modified_at?: string | null;
+    outputs?: {
+        [key: string]: unknown;
+    } | null;
+    source_run_id?: string | null;
+}
+export declare namespace ExampleWithRunsCh {
+    /**
+     * Run schema for comparison view.
+     */
+    interface Run {
+        id: string;
+        name: string;
+        /**
+         * Enum for run types.
+         */
+        run_type: RunsAPI_.RunTypeEnum;
+        session_id: string;
+        status: string;
+        trace_id: string;
+        app_path?: string | null;
+        completion_cost?: string | null;
+        completion_tokens?: number | null;
+        dotted_order?: string | null;
+        end_time?: string | null;
+        error?: string | null;
+        events?: Array<{
+            [key: string]: unknown;
+        }> | null;
+        execution_order?: number;
+        extra?: {
+            [key: string]: unknown;
+        } | null;
+        feedback_stats?: {
+            [key: string]: {
+                [key: string]: unknown;
+            };
+        } | null;
+        first_token_time?: string | null;
+        inputs?: {
+            [key: string]: unknown;
+        } | null;
+        inputs_preview?: string | null;
+        inputs_s3_urls?: {
+            [key: string]: unknown;
+        } | null;
+        manifest_id?: string | null;
+        manifest_s3_id?: string | null;
+        outputs?: {
+            [key: string]: unknown;
+        } | null;
+        outputs_preview?: string | null;
+        outputs_s3_urls?: {
+            [key: string]: unknown;
+        } | null;
+        parent_run_id?: string | null;
+        prompt_cost?: string | null;
+        prompt_tokens?: number | null;
+        reference_example_id?: string | null;
+        s3_urls?: {
+            [key: string]: unknown;
+        } | null;
+        serialized?: {
+            [key: string]: unknown;
+        } | null;
+        start_time?: string;
+        tags?: Array<string> | null;
+        total_cost?: string | null;
+        total_tokens?: number | null;
+        trace_max_start_time?: string | null;
+        trace_min_start_time?: string | null;
+    }
+}
+export interface QueryExampleSchemaWithRuns {
+    session_ids: Array<string>;
+    comparative_experiment_id?: string | null;
+    example_ids?: Array<string> | null;
+    filters?: {
+        [key: string]: Array<string>;
+    } | null;
+    include_annotator_detail?: boolean;
+    limit?: number;
+    offset?: number;
+    preview?: boolean;
+    sort_params?: SortParamsForRunsComparisonView | null;
+}
+export interface QueryFeedbackDelta {
+    baseline_session_id: string;
+    comparison_session_ids: Array<string>;
+    feedback_key: string;
+    comparative_experiment_id?: string | null;
+    filters?: {
+        [key: string]: Array<string>;
+    } | null;
+    limit?: number;
+    offset?: number;
+}
+/**
+ * List of feedback keys with number of improvements and regressions for each.
+ */
+export interface SessionFeedbackDelta {
+    feedback_deltas: {
+        [key: string]: SessionFeedbackDelta.FeedbackDeltas;
+    };
+}
+export declare namespace SessionFeedbackDelta {
+    /**
+     * Feedback key with number of improvements and regressions.
+     */
+    interface FeedbackDeltas {
+        improved_examples: Array<string>;
+        regressed_examples: Array<string>;
+    }
+}
+export interface SortParamsForRunsComparisonView {
+    sort_by: string;
+    sort_order?: 'ASC' | 'DESC';
+}
+export type RunCreateResponse = Array<ExampleWithRunsCh>;
+export interface RunCreateParams {
+    /**
+     * Body param
+     */
+    session_ids: Array<string>;
+    /**
+     * Query param: Response format, e.g., 'csv'
+     */
+    format?: 'csv' | null;
+    /**
+     * Body param
+     */
+    comparative_experiment_id?: string | null;
+    /**
+     * Body param
+     */
+    example_ids?: Array<string> | null;
+    /**
+     * Body param
+     */
+    filters?: {
+        [key: string]: Array<string>;
+    } | null;
+    /**
+     * Body param
+     */
+    include_annotator_detail?: boolean;
+    /**
+     * Body param
+     */
+    limit?: number | null;
+    /**
+     * Body param
+     */
+    offset?: number;
+    /**
+     * Body param
+     */
+    preview?: boolean;
+    /**
+     * Body param
+     */
+    sort_params?: SortParamsForRunsComparisonView | null;
+}
+export interface RunDeltaParams {
+    baseline_session_id: string;
+    comparison_session_ids: Array<string>;
+    feedback_key: string;
+    comparative_experiment_id?: string | null;
+    filters?: {
+        [key: string]: Array<string>;
+    } | null;
+    limit?: number;
+    offset?: number;
+}
+export declare namespace Runs {
+    export { type ExampleWithRunsCh as ExampleWithRunsCh, type QueryExampleSchemaWithRuns as QueryExampleSchemaWithRuns, type QueryFeedbackDelta as QueryFeedbackDelta, type SessionFeedbackDelta as SessionFeedbackDelta, type SortParamsForRunsComparisonView as SortParamsForRunsComparisonView, type RunCreateResponse as RunCreateResponse, type RunCreateParams as RunCreateParams, type RunDeltaParams as RunDeltaParams, };
+}

package/dist/_openapi_client/resources/datasets/runs.js

@@ -0,0 +1,25 @@
+// @ts-nocheck
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+import { APIResource } from '../../core/resource.js';
+import { path } from '../../internal/utils/path.js';
+export class Runs extends APIResource {
+    /**
+     * Fetch examples for a dataset, and fetch the runs for each example if they are
+     * associated with the given session_ids.
+     */
+    create(datasetID, params, options) {
+        const { format, ...body } = params;
+        return this._client.post(path `/api/v1/datasets/${datasetID}/runs`, {
+            query: { format },
+            body,
+            ...options,
+        });
+    }
+    /**
+     * Fetch the number of regressions/improvements for each example in a dataset,
+     * between sessions[0] and sessions[1].
+     */
+    delta(datasetID, body, options) {
+        return this._client.post(path `/api/v1/datasets/${datasetID}/runs/delta`, { body, ...options });
+    }
+}

package/dist/_openapi_client/resources/datasets/share.cjs

@@ -0,0 +1,32 @@
+"use strict";
+// @ts-nocheck
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Share = void 0;
+const resource_js_1 = require("../../core/resource.cjs");
+const path_js_1 = require("../../internal/utils/path.cjs");
+class Share extends resource_js_1.APIResource {
+    /**
+     * Share a dataset.
+     */
+    create(datasetID, params = {}, options) {
+        const { share_projects } = params ?? {};
+        return this._client.put((0, path_js_1.path) `/api/v1/datasets/${datasetID}/share`, {
+            query: { share_projects },
+            ...options,
+        });
+    }
+    /**
+     * Get the state of sharing a dataset
+     */
+    retrieve(datasetID, options) {
+        return this._client.get((0, path_js_1.path) `/api/v1/datasets/${datasetID}/share`, options);
+    }
+    /**
+     * Unshare a dataset.
+     */
+    deleteAll(datasetID, options) {
+        return this._client.delete((0, path_js_1.path) `/api/v1/datasets/${datasetID}/share`, options);
+    }
+}
+exports.Share = Share;

package/dist/_openapi_client/resources/datasets/share.d.ts

@@ -0,0 +1,28 @@
+import { APIResource } from '../../core/resource.js';
+import { APIPromise } from '../../core/api-promise.js';
+import { RequestOptions } from '../../internal/request-options.js';
+export declare class Share extends APIResource {
+    /**
+     * Share a dataset.
+     */
+    create(datasetID: string, params?: ShareCreateParams | null | undefined, options?: RequestOptions): APIPromise<DatasetShareSchema>;
+    /**
+     * Get the state of sharing a dataset
+     */
+    retrieve(datasetID: string, options?: RequestOptions): APIPromise<DatasetShareSchema | null>;
+    /**
+     * Unshare a dataset.
+     */
+    deleteAll(datasetID: string, options?: RequestOptions): APIPromise<unknown>;
+}
+export interface DatasetShareSchema {
+    dataset_id: string;
+    share_token: string;
+}
+export type ShareDeleteAllResponse = unknown;
+export interface ShareCreateParams {
+    share_projects?: boolean;
+}
+export declare namespace Share {
+    export { type DatasetShareSchema as DatasetShareSchema, type ShareDeleteAllResponse as ShareDeleteAllResponse, type ShareCreateParams as ShareCreateParams, };
+}

package/dist/_openapi_client/resources/datasets/share.js

@@ -0,0 +1,28 @@
+// @ts-nocheck
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+import { APIResource } from '../../core/resource.js';
+import { path } from '../../internal/utils/path.js';
+export class Share extends APIResource {
+    /**
+     * Share a dataset.
+     */
+    create(datasetID, params = {}, options) {
+        const { share_projects } = params ?? {};
+        return this._client.put(path `/api/v1/datasets/${datasetID}/share`, {
+            query: { share_projects },
+            ...options,
+        });
+    }
+    /**
+     * Get the state of sharing a dataset
+     */
+    retrieve(datasetID, options) {
+        return this._client.get(path `/api/v1/datasets/${datasetID}/share`, options);
+    }
+    /**
+     * Unshare a dataset.
+     */
+    deleteAll(datasetID, options) {
+        return this._client.delete(path `/api/v1/datasets/${datasetID}/share`, options);
+    }
+}

package/dist/_openapi_client/resources/datasets/splits.cjs

@@ -0,0 +1,22 @@
+"use strict";
+// @ts-nocheck
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Splits = void 0;
+const resource_js_1 = require("../../core/resource.cjs");
+const path_js_1 = require("../../internal/utils/path.cjs");
+class Splits extends resource_js_1.APIResource {
+    /**
+     * Update Dataset Splits
+     */
+    create(datasetID, body, options) {
+        return this._client.put((0, path_js_1.path) `/api/v1/datasets/${datasetID}/splits`, { body, ...options });
+    }
+    /**
+     * Get Dataset Splits
+     */
+    retrieve(datasetID, query = {}, options) {
+        return this._client.get((0, path_js_1.path) `/api/v1/datasets/${datasetID}/splits`, { query, ...options });
+    }
+}
+exports.Splits = Splits;

package/dist/_openapi_client/resources/datasets/splits.d.ts

@@ -0,0 +1,30 @@
+import { APIResource } from '../../core/resource.js';
+import { APIPromise } from '../../core/api-promise.js';
+import { RequestOptions } from '../../internal/request-options.js';
+export declare class Splits extends APIResource {
+    /**
+     * Update Dataset Splits
+     */
+    create(datasetID: string, body: SplitCreateParams, options?: RequestOptions): APIPromise<SplitCreateResponse>;
+    /**
+     * Get Dataset Splits
+     */
+    retrieve(datasetID: string, query?: SplitRetrieveParams | null | undefined, options?: RequestOptions): APIPromise<SplitRetrieveResponse>;
+}
+export type SplitCreateResponse = Array<string>;
+export type SplitRetrieveResponse = Array<string>;
+export interface SplitCreateParams {
+    examples: Array<string>;
+    split_name: string;
+    remove?: boolean;
+}
+export interface SplitRetrieveParams {
+    /**
+     * Only modifications made on or before this time are included. If None, the latest
+     * version of the dataset is used.
+     */
+    as_of?: (string & {}) | string;
+}
+export declare namespace Splits {
+    export { type SplitCreateResponse as SplitCreateResponse, type SplitRetrieveResponse as SplitRetrieveResponse, type SplitCreateParams as SplitCreateParams, type SplitRetrieveParams as SplitRetrieveParams, };
+}

package/dist/_openapi_client/resources/datasets/splits.js

@@ -0,0 +1,18 @@
+// @ts-nocheck
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+import { APIResource } from '../../core/resource.js';
+import { path } from '../../internal/utils/path.js';
+export class Splits extends APIResource {
+    /**
+     * Update Dataset Splits
+     */
+    create(datasetID, body, options) {
+        return this._client.put(path `/api/v1/datasets/${datasetID}/splits`, { body, ...options });
+    }
+    /**
+     * Get Dataset Splits
+     */
+    retrieve(datasetID, query = {}, options) {
+        return this._client.get(path `/api/v1/datasets/${datasetID}/splits`, { query, ...options });
+    }
+}

package/dist/_openapi_client/resources/datasets/versions.cjs

@@ -0,0 +1,23 @@
+"use strict";
+// @ts-nocheck
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Versions = void 0;
+const resource_js_1 = require("../../core/resource.cjs");
+const pagination_js_1 = require("../../core/pagination.cjs");
+const path_js_1 = require("../../internal/utils/path.cjs");
+class Versions extends resource_js_1.APIResource {
+    /**
+     * Get dataset versions.
+     */
+    list(datasetID, query = {}, options) {
+        return this._client.getAPIList((0, path_js_1.path) `/api/v1/datasets/${datasetID}/versions`, (pagination_js_1.OffsetPaginationTopLevelArray), { query, ...options });
+    }
+    /**
+     * Get diff between two dataset versions.
+     */
+    retrieveDiff(datasetID, query, options) {
+        return this._client.get((0, path_js_1.path) `/api/v1/datasets/${datasetID}/versions/diff`, { query, ...options });
+    }
+}
+exports.Versions = Versions;

package/dist/_openapi_client/resources/datasets/versions.d.ts

@@ -0,0 +1,36 @@
+import { APIResource } from '../../core/resource.js';
+import * as DatasetsAPI from './datasets.js';
+import { DatasetVersionsOffsetPaginationTopLevelArray } from './datasets.js';
+import { APIPromise } from '../../core/api-promise.js';
+import { type OffsetPaginationTopLevelArrayParams, PagePromise } from '../../core/pagination.js';
+import { RequestOptions } from '../../internal/request-options.js';
+export declare class Versions extends APIResource {
+    /**
+     * Get dataset versions.
+     */
+    list(datasetID: string, query?: VersionListParams | null | undefined, options?: RequestOptions): PagePromise<DatasetVersionsOffsetPaginationTopLevelArray, DatasetsAPI.DatasetVersion>;
+    /**
+     * Get diff between two dataset versions.
+     */
+    retrieveDiff(datasetID: string, query: VersionRetrieveDiffParams, options?: RequestOptions): APIPromise<VersionRetrieveDiffResponse>;
+}
+/**
+ * Dataset diff schema.
+ */
+export interface VersionRetrieveDiffResponse {
+    examples_added: Array<string>;
+    examples_modified: Array<string>;
+    examples_removed: Array<string>;
+}
+export interface VersionListParams extends OffsetPaginationTopLevelArrayParams {
+    example?: string | null;
+    search?: string | null;
+}
+export interface VersionRetrieveDiffParams {
+    from_version: (string & {}) | string;
+    to_version: (string & {}) | string;
+}
+export declare namespace Versions {
+    export { type VersionRetrieveDiffResponse as VersionRetrieveDiffResponse, type VersionListParams as VersionListParams, type VersionRetrieveDiffParams as VersionRetrieveDiffParams, };
+}
+export { type DatasetVersionsOffsetPaginationTopLevelArray };