langsmith 0.7.13 → 0.7.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/dist/_openapi_client/client.cjs +1 -1
  2. package/dist/_openapi_client/client.d.ts +2 -2
  3. package/dist/_openapi_client/client.js +1 -1
  4. package/dist/_openapi_client/resources/datasets/datasets.cjs +5 -14
  5. package/dist/_openapi_client/resources/datasets/datasets.d.ts +6 -10
  6. package/dist/_openapi_client/resources/datasets/datasets.js +5 -14
  7. package/dist/_openapi_client/resources/datasets/experiment-runs.cjs +18 -0
  8. package/dist/_openapi_client/resources/datasets/experiment-runs.d.ts +109 -0
  9. package/dist/_openapi_client/resources/datasets/experiment-runs.js +14 -0
  10. package/dist/_openapi_client/resources/datasets/runs.cjs +0 -7
  11. package/dist/_openapi_client/resources/datasets/runs.d.ts +1 -45
  12. package/dist/_openapi_client/resources/datasets/runs.js +0 -7
  13. package/dist/_openapi_client/resources/index.cjs +1 -1
  14. package/dist/_openapi_client/resources/index.d.ts +1 -1
  15. package/dist/_openapi_client/resources/index.js +1 -1
  16. package/dist/_openapi_client/resources/runs/runs.d.ts +1 -1
  17. package/dist/_openapi_client/resources/sandboxes/boxes.d.ts +7 -0
  18. package/dist/_openapi_client/resources/sandboxes/sandboxes.d.ts +1 -0
  19. package/dist/_openapi_client/resources/sessions.cjs +53 -0
  20. package/dist/_openapi_client/resources/{sessions/sessions.d.ts → sessions.d.ts} +13 -51
  21. package/dist/_openapi_client/resources/sessions.js +49 -0
  22. package/dist/client.cjs +40 -0
  23. package/dist/client.d.ts +28 -0
  24. package/dist/client.js +40 -0
  25. package/dist/experimental/anthropic/context.cjs +5 -1
  26. package/dist/experimental/anthropic/context.js +5 -1
  27. package/dist/index.cjs +1 -1
  28. package/dist/index.d.ts +1 -1
  29. package/dist/index.js +1 -1
  30. package/package.json +1 -1
  31. package/dist/_openapi_client/resources/datasets/experiments.cjs +0 -16
  32. package/dist/_openapi_client/resources/datasets/experiments.d.ts +0 -23
  33. package/dist/_openapi_client/resources/datasets/experiments.js +0 -12
  34. package/dist/_openapi_client/resources/datasets/group.cjs +0 -17
  35. package/dist/_openapi_client/resources/datasets/group.d.ts +0 -160
  36. package/dist/_openapi_client/resources/datasets/group.js +0 -13
  37. package/dist/_openapi_client/resources/sessions/insights.cjs +0 -54
  38. package/dist/_openapi_client/resources/sessions/insights.d.ts +0 -246
  39. package/dist/_openapi_client/resources/sessions/insights.js +0 -50
  40. package/dist/_openapi_client/resources/sessions/sessions.cjs +0 -109
  41. package/dist/_openapi_client/resources/sessions/sessions.js +0 -72
@@ -0,0 +1,49 @@
1
+ // @ts-nocheck
2
+ // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
3
+ import { APIResource } from '../core/resource.js';
4
+ import { OffsetPaginationTopLevelArray, } from '../core/pagination.js';
5
+ import { buildHeaders } from '../internal/headers.js';
6
+ import { path } from '../internal/utils/path.js';
7
+ export class Sessions extends APIResource {
8
+ /**
9
+ * Create a new project.
10
+ */
11
+ create(params, options) {
12
+ const { upsert, ...body } = params;
13
+ return this._client.post('/api/v1/sessions', { query: { upsert }, body, ...options });
14
+ }
15
+ /**
16
+ * Get a specific project.
17
+ */
18
+ retrieve(project_id, params = {}, options) {
19
+ const { accept, ...query } = params ?? {};
20
+ return this._client.get(path `/api/v1/sessions/${project_id}`, {
21
+ query,
22
+ ...options,
23
+ headers: buildHeaders([{ ...(accept != null ? { accept: accept } : undefined) }, options?.headers]),
24
+ });
25
+ }
26
+ /**
27
+ * Update a project.
28
+ */
29
+ update(project_id, body, options) {
30
+ return this._client.patch(path `/api/v1/sessions/${project_id}`, { body, ...options });
31
+ }
32
+ /**
33
+ * List all projects.
34
+ */
35
+ list(params = {}, options) {
36
+ const { accept, ...query } = params ?? {};
37
+ return this._client.getAPIList('/api/v1/sessions', (OffsetPaginationTopLevelArray), {
38
+ query,
39
+ ...options,
40
+ headers: buildHeaders([{ ...(accept != null ? { accept: accept } : undefined) }, options?.headers]),
41
+ });
42
+ }
43
+ /**
44
+ * Delete a specific project.
45
+ */
46
+ delete(project_id, options) {
47
+ return this._client.delete(path `/api/v1/sessions/${project_id}`, options);
48
+ }
49
+ }
package/dist/client.cjs CHANGED
@@ -543,6 +543,12 @@ class Client {
543
543
  writable: true,
544
544
  value: void 0
545
545
  });
546
+ Object.defineProperty(this, "anonymizer", {
547
+ enumerable: true,
548
+ configurable: true,
549
+ writable: true,
550
+ value: void 0
551
+ });
546
552
  Object.defineProperty(this, "omitTracedRuntimeInfo", {
547
553
  enumerable: true,
548
554
  configurable: true,
@@ -794,6 +800,7 @@ class Client {
794
800
  this.hideOutputs =
795
801
  config.hideOutputs ?? config.anonymizer ?? defaultConfig.hideOutputs;
796
802
  this.hideMetadata = config.hideMetadata ?? defaultConfig.hideMetadata;
803
+ this.anonymizer = config.anonymizer;
797
804
  this.omitTracedRuntimeInfo = config.omitTracedRuntimeInfo ?? false;
798
805
  this.autoBatchTracing = config.autoBatchTracing ?? this.autoBatchTracing;
799
806
  this.autoBatchQueue = new AutoBatchQueue(maxMemory);
@@ -975,6 +982,14 @@ class Client {
975
982
  get sandboxes() {
976
983
  return this.openAPIClient.sandboxes;
977
984
  }
985
+ /** Access the projects resource. */
986
+ get projects() {
987
+ return this.openAPIClient.sessions;
988
+ }
989
+ /** Access the v2 datasets resource (experimentRuns, etc.). */
990
+ get datasets() {
991
+ return this.openAPIClient.datasets;
992
+ }
978
993
  async processInputs(inputs) {
979
994
  if (this.hideInputs === false) {
980
995
  return inputs;
@@ -1011,6 +1026,25 @@ class Client {
1011
1026
  }
1012
1027
  return metadata;
1013
1028
  }
1029
+ /**
1030
+ * Apply the configured anonymizer to a run's error string.
1031
+ *
1032
+ * Unlike inputs/outputs, `error` is a plain string (an exception message or
1033
+ * traceback) that can carry credentials the user never explicitly logged --
1034
+ * e.g. an HTTP-client error whose message embeds an `Authorization` header.
1035
+ * The anonymizer is typed `(KVMap) => KVMap`, so the string is wrapped as
1036
+ * `{ error }`, scrubbed, and unwrapped. Mirrors the Python SDK's
1037
+ * `Client._hide_run_error`.
1038
+ *
1039
+ * TODO: Update anonymizer to always nest inputs/outputs/error for consistency
1040
+ */
1041
+ async processError(error) {
1042
+ if (this.anonymizer == null) {
1043
+ return error;
1044
+ }
1045
+ const result = await this.anonymizer({ error });
1046
+ return typeof result?.error === "string" ? result.error : error;
1047
+ }
1014
1048
  /**
1015
1049
  * Filter content from new_token events to prevent streaming LLM output
1016
1050
  * from being uploaded via events.
@@ -1037,6 +1071,9 @@ class Client {
1037
1071
  if (runParams.outputs !== undefined) {
1038
1072
  runParams.outputs = await this.processOutputs(runParams.outputs);
1039
1073
  }
1074
+ if (runParams.error !== undefined) {
1075
+ runParams.error = await this.processError(runParams.error);
1076
+ }
1040
1077
  if (runParams.extra != null && "metadata" in runParams.extra) {
1041
1078
  runParams.extra = {
1042
1079
  ...runParams.extra,
@@ -1893,6 +1930,9 @@ class Client {
1893
1930
  if (run.outputs) {
1894
1931
  run.outputs = await this.processOutputs(run.outputs);
1895
1932
  }
1933
+ if (run.error) {
1934
+ run.error = await this.processError(run.error);
1935
+ }
1896
1936
  if (run.extra != null && "metadata" in run.extra) {
1897
1937
  run.extra = {
1898
1938
  ...run.extra,
package/dist/client.d.ts CHANGED
@@ -6,6 +6,8 @@ import { EvaluationResult, EvaluationResults } from "./evaluation/evaluator.js";
6
6
  import { OnlineEvaluators } from "./_openapi_client/resources/online-evaluators.js";
7
7
  import { Runs as OpenAPIRuns } from "./_openapi_client/resources/runs.js";
8
8
  import { Sandboxes } from "./_openapi_client/resources/sandboxes/sandboxes.js";
9
+ import { Datasets } from "./_openapi_client/resources/datasets/datasets.js";
10
+ import { Sessions } from "./_openapi_client/resources/sessions.js";
9
11
  import { PromptCache } from "./utils/prompt_cache/index.js";
10
12
  import { ProfileAuth } from "./utils/profiles.js";
11
13
  export interface ClientConfig {
@@ -14,6 +16,14 @@ export interface ClientConfig {
14
16
  callerOptions?: AsyncCallerParams;
15
17
  timeout_ms?: number;
16
18
  webUrl?: string;
19
+ /**
20
+ * A function applied for masking serialized run inputs and outputs,
21
+ * before sending to the API. Can be called with raw inputs, raw
22
+ * outputs, or a nested `{ error: string }` object for errors.
23
+ *
24
+ * If a `hideInputs` or `hideOutputs` function is present,
25
+ * the client will call it instead of the anonymizer as appropriate.
26
+ */
17
27
  anonymizer?: (values: KVMap) => KVMap | Promise<KVMap>;
18
28
  hideInputs?: boolean | ((inputs: KVMap) => KVMap | Promise<KVMap>);
19
29
  hideOutputs?: boolean | ((outputs: KVMap) => KVMap | Promise<KVMap>);
@@ -430,6 +440,7 @@ export declare class Client implements LangSmithTracingClientInterface {
430
440
  private hideInputs?;
431
441
  private hideOutputs?;
432
442
  private hideMetadata?;
443
+ private anonymizer?;
433
444
  private omitTracedRuntimeInfo?;
434
445
  private tracingSampleRate?;
435
446
  private filteredPostUuids;
@@ -507,9 +518,26 @@ export declare class Client implements LangSmithTracingClientInterface {
507
518
  get runs(): OpenAPIRuns;
508
519
  /** Access the v2 sandboxes resource (registries, snapshots, boxes). */
509
520
  get sandboxes(): Sandboxes;
521
+ /** Access the projects resource. */
522
+ get projects(): Sessions;
523
+ /** Access the v2 datasets resource (experimentRuns, etc.). */
524
+ get datasets(): Datasets;
510
525
  private processInputs;
511
526
  private processOutputs;
512
527
  private processMetadata;
528
+ /**
529
+ * Apply the configured anonymizer to a run's error string.
530
+ *
531
+ * Unlike inputs/outputs, `error` is a plain string (an exception message or
532
+ * traceback) that can carry credentials the user never explicitly logged --
533
+ * e.g. an HTTP-client error whose message embeds an `Authorization` header.
534
+ * The anonymizer is typed `(KVMap) => KVMap`, so the string is wrapped as
535
+ * `{ error }`, scrubbed, and unwrapped. Mirrors the Python SDK's
536
+ * `Client._hide_run_error`.
537
+ *
538
+ * TODO: Update anonymizer to always nest inputs/outputs/error for consistency
539
+ */
540
+ private processError;
513
541
  /**
514
542
  * Filter content from new_token events to prevent streaming LLM output
515
543
  * from being uploaded via events.
package/dist/client.js CHANGED
@@ -504,6 +504,12 @@ export class Client {
504
504
  writable: true,
505
505
  value: void 0
506
506
  });
507
+ Object.defineProperty(this, "anonymizer", {
508
+ enumerable: true,
509
+ configurable: true,
510
+ writable: true,
511
+ value: void 0
512
+ });
507
513
  Object.defineProperty(this, "omitTracedRuntimeInfo", {
508
514
  enumerable: true,
509
515
  configurable: true,
@@ -755,6 +761,7 @@ export class Client {
755
761
  this.hideOutputs =
756
762
  config.hideOutputs ?? config.anonymizer ?? defaultConfig.hideOutputs;
757
763
  this.hideMetadata = config.hideMetadata ?? defaultConfig.hideMetadata;
764
+ this.anonymizer = config.anonymizer;
758
765
  this.omitTracedRuntimeInfo = config.omitTracedRuntimeInfo ?? false;
759
766
  this.autoBatchTracing = config.autoBatchTracing ?? this.autoBatchTracing;
760
767
  this.autoBatchQueue = new AutoBatchQueue(maxMemory);
@@ -936,6 +943,14 @@ export class Client {
936
943
  get sandboxes() {
937
944
  return this.openAPIClient.sandboxes;
938
945
  }
946
+ /** Access the projects resource. */
947
+ get projects() {
948
+ return this.openAPIClient.sessions;
949
+ }
950
+ /** Access the v2 datasets resource (experimentRuns, etc.). */
951
+ get datasets() {
952
+ return this.openAPIClient.datasets;
953
+ }
939
954
  async processInputs(inputs) {
940
955
  if (this.hideInputs === false) {
941
956
  return inputs;
@@ -972,6 +987,25 @@ export class Client {
972
987
  }
973
988
  return metadata;
974
989
  }
990
+ /**
991
+ * Apply the configured anonymizer to a run's error string.
992
+ *
993
+ * Unlike inputs/outputs, `error` is a plain string (an exception message or
994
+ * traceback) that can carry credentials the user never explicitly logged --
995
+ * e.g. an HTTP-client error whose message embeds an `Authorization` header.
996
+ * The anonymizer is typed `(KVMap) => KVMap`, so the string is wrapped as
997
+ * `{ error }`, scrubbed, and unwrapped. Mirrors the Python SDK's
998
+ * `Client._hide_run_error`.
999
+ *
1000
+ * TODO: Update anonymizer to always nest inputs/outputs/error for consistency
1001
+ */
1002
+ async processError(error) {
1003
+ if (this.anonymizer == null) {
1004
+ return error;
1005
+ }
1006
+ const result = await this.anonymizer({ error });
1007
+ return typeof result?.error === "string" ? result.error : error;
1008
+ }
975
1009
  /**
976
1010
  * Filter content from new_token events to prevent streaming LLM output
977
1011
  * from being uploaded via events.
@@ -998,6 +1032,9 @@ export class Client {
998
1032
  if (runParams.outputs !== undefined) {
999
1033
  runParams.outputs = await this.processOutputs(runParams.outputs);
1000
1034
  }
1035
+ if (runParams.error !== undefined) {
1036
+ runParams.error = await this.processError(runParams.error);
1037
+ }
1001
1038
  if (runParams.extra != null && "metadata" in runParams.extra) {
1002
1039
  runParams.extra = {
1003
1040
  ...runParams.extra,
@@ -1854,6 +1891,9 @@ export class Client {
1854
1891
  if (run.outputs) {
1855
1892
  run.outputs = await this.processOutputs(run.outputs);
1856
1893
  }
1894
+ if (run.error) {
1895
+ run.error = await this.processError(run.error);
1896
+ }
1857
1897
  if (run.extra != null && "metadata" in run.extra) {
1858
1898
  run.extra = {
1859
1899
  ...run.extra,
@@ -136,7 +136,11 @@ class StreamManager {
136
136
  }
137
137
  this.namespaces["root"].extra ??= {};
138
138
  this.namespaces["root"].extra.metadata ??= {};
139
- this.namespaces["root"].extra.metadata.usage_metadata = usage;
139
+ // Keep the SDK's final aggregate usage available for inspection, but do not
140
+ // store it as `usage_metadata` on the parent chain. LangSmith rollups
141
+ // already aggregate child LLM usage, so parent `usage_metadata` would be
142
+ // counted again (notably doubling Anthropic cache-read/cache-write tokens).
143
+ this.namespaces["root"].extra.metadata.ls_aggregated_usage = usage;
140
144
  this.namespaces["root"].extra.metadata.is_error = message.is_error;
141
145
  this.namespaces["root"].extra.metadata.num_turns = message.num_turns;
142
146
  this.namespaces["root"].extra.metadata.session_id = message.session_id;
@@ -133,7 +133,11 @@ export class StreamManager {
133
133
  }
134
134
  this.namespaces["root"].extra ??= {};
135
135
  this.namespaces["root"].extra.metadata ??= {};
136
- this.namespaces["root"].extra.metadata.usage_metadata = usage;
136
+ // Keep the SDK's final aggregate usage available for inspection, but do not
137
+ // store it as `usage_metadata` on the parent chain. LangSmith rollups
138
+ // already aggregate child LLM usage, so parent `usage_metadata` would be
139
+ // counted again (notably doubling Anthropic cache-read/cache-write tokens).
140
+ this.namespaces["root"].extra.metadata.ls_aggregated_usage = usage;
137
141
  this.namespaces["root"].extra.metadata.is_error = message.is_error;
138
142
  this.namespaces["root"].extra.metadata.num_turns = message.num_turns;
139
143
  this.namespaces["root"].extra.metadata.session_id = message.session_id;
package/dist/index.cjs CHANGED
@@ -20,6 +20,6 @@ Object.defineProperty(exports, "PromptCache", { enumerable: true, get: function
20
20
  Object.defineProperty(exports, "configureGlobalPromptCache", { enumerable: true, get: function () { return index_js_1.configureGlobalPromptCache; } });
21
21
  Object.defineProperty(exports, "promptCacheSingleton", { enumerable: true, get: function () { return index_js_1.promptCacheSingleton; } });
22
22
  // Update using pnpm bump-version
23
- exports.__version__ = "0.7.13";
23
+ exports.__version__ = "0.7.14";
24
24
  // Metadata key to hide a traced run from LangSmith's Messages View.
25
25
  exports.LS_MESSAGE_VIEW_EXCLUDE = "ls_message_view_exclude";
package/dist/index.d.ts CHANGED
@@ -6,5 +6,5 @@ export { getDefaultProjectName } from "./utils/project.js";
6
6
  export { uuid7, uuid7FromTime } from "./uuid.js";
7
7
  export { isTracingEnabled } from "./utils/guard.js";
8
8
  export { Cache, PromptCache, type CacheConfig, type CacheMetrics, configureGlobalPromptCache, promptCacheSingleton, } from "./utils/prompt_cache/index.js";
9
- export declare const __version__ = "0.7.13";
9
+ export declare const __version__ = "0.7.14";
10
10
  export declare const LS_MESSAGE_VIEW_EXCLUDE: "ls_message_view_exclude";
package/dist/index.js CHANGED
@@ -6,6 +6,6 @@ export { uuid7, uuid7FromTime } from "./uuid.js";
6
6
  export { isTracingEnabled } from "./utils/guard.js";
7
7
  export { Cache, PromptCache, configureGlobalPromptCache, promptCacheSingleton, } from "./utils/prompt_cache/index.js";
8
8
  // Update using pnpm bump-version
9
- export const __version__ = "0.7.13";
9
+ export const __version__ = "0.7.14";
10
10
  // Metadata key to hide a traced run from LangSmith's Messages View.
11
11
  export const LS_MESSAGE_VIEW_EXCLUDE = "ls_message_view_exclude";
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "langsmith",
3
- "version": "0.7.13",
3
+ "version": "0.7.14",
4
4
  "description": "Client library to connect to the LangSmith Observability and Evaluation Platform.",
5
5
  "packageManager": "pnpm@10.33.0",
6
6
  "files": [
@@ -1,16 +0,0 @@
1
- "use strict";
2
- // @ts-nocheck
3
- // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
4
- Object.defineProperty(exports, "__esModule", { value: true });
5
- exports.Experiments = void 0;
6
- const resource_js_1 = require("../../core/resource.cjs");
7
- const path_js_1 = require("../../internal/utils/path.cjs");
8
- class Experiments extends resource_js_1.APIResource {
9
- /**
10
- * Stream grouped and aggregated experiments.
11
- */
12
- grouped(datasetID, body, options) {
13
- return this._client.post((0, path_js_1.path) `/api/v1/datasets/${datasetID}/experiments/grouped`, { body, ...options });
14
- }
15
- }
16
- exports.Experiments = Experiments;
@@ -1,23 +0,0 @@
1
- import { APIResource } from '../../core/resource.js';
2
- import { APIPromise } from '../../core/api-promise.js';
3
- import { RequestOptions } from '../../internal/request-options.js';
4
- export declare class Experiments extends APIResource {
5
- /**
6
- * Stream grouped and aggregated experiments.
7
- */
8
- grouped(datasetID: string, body: ExperimentGroupedParams, options?: RequestOptions): APIPromise<unknown>;
9
- }
10
- export type ExperimentGroupedResponse = unknown;
11
- export interface ExperimentGroupedParams {
12
- metadata_keys: Array<string>;
13
- dataset_version?: string | null;
14
- experiment_limit?: number;
15
- filter?: string | null;
16
- name_contains?: string | null;
17
- stats_start_time?: string | null;
18
- tag_value_id?: Array<string> | null;
19
- use_approx_stats?: boolean;
20
- }
21
- export declare namespace Experiments {
22
- export { type ExperimentGroupedResponse as ExperimentGroupedResponse, type ExperimentGroupedParams as ExperimentGroupedParams, };
23
- }
@@ -1,12 +0,0 @@
1
- // @ts-nocheck
2
- // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
3
- import { APIResource } from '../../core/resource.js';
4
- import { path } from '../../internal/utils/path.js';
5
- export class Experiments extends APIResource {
6
- /**
7
- * Stream grouped and aggregated experiments.
8
- */
9
- grouped(datasetID, body, options) {
10
- return this._client.post(path `/api/v1/datasets/${datasetID}/experiments/grouped`, { body, ...options });
11
- }
12
- }
@@ -1,17 +0,0 @@
1
- "use strict";
2
- // @ts-nocheck
3
- // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
4
- Object.defineProperty(exports, "__esModule", { value: true });
5
- exports.Group = void 0;
6
- const resource_js_1 = require("../../core/resource.cjs");
7
- const path_js_1 = require("../../internal/utils/path.cjs");
8
- class Group extends resource_js_1.APIResource {
9
- /**
10
- * Fetch examples for a dataset, and fetch the runs for each example if they are
11
- * associated with the given session_ids.
12
- */
13
- runs(datasetID, body, options) {
14
- return this._client.post((0, path_js_1.path) `/api/v1/datasets/${datasetID}/group/runs`, { body, ...options });
15
- }
16
- }
17
- exports.Group = Group;
@@ -1,160 +0,0 @@
1
- import { APIResource } from '../../core/resource.js';
2
- import * as RunsAPI from './runs.js';
3
- import { APIPromise } from '../../core/api-promise.js';
4
- import { RequestOptions } from '../../internal/request-options.js';
5
- export declare class Group extends APIResource {
6
- /**
7
- * Fetch examples for a dataset, and fetch the runs for each example if they are
8
- * associated with the given session_ids.
9
- */
10
- runs(datasetID: string, body: GroupRunsParams, options?: RequestOptions): APIPromise<GroupRunsResponse>;
11
- }
12
- /**
13
- * Response for grouped comparison view of dataset examples.
14
- *
15
- * Returns dataset examples grouped by a run metadata value (e.g., model='gpt-4').
16
- * Optional filters are applied to all runs before grouping.
17
- *
18
- * Shows:
19
- *
20
- * - Which examples were executed with each metadata value
21
- * - Per-session aggregate statistics for runs on those examples
22
- * - The actual example data with their associated runs
23
- *
24
- * Used for comparing how different sessions performed on the same set of examples.
25
- */
26
- export interface GroupRunsResponse {
27
- groups: Array<GroupRunsResponse.Group>;
28
- }
29
- export declare namespace GroupRunsResponse {
30
- /**
31
- * Group of examples with a specific metadata value across multiple sessions.
32
- *
33
- * Extends RunGroupBase with:
34
- *
35
- * - group_key: metadata value that defines this group
36
- * - sessions: per-session stats for runs matching this metadata value
37
- * - examples: shared examples across all sessions (intersection logic) with flat
38
- * array of runs (each run has session_id field for frontend to determine column)
39
- * - example_count: unique example count (pagination-aware, same across all
40
- * sessions due to intersection)
41
- *
42
- * Inherited from RunGroupBase:
43
- *
44
- * - filter: metadata filter for this group (e.g., "and(eq(is_root, true),
45
- * and(eq(metadata_key, 'model'), eq(metadata_value, 'gpt-4')))")
46
- * - count: total run count across all sessions (includes duplicate runs)
47
- * - total_tokens, total_cost: aggregate across sessions
48
- * - min_start_time, max_start_time: time range across sessions
49
- * - latency_p50, latency_p99: aggregate latency stats across sessions
50
- * - feedback_stats: weighted average feedback across sessions
51
- *
52
- * Additional aggregate stats:
53
- *
54
- * - prompt_tokens, completion_tokens: separate token counts
55
- * - prompt_cost, completion_cost: separate costs
56
- * - error_rate: average error rate
57
- */
58
- interface Group {
59
- example_count: number;
60
- examples: Array<RunsAPI.ExampleWithRunsCh>;
61
- filter: string;
62
- group_key: string | number;
63
- sessions: Array<Group.Session>;
64
- completion_cost?: string | null;
65
- completion_tokens?: number | null;
66
- count?: number | null;
67
- error_rate?: number | null;
68
- feedback_stats?: {
69
- [key: string]: unknown;
70
- } | null;
71
- latency_p50?: number | null;
72
- latency_p99?: number | null;
73
- max_start_time?: string | null;
74
- min_start_time?: string | null;
75
- prompt_cost?: string | null;
76
- prompt_tokens?: number | null;
77
- total_cost?: string | null;
78
- total_tokens?: number | null;
79
- }
80
- namespace Group {
81
- /**
82
- * TracerSession stats filtered to runs matching a specific metadata value.
83
- *
84
- * Extends TracerSession with:
85
- *
86
- * - example_count: unique examples (vs run_count = total runs including
87
- * duplicates)
88
- * - filter: ClickHouse filter for fetching runs in this session/group
89
- * - min/max_start_time: time range for runs in this session/group
90
- */
91
- interface Session {
92
- id: string;
93
- filter: string;
94
- tenant_id: string;
95
- completion_cost?: string | null;
96
- completion_tokens?: number | null;
97
- default_dataset_id?: string | null;
98
- description?: string | null;
99
- end_time?: string | null;
100
- error_rate?: number | null;
101
- example_count?: number | null;
102
- experiment_progress?: Session.ExperimentProgress | null;
103
- extra?: {
104
- [key: string]: unknown;
105
- } | null;
106
- feedback_stats?: {
107
- [key: string]: unknown;
108
- } | null;
109
- first_token_p50?: number | null;
110
- first_token_p99?: number | null;
111
- last_run_start_time?: string | null;
112
- last_run_start_time_live?: string | null;
113
- latency_p50?: number | null;
114
- latency_p99?: number | null;
115
- max_start_time?: string | null;
116
- min_start_time?: string | null;
117
- name?: string;
118
- prompt_cost?: string | null;
119
- prompt_tokens?: number | null;
120
- reference_dataset_id?: string | null;
121
- run_count?: number | null;
122
- run_facets?: Array<{
123
- [key: string]: unknown;
124
- }> | null;
125
- session_feedback_stats?: {
126
- [key: string]: unknown;
127
- } | null;
128
- start_time?: string;
129
- streaming_rate?: number | null;
130
- test_run_number?: number | null;
131
- total_cost?: string | null;
132
- total_tokens?: number | null;
133
- trace_tier?: 'longlived' | 'shortlived' | null;
134
- }
135
- namespace Session {
136
- interface ExperimentProgress {
137
- evaluator_progress: {
138
- [key: string]: number;
139
- };
140
- expected_run_count: number;
141
- run_progress: number;
142
- }
143
- }
144
- }
145
- }
146
- export interface GroupRunsParams {
147
- group_by: 'run_metadata' | 'example_metadata';
148
- metadata_key: string;
149
- session_ids: Array<string>;
150
- filters?: {
151
- [key: string]: Array<string>;
152
- } | null;
153
- limit?: number;
154
- offset?: number;
155
- per_group_limit?: number;
156
- preview?: boolean;
157
- }
158
- export declare namespace Group {
159
- export { type GroupRunsResponse as GroupRunsResponse, type GroupRunsParams as GroupRunsParams };
160
- }
@@ -1,13 +0,0 @@
1
- // @ts-nocheck
2
- // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
3
- import { APIResource } from '../../core/resource.js';
4
- import { path } from '../../internal/utils/path.js';
5
- export class Group extends APIResource {
6
- /**
7
- * Fetch examples for a dataset, and fetch the runs for each example if they are
8
- * associated with the given session_ids.
9
- */
10
- runs(datasetID, body, options) {
11
- return this._client.post(path `/api/v1/datasets/${datasetID}/group/runs`, { body, ...options });
12
- }
13
- }
@@ -1,54 +0,0 @@
1
- "use strict";
2
- // @ts-nocheck
3
- // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
4
- Object.defineProperty(exports, "__esModule", { value: true });
5
- exports.Insights = void 0;
6
- const resource_js_1 = require("../../core/resource.cjs");
7
- const pagination_js_1 = require("../../core/pagination.cjs");
8
- const path_js_1 = require("../../internal/utils/path.cjs");
9
- class Insights extends resource_js_1.APIResource {
10
- /**
11
- * Create an insights job.
12
- */
13
- create(sessionID, body, options) {
14
- return this._client.post((0, path_js_1.path) `/api/v1/sessions/${sessionID}/insights`, { body, ...options });
15
- }
16
- /**
17
- * Update a session cluster job.
18
- */
19
- update(jobID, params, options) {
20
- const { session_id, ...body } = params;
21
- return this._client.patch((0, path_js_1.path) `/api/v1/sessions/${session_id}/insights/${jobID}`, { body, ...options });
22
- }
23
- /**
24
- * Get all clusters for a session.
25
- */
26
- list(sessionID, query = {}, options) {
27
- return this._client.getAPIList((0, path_js_1.path) `/api/v1/sessions/${sessionID}/insights`, (pagination_js_1.OffsetPaginationInsightsClusteringJobs), { query, ...options });
28
- }
29
- /**
30
- * Delete a session cluster job.
31
- */
32
- delete(jobID, params, options) {
33
- const { session_id } = params;
34
- return this._client.delete((0, path_js_1.path) `/api/v1/sessions/${session_id}/insights/${jobID}`, options);
35
- }
36
- /**
37
- * Get a specific cluster job for a session.
38
- */
39
- retrieveJob(jobID, params, options) {
40
- const { session_id } = params;
41
- return this._client.get((0, path_js_1.path) `/api/v1/sessions/${session_id}/insights/${jobID}`, options);
42
- }
43
- /**
44
- * Get all runs for a cluster job, optionally filtered by cluster.
45
- */
46
- retrieveRuns(jobID, params, options) {
47
- const { session_id, ...query } = params;
48
- return this._client.get((0, path_js_1.path) `/api/v1/sessions/${session_id}/insights/${jobID}/runs`, {
49
- query,
50
- ...options,
51
- });
52
- }
53
- }
54
- exports.Insights = Insights;