langsmith 0.7.13 → 0.7.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/dist/_openapi_client/client.cjs +9 -1
  2. package/dist/_openapi_client/client.d.ts +10 -5
  3. package/dist/_openapi_client/client.js +9 -1
  4. package/dist/_openapi_client/core/pagination.cjs +29 -1
  5. package/dist/_openapi_client/core/pagination.d.ts +11 -0
  6. package/dist/_openapi_client/core/pagination.js +27 -0
  7. package/dist/_openapi_client/resources/datasets/datasets.cjs +5 -14
  8. package/dist/_openapi_client/resources/datasets/datasets.d.ts +6 -10
  9. package/dist/_openapi_client/resources/datasets/datasets.js +5 -14
  10. package/dist/_openapi_client/resources/datasets/experiment-runs.cjs +18 -0
  11. package/dist/_openapi_client/resources/datasets/experiment-runs.d.ts +109 -0
  12. package/dist/_openapi_client/resources/datasets/experiment-runs.js +14 -0
  13. package/dist/_openapi_client/resources/datasets/runs.cjs +0 -7
  14. package/dist/_openapi_client/resources/datasets/runs.d.ts +4 -45
  15. package/dist/_openapi_client/resources/datasets/runs.js +0 -7
  16. package/dist/_openapi_client/resources/index.cjs +4 -2
  17. package/dist/_openapi_client/resources/index.d.ts +3 -2
  18. package/dist/_openapi_client/resources/index.js +2 -1
  19. package/dist/_openapi_client/resources/issues.cjs +31 -0
  20. package/dist/_openapi_client/resources/issues.d.ts +78 -0
  21. package/dist/_openapi_client/resources/issues.js +27 -0
  22. package/dist/_openapi_client/resources/runs/index.d.ts +1 -1
  23. package/dist/_openapi_client/resources/runs/runs.cjs +2 -2
  24. package/dist/_openapi_client/resources/runs/runs.d.ts +24 -24
  25. package/dist/_openapi_client/resources/runs/runs.js +2 -2
  26. package/dist/_openapi_client/resources/sandboxes/boxes.d.ts +7 -0
  27. package/dist/_openapi_client/resources/sandboxes/sandboxes.d.ts +1 -0
  28. package/dist/_openapi_client/resources/sessions.cjs +53 -0
  29. package/dist/_openapi_client/resources/{sessions/sessions.d.ts → sessions.d.ts} +14 -52
  30. package/dist/_openapi_client/resources/sessions.js +49 -0
  31. package/dist/client.cjs +40 -0
  32. package/dist/client.d.ts +28 -0
  33. package/dist/client.js +40 -0
  34. package/dist/experimental/anthropic/context.cjs +6 -1
  35. package/dist/experimental/anthropic/context.js +6 -1
  36. package/dist/index.cjs +1 -1
  37. package/dist/index.d.ts +1 -1
  38. package/dist/index.js +1 -1
  39. package/dist/utils/constants.cjs +1 -1
  40. package/dist/utils/constants.d.ts +1 -1
  41. package/dist/utils/constants.js +1 -1
  42. package/dist/utils/env.cjs +1 -1
  43. package/dist/utils/env.d.ts +0 -7
  44. package/dist/utils/env.js +1 -1
  45. package/package.json +10 -10
  46. package/dist/_openapi_client/resources/datasets/experiments.cjs +0 -16
  47. package/dist/_openapi_client/resources/datasets/experiments.d.ts +0 -23
  48. package/dist/_openapi_client/resources/datasets/experiments.js +0 -12
  49. package/dist/_openapi_client/resources/datasets/group.cjs +0 -17
  50. package/dist/_openapi_client/resources/datasets/group.d.ts +0 -160
  51. package/dist/_openapi_client/resources/datasets/group.js +0 -13
  52. package/dist/_openapi_client/resources/sessions/insights.cjs +0 -54
  53. package/dist/_openapi_client/resources/sessions/insights.d.ts +0 -246
  54. package/dist/_openapi_client/resources/sessions/insights.js +0 -50
  55. package/dist/_openapi_client/resources/sessions/sessions.cjs +0 -109
  56. package/dist/_openapi_client/resources/sessions/sessions.js +0 -72
package/dist/client.d.ts CHANGED
@@ -6,6 +6,8 @@ import { EvaluationResult, EvaluationResults } from "./evaluation/evaluator.js";
6
6
  import { OnlineEvaluators } from "./_openapi_client/resources/online-evaluators.js";
7
7
  import { Runs as OpenAPIRuns } from "./_openapi_client/resources/runs.js";
8
8
  import { Sandboxes } from "./_openapi_client/resources/sandboxes/sandboxes.js";
9
+ import { Datasets } from "./_openapi_client/resources/datasets/datasets.js";
10
+ import { Sessions } from "./_openapi_client/resources/sessions.js";
9
11
  import { PromptCache } from "./utils/prompt_cache/index.js";
10
12
  import { ProfileAuth } from "./utils/profiles.js";
11
13
  export interface ClientConfig {
@@ -14,6 +16,14 @@ export interface ClientConfig {
14
16
  callerOptions?: AsyncCallerParams;
15
17
  timeout_ms?: number;
16
18
  webUrl?: string;
19
+ /**
20
+ * A function applied for masking serialized run inputs and outputs,
21
+ * before sending to the API. Can be called with raw inputs, raw
22
+ * outputs, or a nested `{ error: string }` object for errors.
23
+ *
24
+ * If a `hideInputs` or `hideOutputs` function is present,
25
+ * the client will call it instead of the anonymizer as appropriate.
26
+ */
17
27
  anonymizer?: (values: KVMap) => KVMap | Promise<KVMap>;
18
28
  hideInputs?: boolean | ((inputs: KVMap) => KVMap | Promise<KVMap>);
19
29
  hideOutputs?: boolean | ((outputs: KVMap) => KVMap | Promise<KVMap>);
@@ -430,6 +440,7 @@ export declare class Client implements LangSmithTracingClientInterface {
430
440
  private hideInputs?;
431
441
  private hideOutputs?;
432
442
  private hideMetadata?;
443
+ private anonymizer?;
433
444
  private omitTracedRuntimeInfo?;
434
445
  private tracingSampleRate?;
435
446
  private filteredPostUuids;
@@ -507,9 +518,26 @@ export declare class Client implements LangSmithTracingClientInterface {
507
518
  get runs(): OpenAPIRuns;
508
519
  /** Access the v2 sandboxes resource (registries, snapshots, boxes). */
509
520
  get sandboxes(): Sandboxes;
521
+ /** Access the projects resource. */
522
+ get projects(): Sessions;
523
+ /** Access the v2 datasets resource (experimentRuns, etc.). */
524
+ get datasets(): Datasets;
510
525
  private processInputs;
511
526
  private processOutputs;
512
527
  private processMetadata;
528
+ /**
529
+ * Apply the configured anonymizer to a run's error string.
530
+ *
531
+ * Unlike inputs/outputs, `error` is a plain string (an exception message or
532
+ * traceback) that can carry credentials the user never explicitly logged --
533
+ * e.g. an HTTP-client error whose message embeds an `Authorization` header.
534
+ * The anonymizer is typed `(KVMap) => KVMap`, so the string is wrapped as
535
+ * `{ error }`, scrubbed, and unwrapped. Mirrors the Python SDK's
536
+ * `Client._hide_run_error`.
537
+ *
538
+ * TODO: Update anonymizer to always nest inputs/outputs/error for consistency
539
+ */
540
+ private processError;
513
541
  /**
514
542
  * Filter content from new_token events to prevent streaming LLM output
515
543
  * from being uploaded via events.
package/dist/client.js CHANGED
@@ -504,6 +504,12 @@ export class Client {
504
504
  writable: true,
505
505
  value: void 0
506
506
  });
507
+ Object.defineProperty(this, "anonymizer", {
508
+ enumerable: true,
509
+ configurable: true,
510
+ writable: true,
511
+ value: void 0
512
+ });
507
513
  Object.defineProperty(this, "omitTracedRuntimeInfo", {
508
514
  enumerable: true,
509
515
  configurable: true,
@@ -755,6 +761,7 @@ export class Client {
755
761
  this.hideOutputs =
756
762
  config.hideOutputs ?? config.anonymizer ?? defaultConfig.hideOutputs;
757
763
  this.hideMetadata = config.hideMetadata ?? defaultConfig.hideMetadata;
764
+ this.anonymizer = config.anonymizer;
758
765
  this.omitTracedRuntimeInfo = config.omitTracedRuntimeInfo ?? false;
759
766
  this.autoBatchTracing = config.autoBatchTracing ?? this.autoBatchTracing;
760
767
  this.autoBatchQueue = new AutoBatchQueue(maxMemory);
@@ -936,6 +943,14 @@ export class Client {
936
943
  get sandboxes() {
937
944
  return this.openAPIClient.sandboxes;
938
945
  }
946
+ /** Access the projects resource. */
947
+ get projects() {
948
+ return this.openAPIClient.sessions;
949
+ }
950
+ /** Access the v2 datasets resource (experimentRuns, etc.). */
951
+ get datasets() {
952
+ return this.openAPIClient.datasets;
953
+ }
939
954
  async processInputs(inputs) {
940
955
  if (this.hideInputs === false) {
941
956
  return inputs;
@@ -972,6 +987,25 @@ export class Client {
972
987
  }
973
988
  return metadata;
974
989
  }
990
+ /**
991
+ * Apply the configured anonymizer to a run's error string.
992
+ *
993
+ * Unlike inputs/outputs, `error` is a plain string (an exception message or
994
+ * traceback) that can carry credentials the user never explicitly logged --
995
+ * e.g. an HTTP-client error whose message embeds an `Authorization` header.
996
+ * The anonymizer is typed `(KVMap) => KVMap`, so the string is wrapped as
997
+ * `{ error }`, scrubbed, and unwrapped. Mirrors the Python SDK's
998
+ * `Client._hide_run_error`.
999
+ *
1000
+ * TODO: Update anonymizer to always nest inputs/outputs/error for consistency
1001
+ */
1002
+ async processError(error) {
1003
+ if (this.anonymizer == null) {
1004
+ return error;
1005
+ }
1006
+ const result = await this.anonymizer({ error });
1007
+ return typeof result?.error === "string" ? result.error : error;
1008
+ }
975
1009
  /**
976
1010
  * Filter content from new_token events to prevent streaming LLM output
977
1011
  * from being uploaded via events.
@@ -998,6 +1032,9 @@ export class Client {
998
1032
  if (runParams.outputs !== undefined) {
999
1033
  runParams.outputs = await this.processOutputs(runParams.outputs);
1000
1034
  }
1035
+ if (runParams.error !== undefined) {
1036
+ runParams.error = await this.processError(runParams.error);
1037
+ }
1001
1038
  if (runParams.extra != null && "metadata" in runParams.extra) {
1002
1039
  runParams.extra = {
1003
1040
  ...runParams.extra,
@@ -1854,6 +1891,9 @@ export class Client {
1854
1891
  if (run.outputs) {
1855
1892
  run.outputs = await this.processOutputs(run.outputs);
1856
1893
  }
1894
+ if (run.error) {
1895
+ run.error = await this.processError(run.error);
1896
+ }
1857
1897
  if (run.extra != null && "metadata" in run.extra) {
1858
1898
  run.extra = {
1859
1899
  ...run.extra,
@@ -136,10 +136,15 @@ class StreamManager {
136
136
  }
137
137
  this.namespaces["root"].extra ??= {};
138
138
  this.namespaces["root"].extra.metadata ??= {};
139
- this.namespaces["root"].extra.metadata.usage_metadata = usage;
139
+ // Keep the SDK's final aggregate usage available for inspection, but do not
140
+ // store it as `usage_metadata` on the parent chain. LangSmith rollups
141
+ // already aggregate child LLM usage, so parent `usage_metadata` would be
142
+ // counted again (notably doubling Anthropic cache-read/cache-write tokens).
143
+ this.namespaces["root"].extra.metadata.ls_aggregated_usage = usage;
140
144
  this.namespaces["root"].extra.metadata.is_error = message.is_error;
141
145
  this.namespaces["root"].extra.metadata.num_turns = message.num_turns;
142
146
  this.namespaces["root"].extra.metadata.session_id = message.session_id;
147
+ this.namespaces["root"].extra.metadata.thread_id = message.session_id;
143
148
  this.namespaces["root"].extra.metadata.duration_ms = message.duration_ms;
144
149
  this.namespaces["root"].extra.metadata.duration_api_ms =
145
150
  message.duration_api_ms;
@@ -133,10 +133,15 @@ export class StreamManager {
133
133
  }
134
134
  this.namespaces["root"].extra ??= {};
135
135
  this.namespaces["root"].extra.metadata ??= {};
136
- this.namespaces["root"].extra.metadata.usage_metadata = usage;
136
+ // Keep the SDK's final aggregate usage available for inspection, but do not
137
+ // store it as `usage_metadata` on the parent chain. LangSmith rollups
138
+ // already aggregate child LLM usage, so parent `usage_metadata` would be
139
+ // counted again (notably doubling Anthropic cache-read/cache-write tokens).
140
+ this.namespaces["root"].extra.metadata.ls_aggregated_usage = usage;
137
141
  this.namespaces["root"].extra.metadata.is_error = message.is_error;
138
142
  this.namespaces["root"].extra.metadata.num_turns = message.num_turns;
139
143
  this.namespaces["root"].extra.metadata.session_id = message.session_id;
144
+ this.namespaces["root"].extra.metadata.thread_id = message.session_id;
140
145
  this.namespaces["root"].extra.metadata.duration_ms = message.duration_ms;
141
146
  this.namespaces["root"].extra.metadata.duration_api_ms =
142
147
  message.duration_api_ms;
package/dist/index.cjs CHANGED
@@ -20,6 +20,6 @@ Object.defineProperty(exports, "PromptCache", { enumerable: true, get: function
20
20
  Object.defineProperty(exports, "configureGlobalPromptCache", { enumerable: true, get: function () { return index_js_1.configureGlobalPromptCache; } });
21
21
  Object.defineProperty(exports, "promptCacheSingleton", { enumerable: true, get: function () { return index_js_1.promptCacheSingleton; } });
22
22
  // Update using pnpm bump-version
23
- exports.__version__ = "0.7.13";
23
+ exports.__version__ = "0.7.15";
24
24
  // Metadata key to hide a traced run from LangSmith's Messages View.
25
25
  exports.LS_MESSAGE_VIEW_EXCLUDE = "ls_message_view_exclude";
package/dist/index.d.ts CHANGED
@@ -6,5 +6,5 @@ export { getDefaultProjectName } from "./utils/project.js";
6
6
  export { uuid7, uuid7FromTime } from "./uuid.js";
7
7
  export { isTracingEnabled } from "./utils/guard.js";
8
8
  export { Cache, PromptCache, type CacheConfig, type CacheMetrics, configureGlobalPromptCache, promptCacheSingleton, } from "./utils/prompt_cache/index.js";
9
- export declare const __version__ = "0.7.13";
9
+ export declare const __version__ = "0.7.15";
10
10
  export declare const LS_MESSAGE_VIEW_EXCLUDE: "ls_message_view_exclude";
package/dist/index.js CHANGED
@@ -6,6 +6,6 @@ export { uuid7, uuid7FromTime } from "./uuid.js";
6
6
  export { isTracingEnabled } from "./utils/guard.js";
7
7
  export { Cache, PromptCache, configureGlobalPromptCache, promptCacheSingleton, } from "./utils/prompt_cache/index.js";
8
8
  // Update using pnpm bump-version
9
- export const __version__ = "0.7.13";
9
+ export const __version__ = "0.7.15";
10
10
  // Metadata key to hide a traced run from LangSmith's Messages View.
11
11
  export const LS_MESSAGE_VIEW_EXCLUDE = "ls_message_view_exclude";
@@ -1,4 +1,4 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports._MIN_BACKEND_VERSION = void 0;
4
- exports._MIN_BACKEND_VERSION = "0.16.6rc1";
4
+ exports._MIN_BACKEND_VERSION = "0.16.10rc1";
@@ -1 +1 @@
1
- export declare const _MIN_BACKEND_VERSION = "0.16.6rc1";
1
+ export declare const _MIN_BACKEND_VERSION = "0.16.10rc1";
@@ -1 +1 @@
1
- export const _MIN_BACKEND_VERSION = "0.16.6rc1";
1
+ export const _MIN_BACKEND_VERSION = "0.16.10rc1";
@@ -24,7 +24,7 @@ const isJsDom = () => (typeof window !== "undefined" && window.name === "nodejs"
24
24
  exports.isJsDom = isJsDom;
25
25
  // Supabase Edge Function provides a `Deno` global object
26
26
  // without `version` property
27
- const isDeno = () => typeof Deno !== "undefined";
27
+ const isDeno = () => typeof globalThis.Deno !== "undefined";
28
28
  exports.isDeno = isDeno;
29
29
  // Mark not-as-node if in Supabase Edge Function
30
30
  const isNode = () => typeof process !== "undefined" &&
@@ -1,10 +1,3 @@
1
- declare global {
2
- const Deno: {
3
- version: {
4
- deno: string;
5
- };
6
- } | undefined;
7
- }
8
1
  export declare const isBrowser: () => boolean;
9
2
  export declare const isWebWorker: () => boolean;
10
3
  export declare const isJsDom: () => boolean;
package/dist/utils/env.js CHANGED
@@ -9,7 +9,7 @@ export const isJsDom = () => (typeof window !== "undefined" && window.name === "
9
9
  (typeof navigator !== "undefined" && navigator.userAgent.includes("jsdom"));
10
10
  // Supabase Edge Function provides a `Deno` global object
11
11
  // without `version` property
12
- export const isDeno = () => typeof Deno !== "undefined";
12
+ export const isDeno = () => typeof globalThis.Deno !== "undefined";
13
13
  // Mark not-as-node if in Supabase Edge Function
14
14
  export const isNode = () => typeof process !== "undefined" &&
15
15
  typeof process.versions !== "undefined" &&
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "langsmith",
3
- "version": "0.7.13",
3
+ "version": "0.7.15",
4
4
  "description": "Client library to connect to the LangSmith Observability and Evaluation Platform.",
5
5
  "packageManager": "pnpm@10.33.0",
6
6
  "files": [
@@ -156,12 +156,12 @@
156
156
  "p-queue": "6.6.2"
157
157
  },
158
158
  "devDependencies": {
159
- "@ai-sdk/openai": "4.0.0-beta.74",
160
- "@ai-sdk/provider": "4.0.0-beta.19",
159
+ "@ai-sdk/openai": "4.0.4",
160
+ "@ai-sdk/provider": "4.0.1",
161
161
  "@ai-sdk/anthropic": "4.0.0-beta.67",
162
162
  "@anthropic-ai/claude-agent-sdk": "^0.3.150",
163
163
  "@anthropic-ai/sdk": "^0.98.0",
164
- "@babel/preset-env": "^7.22.4",
164
+ "@babel/preset-env": "^8.0.2",
165
165
  "@faker-js/faker": "^8.4.1",
166
166
  "@google/genai": "^2.0.1",
167
167
  "@jest/globals": "^30.4.1",
@@ -169,11 +169,11 @@
169
169
  "@langchain/core": "^0.3.72",
170
170
  "@langchain/langgraph": "^0.3.6",
171
171
  "@langchain/openai": "^0.6.17",
172
- "@openai/agents": "^0.11.5",
172
+ "@openai/agents": "^0.12.0",
173
173
  "@opentelemetry/api": "^1.9.0",
174
- "@opentelemetry/auto-instrumentations-node": "^0.76.0",
174
+ "@opentelemetry/auto-instrumentations-node": "^0.77.0",
175
175
  "@opentelemetry/context-async-hooks": "^2.6.1",
176
- "@opentelemetry/sdk-node": "^0.218.0",
176
+ "@opentelemetry/sdk-node": "^0.219.0",
177
177
  "@opentelemetry/sdk-trace-base": "^2.0.0",
178
178
  "@opentelemetry/sdk-trace-node": "^2.0.0",
179
179
  "@tsconfig/recommended": "^1.0.2",
@@ -182,8 +182,8 @@
182
182
  "@types/node-fetch": "^2.6.12",
183
183
  "@types/semver": "^7.7.1",
184
184
  "@types/ws": "^8.18.1",
185
- "@typescript-eslint/eslint-plugin": "^5.59.8",
186
- "@typescript-eslint/parser": "^5.59.8",
185
+ "@typescript-eslint/eslint-plugin": "^8.62.1",
186
+ "@typescript-eslint/parser": "^8.62.1",
187
187
  "ai": "7.0.0-beta.178",
188
188
  "babel-jest": "^30.2.0",
189
189
  "cross-env": "^10.1.0",
@@ -194,7 +194,7 @@
194
194
  "msw": "^2.11.2",
195
195
  "node-fetch": "^3.3.2",
196
196
  "openai": "^6.18.0",
197
- "oxfmt": "^0.47.0",
197
+ "oxfmt": "^0.57.0",
198
198
  "oxlint": "^1.62.0",
199
199
  "semver": "^7.7.4",
200
200
  "ts-jest": "^29.1.0",
@@ -1,16 +0,0 @@
1
- "use strict";
2
- // @ts-nocheck
3
- // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
4
- Object.defineProperty(exports, "__esModule", { value: true });
5
- exports.Experiments = void 0;
6
- const resource_js_1 = require("../../core/resource.cjs");
7
- const path_js_1 = require("../../internal/utils/path.cjs");
8
- class Experiments extends resource_js_1.APIResource {
9
- /**
10
- * Stream grouped and aggregated experiments.
11
- */
12
- grouped(datasetID, body, options) {
13
- return this._client.post((0, path_js_1.path) `/api/v1/datasets/${datasetID}/experiments/grouped`, { body, ...options });
14
- }
15
- }
16
- exports.Experiments = Experiments;
@@ -1,23 +0,0 @@
1
- import { APIResource } from '../../core/resource.js';
2
- import { APIPromise } from '../../core/api-promise.js';
3
- import { RequestOptions } from '../../internal/request-options.js';
4
- export declare class Experiments extends APIResource {
5
- /**
6
- * Stream grouped and aggregated experiments.
7
- */
8
- grouped(datasetID: string, body: ExperimentGroupedParams, options?: RequestOptions): APIPromise<unknown>;
9
- }
10
- export type ExperimentGroupedResponse = unknown;
11
- export interface ExperimentGroupedParams {
12
- metadata_keys: Array<string>;
13
- dataset_version?: string | null;
14
- experiment_limit?: number;
15
- filter?: string | null;
16
- name_contains?: string | null;
17
- stats_start_time?: string | null;
18
- tag_value_id?: Array<string> | null;
19
- use_approx_stats?: boolean;
20
- }
21
- export declare namespace Experiments {
22
- export { type ExperimentGroupedResponse as ExperimentGroupedResponse, type ExperimentGroupedParams as ExperimentGroupedParams, };
23
- }
@@ -1,12 +0,0 @@
1
- // @ts-nocheck
2
- // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
3
- import { APIResource } from '../../core/resource.js';
4
- import { path } from '../../internal/utils/path.js';
5
- export class Experiments extends APIResource {
6
- /**
7
- * Stream grouped and aggregated experiments.
8
- */
9
- grouped(datasetID, body, options) {
10
- return this._client.post(path `/api/v1/datasets/${datasetID}/experiments/grouped`, { body, ...options });
11
- }
12
- }
@@ -1,17 +0,0 @@
1
- "use strict";
2
- // @ts-nocheck
3
- // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
4
- Object.defineProperty(exports, "__esModule", { value: true });
5
- exports.Group = void 0;
6
- const resource_js_1 = require("../../core/resource.cjs");
7
- const path_js_1 = require("../../internal/utils/path.cjs");
8
- class Group extends resource_js_1.APIResource {
9
- /**
10
- * Fetch examples for a dataset, and fetch the runs for each example if they are
11
- * associated with the given session_ids.
12
- */
13
- runs(datasetID, body, options) {
14
- return this._client.post((0, path_js_1.path) `/api/v1/datasets/${datasetID}/group/runs`, { body, ...options });
15
- }
16
- }
17
- exports.Group = Group;
@@ -1,160 +0,0 @@
1
- import { APIResource } from '../../core/resource.js';
2
- import * as RunsAPI from './runs.js';
3
- import { APIPromise } from '../../core/api-promise.js';
4
- import { RequestOptions } from '../../internal/request-options.js';
5
- export declare class Group extends APIResource {
6
- /**
7
- * Fetch examples for a dataset, and fetch the runs for each example if they are
8
- * associated with the given session_ids.
9
- */
10
- runs(datasetID: string, body: GroupRunsParams, options?: RequestOptions): APIPromise<GroupRunsResponse>;
11
- }
12
- /**
13
- * Response for grouped comparison view of dataset examples.
14
- *
15
- * Returns dataset examples grouped by a run metadata value (e.g., model='gpt-4').
16
- * Optional filters are applied to all runs before grouping.
17
- *
18
- * Shows:
19
- *
20
- * - Which examples were executed with each metadata value
21
- * - Per-session aggregate statistics for runs on those examples
22
- * - The actual example data with their associated runs
23
- *
24
- * Used for comparing how different sessions performed on the same set of examples.
25
- */
26
- export interface GroupRunsResponse {
27
- groups: Array<GroupRunsResponse.Group>;
28
- }
29
- export declare namespace GroupRunsResponse {
30
- /**
31
- * Group of examples with a specific metadata value across multiple sessions.
32
- *
33
- * Extends RunGroupBase with:
34
- *
35
- * - group_key: metadata value that defines this group
36
- * - sessions: per-session stats for runs matching this metadata value
37
- * - examples: shared examples across all sessions (intersection logic) with flat
38
- * array of runs (each run has session_id field for frontend to determine column)
39
- * - example_count: unique example count (pagination-aware, same across all
40
- * sessions due to intersection)
41
- *
42
- * Inherited from RunGroupBase:
43
- *
44
- * - filter: metadata filter for this group (e.g., "and(eq(is_root, true),
45
- * and(eq(metadata_key, 'model'), eq(metadata_value, 'gpt-4')))")
46
- * - count: total run count across all sessions (includes duplicate runs)
47
- * - total_tokens, total_cost: aggregate across sessions
48
- * - min_start_time, max_start_time: time range across sessions
49
- * - latency_p50, latency_p99: aggregate latency stats across sessions
50
- * - feedback_stats: weighted average feedback across sessions
51
- *
52
- * Additional aggregate stats:
53
- *
54
- * - prompt_tokens, completion_tokens: separate token counts
55
- * - prompt_cost, completion_cost: separate costs
56
- * - error_rate: average error rate
57
- */
58
- interface Group {
59
- example_count: number;
60
- examples: Array<RunsAPI.ExampleWithRunsCh>;
61
- filter: string;
62
- group_key: string | number;
63
- sessions: Array<Group.Session>;
64
- completion_cost?: string | null;
65
- completion_tokens?: number | null;
66
- count?: number | null;
67
- error_rate?: number | null;
68
- feedback_stats?: {
69
- [key: string]: unknown;
70
- } | null;
71
- latency_p50?: number | null;
72
- latency_p99?: number | null;
73
- max_start_time?: string | null;
74
- min_start_time?: string | null;
75
- prompt_cost?: string | null;
76
- prompt_tokens?: number | null;
77
- total_cost?: string | null;
78
- total_tokens?: number | null;
79
- }
80
- namespace Group {
81
- /**
82
- * TracerSession stats filtered to runs matching a specific metadata value.
83
- *
84
- * Extends TracerSession with:
85
- *
86
- * - example_count: unique examples (vs run_count = total runs including
87
- * duplicates)
88
- * - filter: ClickHouse filter for fetching runs in this session/group
89
- * - min/max_start_time: time range for runs in this session/group
90
- */
91
- interface Session {
92
- id: string;
93
- filter: string;
94
- tenant_id: string;
95
- completion_cost?: string | null;
96
- completion_tokens?: number | null;
97
- default_dataset_id?: string | null;
98
- description?: string | null;
99
- end_time?: string | null;
100
- error_rate?: number | null;
101
- example_count?: number | null;
102
- experiment_progress?: Session.ExperimentProgress | null;
103
- extra?: {
104
- [key: string]: unknown;
105
- } | null;
106
- feedback_stats?: {
107
- [key: string]: unknown;
108
- } | null;
109
- first_token_p50?: number | null;
110
- first_token_p99?: number | null;
111
- last_run_start_time?: string | null;
112
- last_run_start_time_live?: string | null;
113
- latency_p50?: number | null;
114
- latency_p99?: number | null;
115
- max_start_time?: string | null;
116
- min_start_time?: string | null;
117
- name?: string;
118
- prompt_cost?: string | null;
119
- prompt_tokens?: number | null;
120
- reference_dataset_id?: string | null;
121
- run_count?: number | null;
122
- run_facets?: Array<{
123
- [key: string]: unknown;
124
- }> | null;
125
- session_feedback_stats?: {
126
- [key: string]: unknown;
127
- } | null;
128
- start_time?: string;
129
- streaming_rate?: number | null;
130
- test_run_number?: number | null;
131
- total_cost?: string | null;
132
- total_tokens?: number | null;
133
- trace_tier?: 'longlived' | 'shortlived' | null;
134
- }
135
- namespace Session {
136
- interface ExperimentProgress {
137
- evaluator_progress: {
138
- [key: string]: number;
139
- };
140
- expected_run_count: number;
141
- run_progress: number;
142
- }
143
- }
144
- }
145
- }
146
- export interface GroupRunsParams {
147
- group_by: 'run_metadata' | 'example_metadata';
148
- metadata_key: string;
149
- session_ids: Array<string>;
150
- filters?: {
151
- [key: string]: Array<string>;
152
- } | null;
153
- limit?: number;
154
- offset?: number;
155
- per_group_limit?: number;
156
- preview?: boolean;
157
- }
158
- export declare namespace Group {
159
- export { type GroupRunsResponse as GroupRunsResponse, type GroupRunsParams as GroupRunsParams };
160
- }
@@ -1,13 +0,0 @@
1
- // @ts-nocheck
2
- // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
3
- import { APIResource } from '../../core/resource.js';
4
- import { path } from '../../internal/utils/path.js';
5
- export class Group extends APIResource {
6
- /**
7
- * Fetch examples for a dataset, and fetch the runs for each example if they are
8
- * associated with the given session_ids.
9
- */
10
- runs(datasetID, body, options) {
11
- return this._client.post(path `/api/v1/datasets/${datasetID}/group/runs`, { body, ...options });
12
- }
13
- }
@@ -1,54 +0,0 @@
1
- "use strict";
2
- // @ts-nocheck
3
- // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
4
- Object.defineProperty(exports, "__esModule", { value: true });
5
- exports.Insights = void 0;
6
- const resource_js_1 = require("../../core/resource.cjs");
7
- const pagination_js_1 = require("../../core/pagination.cjs");
8
- const path_js_1 = require("../../internal/utils/path.cjs");
9
- class Insights extends resource_js_1.APIResource {
10
- /**
11
- * Create an insights job.
12
- */
13
- create(sessionID, body, options) {
14
- return this._client.post((0, path_js_1.path) `/api/v1/sessions/${sessionID}/insights`, { body, ...options });
15
- }
16
- /**
17
- * Update a session cluster job.
18
- */
19
- update(jobID, params, options) {
20
- const { session_id, ...body } = params;
21
- return this._client.patch((0, path_js_1.path) `/api/v1/sessions/${session_id}/insights/${jobID}`, { body, ...options });
22
- }
23
- /**
24
- * Get all clusters for a session.
25
- */
26
- list(sessionID, query = {}, options) {
27
- return this._client.getAPIList((0, path_js_1.path) `/api/v1/sessions/${sessionID}/insights`, (pagination_js_1.OffsetPaginationInsightsClusteringJobs), { query, ...options });
28
- }
29
- /**
30
- * Delete a session cluster job.
31
- */
32
- delete(jobID, params, options) {
33
- const { session_id } = params;
34
- return this._client.delete((0, path_js_1.path) `/api/v1/sessions/${session_id}/insights/${jobID}`, options);
35
- }
36
- /**
37
- * Get a specific cluster job for a session.
38
- */
39
- retrieveJob(jobID, params, options) {
40
- const { session_id } = params;
41
- return this._client.get((0, path_js_1.path) `/api/v1/sessions/${session_id}/insights/${jobID}`, options);
42
- }
43
- /**
44
- * Get all runs for a cluster job, optionally filtered by cluster.
45
- */
46
- retrieveRuns(jobID, params, options) {
47
- const { session_id, ...query } = params;
48
- return this._client.get((0, path_js_1.path) `/api/v1/sessions/${session_id}/insights/${jobID}/runs`, {
49
- query,
50
- ...options,
51
- });
52
- }
53
- }
54
- exports.Insights = Insights;