langsmith 0.7.13 → 0.7.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_openapi_client/client.cjs +9 -1
- package/dist/_openapi_client/client.d.ts +10 -5
- package/dist/_openapi_client/client.js +9 -1
- package/dist/_openapi_client/core/pagination.cjs +29 -1
- package/dist/_openapi_client/core/pagination.d.ts +11 -0
- package/dist/_openapi_client/core/pagination.js +27 -0
- package/dist/_openapi_client/resources/datasets/datasets.cjs +5 -14
- package/dist/_openapi_client/resources/datasets/datasets.d.ts +6 -10
- package/dist/_openapi_client/resources/datasets/datasets.js +5 -14
- package/dist/_openapi_client/resources/datasets/experiment-runs.cjs +18 -0
- package/dist/_openapi_client/resources/datasets/experiment-runs.d.ts +109 -0
- package/dist/_openapi_client/resources/datasets/experiment-runs.js +14 -0
- package/dist/_openapi_client/resources/datasets/runs.cjs +0 -7
- package/dist/_openapi_client/resources/datasets/runs.d.ts +4 -45
- package/dist/_openapi_client/resources/datasets/runs.js +0 -7
- package/dist/_openapi_client/resources/index.cjs +4 -2
- package/dist/_openapi_client/resources/index.d.ts +3 -2
- package/dist/_openapi_client/resources/index.js +2 -1
- package/dist/_openapi_client/resources/issues.cjs +31 -0
- package/dist/_openapi_client/resources/issues.d.ts +78 -0
- package/dist/_openapi_client/resources/issues.js +27 -0
- package/dist/_openapi_client/resources/runs/index.d.ts +1 -1
- package/dist/_openapi_client/resources/runs/runs.cjs +2 -2
- package/dist/_openapi_client/resources/runs/runs.d.ts +24 -24
- package/dist/_openapi_client/resources/runs/runs.js +2 -2
- package/dist/_openapi_client/resources/sandboxes/boxes.d.ts +7 -0
- package/dist/_openapi_client/resources/sandboxes/sandboxes.d.ts +1 -0
- package/dist/_openapi_client/resources/sessions.cjs +53 -0
- package/dist/_openapi_client/resources/{sessions/sessions.d.ts → sessions.d.ts} +14 -52
- package/dist/_openapi_client/resources/sessions.js +49 -0
- package/dist/client.cjs +40 -0
- package/dist/client.d.ts +28 -0
- package/dist/client.js +40 -0
- package/dist/experimental/anthropic/context.cjs +6 -1
- package/dist/experimental/anthropic/context.js +6 -1
- package/dist/index.cjs +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/utils/constants.cjs +1 -1
- package/dist/utils/constants.d.ts +1 -1
- package/dist/utils/constants.js +1 -1
- package/dist/utils/env.cjs +1 -1
- package/dist/utils/env.d.ts +0 -7
- package/dist/utils/env.js +1 -1
- package/package.json +10 -10
- package/dist/_openapi_client/resources/datasets/experiments.cjs +0 -16
- package/dist/_openapi_client/resources/datasets/experiments.d.ts +0 -23
- package/dist/_openapi_client/resources/datasets/experiments.js +0 -12
- package/dist/_openapi_client/resources/datasets/group.cjs +0 -17
- package/dist/_openapi_client/resources/datasets/group.d.ts +0 -160
- package/dist/_openapi_client/resources/datasets/group.js +0 -13
- package/dist/_openapi_client/resources/sessions/insights.cjs +0 -54
- package/dist/_openapi_client/resources/sessions/insights.d.ts +0 -246
- package/dist/_openapi_client/resources/sessions/insights.js +0 -50
- package/dist/_openapi_client/resources/sessions/sessions.cjs +0 -109
- package/dist/_openapi_client/resources/sessions/sessions.js +0 -72
package/dist/client.d.ts
CHANGED
|
@@ -6,6 +6,8 @@ import { EvaluationResult, EvaluationResults } from "./evaluation/evaluator.js";
|
|
|
6
6
|
import { OnlineEvaluators } from "./_openapi_client/resources/online-evaluators.js";
|
|
7
7
|
import { Runs as OpenAPIRuns } from "./_openapi_client/resources/runs.js";
|
|
8
8
|
import { Sandboxes } from "./_openapi_client/resources/sandboxes/sandboxes.js";
|
|
9
|
+
import { Datasets } from "./_openapi_client/resources/datasets/datasets.js";
|
|
10
|
+
import { Sessions } from "./_openapi_client/resources/sessions.js";
|
|
9
11
|
import { PromptCache } from "./utils/prompt_cache/index.js";
|
|
10
12
|
import { ProfileAuth } from "./utils/profiles.js";
|
|
11
13
|
export interface ClientConfig {
|
|
@@ -14,6 +16,14 @@ export interface ClientConfig {
|
|
|
14
16
|
callerOptions?: AsyncCallerParams;
|
|
15
17
|
timeout_ms?: number;
|
|
16
18
|
webUrl?: string;
|
|
19
|
+
/**
|
|
20
|
+
* A function applied for masking serialized run inputs and outputs,
|
|
21
|
+
* before sending to the API. Can be called with raw inputs, raw
|
|
22
|
+
* outputs, or a nested `{ error: string }` object for errors.
|
|
23
|
+
*
|
|
24
|
+
* If a `hideInputs` or `hideOutputs` function is present,
|
|
25
|
+
* the client will call it instead of the anonymizer as appropriate.
|
|
26
|
+
*/
|
|
17
27
|
anonymizer?: (values: KVMap) => KVMap | Promise<KVMap>;
|
|
18
28
|
hideInputs?: boolean | ((inputs: KVMap) => KVMap | Promise<KVMap>);
|
|
19
29
|
hideOutputs?: boolean | ((outputs: KVMap) => KVMap | Promise<KVMap>);
|
|
@@ -430,6 +440,7 @@ export declare class Client implements LangSmithTracingClientInterface {
|
|
|
430
440
|
private hideInputs?;
|
|
431
441
|
private hideOutputs?;
|
|
432
442
|
private hideMetadata?;
|
|
443
|
+
private anonymizer?;
|
|
433
444
|
private omitTracedRuntimeInfo?;
|
|
434
445
|
private tracingSampleRate?;
|
|
435
446
|
private filteredPostUuids;
|
|
@@ -507,9 +518,26 @@ export declare class Client implements LangSmithTracingClientInterface {
|
|
|
507
518
|
get runs(): OpenAPIRuns;
|
|
508
519
|
/** Access the v2 sandboxes resource (registries, snapshots, boxes). */
|
|
509
520
|
get sandboxes(): Sandboxes;
|
|
521
|
+
/** Access the projects resource. */
|
|
522
|
+
get projects(): Sessions;
|
|
523
|
+
/** Access the v2 datasets resource (experimentRuns, etc.). */
|
|
524
|
+
get datasets(): Datasets;
|
|
510
525
|
private processInputs;
|
|
511
526
|
private processOutputs;
|
|
512
527
|
private processMetadata;
|
|
528
|
+
/**
|
|
529
|
+
* Apply the configured anonymizer to a run's error string.
|
|
530
|
+
*
|
|
531
|
+
* Unlike inputs/outputs, `error` is a plain string (an exception message or
|
|
532
|
+
* traceback) that can carry credentials the user never explicitly logged --
|
|
533
|
+
* e.g. an HTTP-client error whose message embeds an `Authorization` header.
|
|
534
|
+
* The anonymizer is typed `(KVMap) => KVMap`, so the string is wrapped as
|
|
535
|
+
* `{ error }`, scrubbed, and unwrapped. Mirrors the Python SDK's
|
|
536
|
+
* `Client._hide_run_error`.
|
|
537
|
+
*
|
|
538
|
+
* TODO: Update anonymizer to always nest inputs/outputs/error for consistency
|
|
539
|
+
*/
|
|
540
|
+
private processError;
|
|
513
541
|
/**
|
|
514
542
|
* Filter content from new_token events to prevent streaming LLM output
|
|
515
543
|
* from being uploaded via events.
|
package/dist/client.js
CHANGED
|
@@ -504,6 +504,12 @@ export class Client {
|
|
|
504
504
|
writable: true,
|
|
505
505
|
value: void 0
|
|
506
506
|
});
|
|
507
|
+
Object.defineProperty(this, "anonymizer", {
|
|
508
|
+
enumerable: true,
|
|
509
|
+
configurable: true,
|
|
510
|
+
writable: true,
|
|
511
|
+
value: void 0
|
|
512
|
+
});
|
|
507
513
|
Object.defineProperty(this, "omitTracedRuntimeInfo", {
|
|
508
514
|
enumerable: true,
|
|
509
515
|
configurable: true,
|
|
@@ -755,6 +761,7 @@ export class Client {
|
|
|
755
761
|
this.hideOutputs =
|
|
756
762
|
config.hideOutputs ?? config.anonymizer ?? defaultConfig.hideOutputs;
|
|
757
763
|
this.hideMetadata = config.hideMetadata ?? defaultConfig.hideMetadata;
|
|
764
|
+
this.anonymizer = config.anonymizer;
|
|
758
765
|
this.omitTracedRuntimeInfo = config.omitTracedRuntimeInfo ?? false;
|
|
759
766
|
this.autoBatchTracing = config.autoBatchTracing ?? this.autoBatchTracing;
|
|
760
767
|
this.autoBatchQueue = new AutoBatchQueue(maxMemory);
|
|
@@ -936,6 +943,14 @@ export class Client {
|
|
|
936
943
|
get sandboxes() {
|
|
937
944
|
return this.openAPIClient.sandboxes;
|
|
938
945
|
}
|
|
946
|
+
/** Access the projects resource. */
|
|
947
|
+
get projects() {
|
|
948
|
+
return this.openAPIClient.sessions;
|
|
949
|
+
}
|
|
950
|
+
/** Access the v2 datasets resource (experimentRuns, etc.). */
|
|
951
|
+
get datasets() {
|
|
952
|
+
return this.openAPIClient.datasets;
|
|
953
|
+
}
|
|
939
954
|
async processInputs(inputs) {
|
|
940
955
|
if (this.hideInputs === false) {
|
|
941
956
|
return inputs;
|
|
@@ -972,6 +987,25 @@ export class Client {
|
|
|
972
987
|
}
|
|
973
988
|
return metadata;
|
|
974
989
|
}
|
|
990
|
+
/**
|
|
991
|
+
* Apply the configured anonymizer to a run's error string.
|
|
992
|
+
*
|
|
993
|
+
* Unlike inputs/outputs, `error` is a plain string (an exception message or
|
|
994
|
+
* traceback) that can carry credentials the user never explicitly logged --
|
|
995
|
+
* e.g. an HTTP-client error whose message embeds an `Authorization` header.
|
|
996
|
+
* The anonymizer is typed `(KVMap) => KVMap`, so the string is wrapped as
|
|
997
|
+
* `{ error }`, scrubbed, and unwrapped. Mirrors the Python SDK's
|
|
998
|
+
* `Client._hide_run_error`.
|
|
999
|
+
*
|
|
1000
|
+
* TODO: Update anonymizer to always nest inputs/outputs/error for consistency
|
|
1001
|
+
*/
|
|
1002
|
+
async processError(error) {
|
|
1003
|
+
if (this.anonymizer == null) {
|
|
1004
|
+
return error;
|
|
1005
|
+
}
|
|
1006
|
+
const result = await this.anonymizer({ error });
|
|
1007
|
+
return typeof result?.error === "string" ? result.error : error;
|
|
1008
|
+
}
|
|
975
1009
|
/**
|
|
976
1010
|
* Filter content from new_token events to prevent streaming LLM output
|
|
977
1011
|
* from being uploaded via events.
|
|
@@ -998,6 +1032,9 @@ export class Client {
|
|
|
998
1032
|
if (runParams.outputs !== undefined) {
|
|
999
1033
|
runParams.outputs = await this.processOutputs(runParams.outputs);
|
|
1000
1034
|
}
|
|
1035
|
+
if (runParams.error !== undefined) {
|
|
1036
|
+
runParams.error = await this.processError(runParams.error);
|
|
1037
|
+
}
|
|
1001
1038
|
if (runParams.extra != null && "metadata" in runParams.extra) {
|
|
1002
1039
|
runParams.extra = {
|
|
1003
1040
|
...runParams.extra,
|
|
@@ -1854,6 +1891,9 @@ export class Client {
|
|
|
1854
1891
|
if (run.outputs) {
|
|
1855
1892
|
run.outputs = await this.processOutputs(run.outputs);
|
|
1856
1893
|
}
|
|
1894
|
+
if (run.error) {
|
|
1895
|
+
run.error = await this.processError(run.error);
|
|
1896
|
+
}
|
|
1857
1897
|
if (run.extra != null && "metadata" in run.extra) {
|
|
1858
1898
|
run.extra = {
|
|
1859
1899
|
...run.extra,
|
|
@@ -136,10 +136,15 @@ class StreamManager {
|
|
|
136
136
|
}
|
|
137
137
|
this.namespaces["root"].extra ??= {};
|
|
138
138
|
this.namespaces["root"].extra.metadata ??= {};
|
|
139
|
-
|
|
139
|
+
// Keep the SDK's final aggregate usage available for inspection, but do not
|
|
140
|
+
// store it as `usage_metadata` on the parent chain. LangSmith rollups
|
|
141
|
+
// already aggregate child LLM usage, so parent `usage_metadata` would be
|
|
142
|
+
// counted again (notably doubling Anthropic cache-read/cache-write tokens).
|
|
143
|
+
this.namespaces["root"].extra.metadata.ls_aggregated_usage = usage;
|
|
140
144
|
this.namespaces["root"].extra.metadata.is_error = message.is_error;
|
|
141
145
|
this.namespaces["root"].extra.metadata.num_turns = message.num_turns;
|
|
142
146
|
this.namespaces["root"].extra.metadata.session_id = message.session_id;
|
|
147
|
+
this.namespaces["root"].extra.metadata.thread_id = message.session_id;
|
|
143
148
|
this.namespaces["root"].extra.metadata.duration_ms = message.duration_ms;
|
|
144
149
|
this.namespaces["root"].extra.metadata.duration_api_ms =
|
|
145
150
|
message.duration_api_ms;
|
|
@@ -133,10 +133,15 @@ export class StreamManager {
|
|
|
133
133
|
}
|
|
134
134
|
this.namespaces["root"].extra ??= {};
|
|
135
135
|
this.namespaces["root"].extra.metadata ??= {};
|
|
136
|
-
|
|
136
|
+
// Keep the SDK's final aggregate usage available for inspection, but do not
|
|
137
|
+
// store it as `usage_metadata` on the parent chain. LangSmith rollups
|
|
138
|
+
// already aggregate child LLM usage, so parent `usage_metadata` would be
|
|
139
|
+
// counted again (notably doubling Anthropic cache-read/cache-write tokens).
|
|
140
|
+
this.namespaces["root"].extra.metadata.ls_aggregated_usage = usage;
|
|
137
141
|
this.namespaces["root"].extra.metadata.is_error = message.is_error;
|
|
138
142
|
this.namespaces["root"].extra.metadata.num_turns = message.num_turns;
|
|
139
143
|
this.namespaces["root"].extra.metadata.session_id = message.session_id;
|
|
144
|
+
this.namespaces["root"].extra.metadata.thread_id = message.session_id;
|
|
140
145
|
this.namespaces["root"].extra.metadata.duration_ms = message.duration_ms;
|
|
141
146
|
this.namespaces["root"].extra.metadata.duration_api_ms =
|
|
142
147
|
message.duration_api_ms;
|
package/dist/index.cjs
CHANGED
|
@@ -20,6 +20,6 @@ Object.defineProperty(exports, "PromptCache", { enumerable: true, get: function
|
|
|
20
20
|
Object.defineProperty(exports, "configureGlobalPromptCache", { enumerable: true, get: function () { return index_js_1.configureGlobalPromptCache; } });
|
|
21
21
|
Object.defineProperty(exports, "promptCacheSingleton", { enumerable: true, get: function () { return index_js_1.promptCacheSingleton; } });
|
|
22
22
|
// Update using pnpm bump-version
|
|
23
|
-
exports.__version__ = "0.7.
|
|
23
|
+
exports.__version__ = "0.7.15";
|
|
24
24
|
// Metadata key to hide a traced run from LangSmith's Messages View.
|
|
25
25
|
exports.LS_MESSAGE_VIEW_EXCLUDE = "ls_message_view_exclude";
|
package/dist/index.d.ts
CHANGED
|
@@ -6,5 +6,5 @@ export { getDefaultProjectName } from "./utils/project.js";
|
|
|
6
6
|
export { uuid7, uuid7FromTime } from "./uuid.js";
|
|
7
7
|
export { isTracingEnabled } from "./utils/guard.js";
|
|
8
8
|
export { Cache, PromptCache, type CacheConfig, type CacheMetrics, configureGlobalPromptCache, promptCacheSingleton, } from "./utils/prompt_cache/index.js";
|
|
9
|
-
export declare const __version__ = "0.7.
|
|
9
|
+
export declare const __version__ = "0.7.15";
|
|
10
10
|
export declare const LS_MESSAGE_VIEW_EXCLUDE: "ls_message_view_exclude";
|
package/dist/index.js
CHANGED
|
@@ -6,6 +6,6 @@ export { uuid7, uuid7FromTime } from "./uuid.js";
|
|
|
6
6
|
export { isTracingEnabled } from "./utils/guard.js";
|
|
7
7
|
export { Cache, PromptCache, configureGlobalPromptCache, promptCacheSingleton, } from "./utils/prompt_cache/index.js";
|
|
8
8
|
// Update using pnpm bump-version
|
|
9
|
-
export const __version__ = "0.7.
|
|
9
|
+
export const __version__ = "0.7.15";
|
|
10
10
|
// Metadata key to hide a traced run from LangSmith's Messages View.
|
|
11
11
|
export const LS_MESSAGE_VIEW_EXCLUDE = "ls_message_view_exclude";
|
package/dist/utils/constants.cjs
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const _MIN_BACKEND_VERSION = "0.16.
|
|
1
|
+
export declare const _MIN_BACKEND_VERSION = "0.16.10rc1";
|
package/dist/utils/constants.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export const _MIN_BACKEND_VERSION = "0.16.
|
|
1
|
+
export const _MIN_BACKEND_VERSION = "0.16.10rc1";
|
package/dist/utils/env.cjs
CHANGED
|
@@ -24,7 +24,7 @@ const isJsDom = () => (typeof window !== "undefined" && window.name === "nodejs"
|
|
|
24
24
|
exports.isJsDom = isJsDom;
|
|
25
25
|
// Supabase Edge Function provides a `Deno` global object
|
|
26
26
|
// without `version` property
|
|
27
|
-
const isDeno = () => typeof Deno !== "undefined";
|
|
27
|
+
const isDeno = () => typeof globalThis.Deno !== "undefined";
|
|
28
28
|
exports.isDeno = isDeno;
|
|
29
29
|
// Mark not-as-node if in Supabase Edge Function
|
|
30
30
|
const isNode = () => typeof process !== "undefined" &&
|
package/dist/utils/env.d.ts
CHANGED
package/dist/utils/env.js
CHANGED
|
@@ -9,7 +9,7 @@ export const isJsDom = () => (typeof window !== "undefined" && window.name === "
|
|
|
9
9
|
(typeof navigator !== "undefined" && navigator.userAgent.includes("jsdom"));
|
|
10
10
|
// Supabase Edge Function provides a `Deno` global object
|
|
11
11
|
// without `version` property
|
|
12
|
-
export const isDeno = () => typeof Deno !== "undefined";
|
|
12
|
+
export const isDeno = () => typeof globalThis.Deno !== "undefined";
|
|
13
13
|
// Mark not-as-node if in Supabase Edge Function
|
|
14
14
|
export const isNode = () => typeof process !== "undefined" &&
|
|
15
15
|
typeof process.versions !== "undefined" &&
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "langsmith",
|
|
3
|
-
"version": "0.7.
|
|
3
|
+
"version": "0.7.15",
|
|
4
4
|
"description": "Client library to connect to the LangSmith Observability and Evaluation Platform.",
|
|
5
5
|
"packageManager": "pnpm@10.33.0",
|
|
6
6
|
"files": [
|
|
@@ -156,12 +156,12 @@
|
|
|
156
156
|
"p-queue": "6.6.2"
|
|
157
157
|
},
|
|
158
158
|
"devDependencies": {
|
|
159
|
-
"@ai-sdk/openai": "4.0.
|
|
160
|
-
"@ai-sdk/provider": "4.0.
|
|
159
|
+
"@ai-sdk/openai": "4.0.4",
|
|
160
|
+
"@ai-sdk/provider": "4.0.1",
|
|
161
161
|
"@ai-sdk/anthropic": "4.0.0-beta.67",
|
|
162
162
|
"@anthropic-ai/claude-agent-sdk": "^0.3.150",
|
|
163
163
|
"@anthropic-ai/sdk": "^0.98.0",
|
|
164
|
-
"@babel/preset-env": "^
|
|
164
|
+
"@babel/preset-env": "^8.0.2",
|
|
165
165
|
"@faker-js/faker": "^8.4.1",
|
|
166
166
|
"@google/genai": "^2.0.1",
|
|
167
167
|
"@jest/globals": "^30.4.1",
|
|
@@ -169,11 +169,11 @@
|
|
|
169
169
|
"@langchain/core": "^0.3.72",
|
|
170
170
|
"@langchain/langgraph": "^0.3.6",
|
|
171
171
|
"@langchain/openai": "^0.6.17",
|
|
172
|
-
"@openai/agents": "^0.
|
|
172
|
+
"@openai/agents": "^0.12.0",
|
|
173
173
|
"@opentelemetry/api": "^1.9.0",
|
|
174
|
-
"@opentelemetry/auto-instrumentations-node": "^0.
|
|
174
|
+
"@opentelemetry/auto-instrumentations-node": "^0.77.0",
|
|
175
175
|
"@opentelemetry/context-async-hooks": "^2.6.1",
|
|
176
|
-
"@opentelemetry/sdk-node": "^0.
|
|
176
|
+
"@opentelemetry/sdk-node": "^0.219.0",
|
|
177
177
|
"@opentelemetry/sdk-trace-base": "^2.0.0",
|
|
178
178
|
"@opentelemetry/sdk-trace-node": "^2.0.0",
|
|
179
179
|
"@tsconfig/recommended": "^1.0.2",
|
|
@@ -182,8 +182,8 @@
|
|
|
182
182
|
"@types/node-fetch": "^2.6.12",
|
|
183
183
|
"@types/semver": "^7.7.1",
|
|
184
184
|
"@types/ws": "^8.18.1",
|
|
185
|
-
"@typescript-eslint/eslint-plugin": "^
|
|
186
|
-
"@typescript-eslint/parser": "^
|
|
185
|
+
"@typescript-eslint/eslint-plugin": "^8.62.1",
|
|
186
|
+
"@typescript-eslint/parser": "^8.62.1",
|
|
187
187
|
"ai": "7.0.0-beta.178",
|
|
188
188
|
"babel-jest": "^30.2.0",
|
|
189
189
|
"cross-env": "^10.1.0",
|
|
@@ -194,7 +194,7 @@
|
|
|
194
194
|
"msw": "^2.11.2",
|
|
195
195
|
"node-fetch": "^3.3.2",
|
|
196
196
|
"openai": "^6.18.0",
|
|
197
|
-
"oxfmt": "^0.
|
|
197
|
+
"oxfmt": "^0.57.0",
|
|
198
198
|
"oxlint": "^1.62.0",
|
|
199
199
|
"semver": "^7.7.4",
|
|
200
200
|
"ts-jest": "^29.1.0",
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
// @ts-nocheck
|
|
3
|
-
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
|
|
4
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.Experiments = void 0;
|
|
6
|
-
const resource_js_1 = require("../../core/resource.cjs");
|
|
7
|
-
const path_js_1 = require("../../internal/utils/path.cjs");
|
|
8
|
-
class Experiments extends resource_js_1.APIResource {
|
|
9
|
-
/**
|
|
10
|
-
* Stream grouped and aggregated experiments.
|
|
11
|
-
*/
|
|
12
|
-
grouped(datasetID, body, options) {
|
|
13
|
-
return this._client.post((0, path_js_1.path) `/api/v1/datasets/${datasetID}/experiments/grouped`, { body, ...options });
|
|
14
|
-
}
|
|
15
|
-
}
|
|
16
|
-
exports.Experiments = Experiments;
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
import { APIResource } from '../../core/resource.js';
|
|
2
|
-
import { APIPromise } from '../../core/api-promise.js';
|
|
3
|
-
import { RequestOptions } from '../../internal/request-options.js';
|
|
4
|
-
export declare class Experiments extends APIResource {
|
|
5
|
-
/**
|
|
6
|
-
* Stream grouped and aggregated experiments.
|
|
7
|
-
*/
|
|
8
|
-
grouped(datasetID: string, body: ExperimentGroupedParams, options?: RequestOptions): APIPromise<unknown>;
|
|
9
|
-
}
|
|
10
|
-
export type ExperimentGroupedResponse = unknown;
|
|
11
|
-
export interface ExperimentGroupedParams {
|
|
12
|
-
metadata_keys: Array<string>;
|
|
13
|
-
dataset_version?: string | null;
|
|
14
|
-
experiment_limit?: number;
|
|
15
|
-
filter?: string | null;
|
|
16
|
-
name_contains?: string | null;
|
|
17
|
-
stats_start_time?: string | null;
|
|
18
|
-
tag_value_id?: Array<string> | null;
|
|
19
|
-
use_approx_stats?: boolean;
|
|
20
|
-
}
|
|
21
|
-
export declare namespace Experiments {
|
|
22
|
-
export { type ExperimentGroupedResponse as ExperimentGroupedResponse, type ExperimentGroupedParams as ExperimentGroupedParams, };
|
|
23
|
-
}
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
// @ts-nocheck
|
|
2
|
-
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
|
|
3
|
-
import { APIResource } from '../../core/resource.js';
|
|
4
|
-
import { path } from '../../internal/utils/path.js';
|
|
5
|
-
export class Experiments extends APIResource {
|
|
6
|
-
/**
|
|
7
|
-
* Stream grouped and aggregated experiments.
|
|
8
|
-
*/
|
|
9
|
-
grouped(datasetID, body, options) {
|
|
10
|
-
return this._client.post(path `/api/v1/datasets/${datasetID}/experiments/grouped`, { body, ...options });
|
|
11
|
-
}
|
|
12
|
-
}
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
// @ts-nocheck
|
|
3
|
-
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
|
|
4
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.Group = void 0;
|
|
6
|
-
const resource_js_1 = require("../../core/resource.cjs");
|
|
7
|
-
const path_js_1 = require("../../internal/utils/path.cjs");
|
|
8
|
-
class Group extends resource_js_1.APIResource {
|
|
9
|
-
/**
|
|
10
|
-
* Fetch examples for a dataset, and fetch the runs for each example if they are
|
|
11
|
-
* associated with the given session_ids.
|
|
12
|
-
*/
|
|
13
|
-
runs(datasetID, body, options) {
|
|
14
|
-
return this._client.post((0, path_js_1.path) `/api/v1/datasets/${datasetID}/group/runs`, { body, ...options });
|
|
15
|
-
}
|
|
16
|
-
}
|
|
17
|
-
exports.Group = Group;
|
|
@@ -1,160 +0,0 @@
|
|
|
1
|
-
import { APIResource } from '../../core/resource.js';
|
|
2
|
-
import * as RunsAPI from './runs.js';
|
|
3
|
-
import { APIPromise } from '../../core/api-promise.js';
|
|
4
|
-
import { RequestOptions } from '../../internal/request-options.js';
|
|
5
|
-
export declare class Group extends APIResource {
|
|
6
|
-
/**
|
|
7
|
-
* Fetch examples for a dataset, and fetch the runs for each example if they are
|
|
8
|
-
* associated with the given session_ids.
|
|
9
|
-
*/
|
|
10
|
-
runs(datasetID: string, body: GroupRunsParams, options?: RequestOptions): APIPromise<GroupRunsResponse>;
|
|
11
|
-
}
|
|
12
|
-
/**
|
|
13
|
-
* Response for grouped comparison view of dataset examples.
|
|
14
|
-
*
|
|
15
|
-
* Returns dataset examples grouped by a run metadata value (e.g., model='gpt-4').
|
|
16
|
-
* Optional filters are applied to all runs before grouping.
|
|
17
|
-
*
|
|
18
|
-
* Shows:
|
|
19
|
-
*
|
|
20
|
-
* - Which examples were executed with each metadata value
|
|
21
|
-
* - Per-session aggregate statistics for runs on those examples
|
|
22
|
-
* - The actual example data with their associated runs
|
|
23
|
-
*
|
|
24
|
-
* Used for comparing how different sessions performed on the same set of examples.
|
|
25
|
-
*/
|
|
26
|
-
export interface GroupRunsResponse {
|
|
27
|
-
groups: Array<GroupRunsResponse.Group>;
|
|
28
|
-
}
|
|
29
|
-
export declare namespace GroupRunsResponse {
|
|
30
|
-
/**
|
|
31
|
-
* Group of examples with a specific metadata value across multiple sessions.
|
|
32
|
-
*
|
|
33
|
-
* Extends RunGroupBase with:
|
|
34
|
-
*
|
|
35
|
-
* - group_key: metadata value that defines this group
|
|
36
|
-
* - sessions: per-session stats for runs matching this metadata value
|
|
37
|
-
* - examples: shared examples across all sessions (intersection logic) with flat
|
|
38
|
-
* array of runs (each run has session_id field for frontend to determine column)
|
|
39
|
-
* - example_count: unique example count (pagination-aware, same across all
|
|
40
|
-
* sessions due to intersection)
|
|
41
|
-
*
|
|
42
|
-
* Inherited from RunGroupBase:
|
|
43
|
-
*
|
|
44
|
-
* - filter: metadata filter for this group (e.g., "and(eq(is_root, true),
|
|
45
|
-
* and(eq(metadata_key, 'model'), eq(metadata_value, 'gpt-4')))")
|
|
46
|
-
* - count: total run count across all sessions (includes duplicate runs)
|
|
47
|
-
* - total_tokens, total_cost: aggregate across sessions
|
|
48
|
-
* - min_start_time, max_start_time: time range across sessions
|
|
49
|
-
* - latency_p50, latency_p99: aggregate latency stats across sessions
|
|
50
|
-
* - feedback_stats: weighted average feedback across sessions
|
|
51
|
-
*
|
|
52
|
-
* Additional aggregate stats:
|
|
53
|
-
*
|
|
54
|
-
* - prompt_tokens, completion_tokens: separate token counts
|
|
55
|
-
* - prompt_cost, completion_cost: separate costs
|
|
56
|
-
* - error_rate: average error rate
|
|
57
|
-
*/
|
|
58
|
-
interface Group {
|
|
59
|
-
example_count: number;
|
|
60
|
-
examples: Array<RunsAPI.ExampleWithRunsCh>;
|
|
61
|
-
filter: string;
|
|
62
|
-
group_key: string | number;
|
|
63
|
-
sessions: Array<Group.Session>;
|
|
64
|
-
completion_cost?: string | null;
|
|
65
|
-
completion_tokens?: number | null;
|
|
66
|
-
count?: number | null;
|
|
67
|
-
error_rate?: number | null;
|
|
68
|
-
feedback_stats?: {
|
|
69
|
-
[key: string]: unknown;
|
|
70
|
-
} | null;
|
|
71
|
-
latency_p50?: number | null;
|
|
72
|
-
latency_p99?: number | null;
|
|
73
|
-
max_start_time?: string | null;
|
|
74
|
-
min_start_time?: string | null;
|
|
75
|
-
prompt_cost?: string | null;
|
|
76
|
-
prompt_tokens?: number | null;
|
|
77
|
-
total_cost?: string | null;
|
|
78
|
-
total_tokens?: number | null;
|
|
79
|
-
}
|
|
80
|
-
namespace Group {
|
|
81
|
-
/**
|
|
82
|
-
* TracerSession stats filtered to runs matching a specific metadata value.
|
|
83
|
-
*
|
|
84
|
-
* Extends TracerSession with:
|
|
85
|
-
*
|
|
86
|
-
* - example_count: unique examples (vs run_count = total runs including
|
|
87
|
-
* duplicates)
|
|
88
|
-
* - filter: ClickHouse filter for fetching runs in this session/group
|
|
89
|
-
* - min/max_start_time: time range for runs in this session/group
|
|
90
|
-
*/
|
|
91
|
-
interface Session {
|
|
92
|
-
id: string;
|
|
93
|
-
filter: string;
|
|
94
|
-
tenant_id: string;
|
|
95
|
-
completion_cost?: string | null;
|
|
96
|
-
completion_tokens?: number | null;
|
|
97
|
-
default_dataset_id?: string | null;
|
|
98
|
-
description?: string | null;
|
|
99
|
-
end_time?: string | null;
|
|
100
|
-
error_rate?: number | null;
|
|
101
|
-
example_count?: number | null;
|
|
102
|
-
experiment_progress?: Session.ExperimentProgress | null;
|
|
103
|
-
extra?: {
|
|
104
|
-
[key: string]: unknown;
|
|
105
|
-
} | null;
|
|
106
|
-
feedback_stats?: {
|
|
107
|
-
[key: string]: unknown;
|
|
108
|
-
} | null;
|
|
109
|
-
first_token_p50?: number | null;
|
|
110
|
-
first_token_p99?: number | null;
|
|
111
|
-
last_run_start_time?: string | null;
|
|
112
|
-
last_run_start_time_live?: string | null;
|
|
113
|
-
latency_p50?: number | null;
|
|
114
|
-
latency_p99?: number | null;
|
|
115
|
-
max_start_time?: string | null;
|
|
116
|
-
min_start_time?: string | null;
|
|
117
|
-
name?: string;
|
|
118
|
-
prompt_cost?: string | null;
|
|
119
|
-
prompt_tokens?: number | null;
|
|
120
|
-
reference_dataset_id?: string | null;
|
|
121
|
-
run_count?: number | null;
|
|
122
|
-
run_facets?: Array<{
|
|
123
|
-
[key: string]: unknown;
|
|
124
|
-
}> | null;
|
|
125
|
-
session_feedback_stats?: {
|
|
126
|
-
[key: string]: unknown;
|
|
127
|
-
} | null;
|
|
128
|
-
start_time?: string;
|
|
129
|
-
streaming_rate?: number | null;
|
|
130
|
-
test_run_number?: number | null;
|
|
131
|
-
total_cost?: string | null;
|
|
132
|
-
total_tokens?: number | null;
|
|
133
|
-
trace_tier?: 'longlived' | 'shortlived' | null;
|
|
134
|
-
}
|
|
135
|
-
namespace Session {
|
|
136
|
-
interface ExperimentProgress {
|
|
137
|
-
evaluator_progress: {
|
|
138
|
-
[key: string]: number;
|
|
139
|
-
};
|
|
140
|
-
expected_run_count: number;
|
|
141
|
-
run_progress: number;
|
|
142
|
-
}
|
|
143
|
-
}
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
export interface GroupRunsParams {
|
|
147
|
-
group_by: 'run_metadata' | 'example_metadata';
|
|
148
|
-
metadata_key: string;
|
|
149
|
-
session_ids: Array<string>;
|
|
150
|
-
filters?: {
|
|
151
|
-
[key: string]: Array<string>;
|
|
152
|
-
} | null;
|
|
153
|
-
limit?: number;
|
|
154
|
-
offset?: number;
|
|
155
|
-
per_group_limit?: number;
|
|
156
|
-
preview?: boolean;
|
|
157
|
-
}
|
|
158
|
-
export declare namespace Group {
|
|
159
|
-
export { type GroupRunsResponse as GroupRunsResponse, type GroupRunsParams as GroupRunsParams };
|
|
160
|
-
}
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
// @ts-nocheck
|
|
2
|
-
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
|
|
3
|
-
import { APIResource } from '../../core/resource.js';
|
|
4
|
-
import { path } from '../../internal/utils/path.js';
|
|
5
|
-
export class Group extends APIResource {
|
|
6
|
-
/**
|
|
7
|
-
* Fetch examples for a dataset, and fetch the runs for each example if they are
|
|
8
|
-
* associated with the given session_ids.
|
|
9
|
-
*/
|
|
10
|
-
runs(datasetID, body, options) {
|
|
11
|
-
return this._client.post(path `/api/v1/datasets/${datasetID}/group/runs`, { body, ...options });
|
|
12
|
-
}
|
|
13
|
-
}
|
|
@@ -1,54 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
// @ts-nocheck
|
|
3
|
-
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
|
|
4
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.Insights = void 0;
|
|
6
|
-
const resource_js_1 = require("../../core/resource.cjs");
|
|
7
|
-
const pagination_js_1 = require("../../core/pagination.cjs");
|
|
8
|
-
const path_js_1 = require("../../internal/utils/path.cjs");
|
|
9
|
-
class Insights extends resource_js_1.APIResource {
|
|
10
|
-
/**
|
|
11
|
-
* Create an insights job.
|
|
12
|
-
*/
|
|
13
|
-
create(sessionID, body, options) {
|
|
14
|
-
return this._client.post((0, path_js_1.path) `/api/v1/sessions/${sessionID}/insights`, { body, ...options });
|
|
15
|
-
}
|
|
16
|
-
/**
|
|
17
|
-
* Update a session cluster job.
|
|
18
|
-
*/
|
|
19
|
-
update(jobID, params, options) {
|
|
20
|
-
const { session_id, ...body } = params;
|
|
21
|
-
return this._client.patch((0, path_js_1.path) `/api/v1/sessions/${session_id}/insights/${jobID}`, { body, ...options });
|
|
22
|
-
}
|
|
23
|
-
/**
|
|
24
|
-
* Get all clusters for a session.
|
|
25
|
-
*/
|
|
26
|
-
list(sessionID, query = {}, options) {
|
|
27
|
-
return this._client.getAPIList((0, path_js_1.path) `/api/v1/sessions/${sessionID}/insights`, (pagination_js_1.OffsetPaginationInsightsClusteringJobs), { query, ...options });
|
|
28
|
-
}
|
|
29
|
-
/**
|
|
30
|
-
* Delete a session cluster job.
|
|
31
|
-
*/
|
|
32
|
-
delete(jobID, params, options) {
|
|
33
|
-
const { session_id } = params;
|
|
34
|
-
return this._client.delete((0, path_js_1.path) `/api/v1/sessions/${session_id}/insights/${jobID}`, options);
|
|
35
|
-
}
|
|
36
|
-
/**
|
|
37
|
-
* Get a specific cluster job for a session.
|
|
38
|
-
*/
|
|
39
|
-
retrieveJob(jobID, params, options) {
|
|
40
|
-
const { session_id } = params;
|
|
41
|
-
return this._client.get((0, path_js_1.path) `/api/v1/sessions/${session_id}/insights/${jobID}`, options);
|
|
42
|
-
}
|
|
43
|
-
/**
|
|
44
|
-
* Get all runs for a cluster job, optionally filtered by cluster.
|
|
45
|
-
*/
|
|
46
|
-
retrieveRuns(jobID, params, options) {
|
|
47
|
-
const { session_id, ...query } = params;
|
|
48
|
-
return this._client.get((0, path_js_1.path) `/api/v1/sessions/${session_id}/insights/${jobID}/runs`, {
|
|
49
|
-
query,
|
|
50
|
-
...options,
|
|
51
|
-
});
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
exports.Insights = Insights;
|