langwatch 1.14.0 → 1.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/index.js +1 -1
- package/dist/agent/index.js.map +1 -1
- package/dist/agent/index.mjs +1 -1
- package/dist/agent/index.mjs.map +1 -1
- package/dist/{chunk-MGASIVXD.js → chunk-6SSJPQAW.js} +2 -2
- package/dist/chunk-6SSJPQAW.js.map +1 -0
- package/dist/{chunk-7FIG44IG.js → chunk-FUDF46YW.js} +12 -12
- package/dist/{chunk-7FIG44IG.js.map → chunk-FUDF46YW.js.map} +1 -1
- package/dist/{chunk-QKIDVFHZ.mjs → chunk-NZP72HTX.mjs} +2 -2
- package/dist/chunk-NZP72HTX.mjs.map +1 -0
- package/dist/{chunk-MCM4C66E.mjs → chunk-XC6AXWVR.mjs} +2 -2
- package/dist/cli/bundle.js +328 -294
- package/dist/{implementation-BXX0qdV7.d.mts → implementation-BvHTdJLg.d.mts} +1 -1
- package/dist/{implementation-Cv7sBdwj.d.ts → implementation-Dlxw5hlM.d.ts} +1 -1
- package/dist/index.d.mts +141 -28
- package/dist/index.d.ts +141 -28
- package/dist/index.js +229 -43
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +198 -12
- package/dist/index.mjs.map +1 -1
- package/dist/observability-sdk/index.d.mts +3 -3
- package/dist/observability-sdk/index.d.ts +3 -3
- package/dist/observability-sdk/index.js +2 -2
- package/dist/observability-sdk/index.mjs +1 -1
- package/dist/observability-sdk/instrumentation/langchain/index.d.mts +1 -1
- package/dist/observability-sdk/instrumentation/langchain/index.d.ts +1 -1
- package/dist/observability-sdk/setup/node/index.js +3 -3
- package/dist/observability-sdk/setup/node/index.mjs +2 -2
- package/dist/{types-BejNp4Dw.d.ts → types-CzElA_6o.d.ts} +1439 -247
- package/dist/{types-DXNIJdZx.d.mts → types-Dd2d9hCy.d.mts} +1439 -247
- package/package.json +1 -3
- package/dist/chunk-MGASIVXD.js.map +0 -1
- package/dist/chunk-QKIDVFHZ.mjs.map +0 -1
- /package/dist/{chunk-MCM4C66E.mjs.map → chunk-XC6AXWVR.mjs.map} +0 -0
|
@@ -1390,11 +1390,15 @@ interface paths {
|
|
|
1390
1390
|
put?: never;
|
|
1391
1391
|
/**
|
|
1392
1392
|
* Run a LangWatchQL query
|
|
1393
|
-
* @description Executes one read-only LangWatchQL SELECT over the analytics
|
|
1393
|
+
* @description Executes one read-only LangWatchQL SELECT over the analytics views and returns typed columns, rows, execution statistics and diagnostics. The query runs as a restricted database identity scoped to the projects this key can read.
|
|
1394
1394
|
*
|
|
1395
1395
|
* Diagnostics are advisory and never reject a query. An empty diagnostics list means no known issue was detected. It is not proof that the answer is the one you meant.
|
|
1396
1396
|
*
|
|
1397
|
-
*
|
|
1397
|
+
* A projection may call the app functions the schema endpoint lists (`conversation`, `llm_readable_trace`, `llm_messages`, and so on). Those are computed by the application after the query, so they are allowed only as aliased entries in the top-level SELECT list; a call in WHERE, GROUP BY, ORDER BY, a join, a subquery or a nested expression is refused, and a UNION disqualifies both of its branches even where each reads as a top-level projection. A projection may also call the eval functions, which judge a text with a model and are charged for; their key is the text itself. A run that would need more distinct conversations, traces, spans or texts than the published cap answers 422 rather than a partial result, and a run whose texts would exceed the per-query token budget answers 422 before anything is sent.
|
|
1398
|
+
*
|
|
1399
|
+
* Any LangWatch API key — project, organization or personal — reaches every project it can read `analytics:view` on: an organization or personal key spans its projects, a project key its one. Rows from more than one project come back flagged with the `MULTI_PROJECT_RESULT` diagnostic — to read a single project, filter inside the statement with `WHERE TenantId = '<project id>'`.
|
|
1400
|
+
*
|
|
1401
|
+
* A statement that names no `LIMIT` is capped at 10,000 rows: that `LIMIT` is appended before the query runs. A statement whose own `LIMIT` asks for more is refused with `LIMIT_TOO_HIGH` — lower it and page the rest with `LIMIT`/`OFFSET` and an `ORDER BY`. When using `UNION`, every top-level branch must carry its own `LIMIT` clause of 10,000 rows or fewer, or the query is refused with `LIMIT_REQUIRED_PER_BRANCH`. A result whose body exceeds about 8,000,000 bytes is refused outright with `lwql_result_too_large`, never cut — select fewer columns or a smaller `LIMIT`.
|
|
1398
1402
|
*
|
|
1399
1403
|
* Failures answer with their real HTTP status (a refused query is 403, not 200) and this API's canonical error envelope — the same `code` and `meta` every other REST family publishes.
|
|
1400
1404
|
*/
|
|
@@ -1414,9 +1418,13 @@ interface paths {
|
|
|
1414
1418
|
};
|
|
1415
1419
|
/**
|
|
1416
1420
|
* Discover the queryable LangWatchQL schema
|
|
1417
|
-
* @description Lists the LangWatchQL analytics
|
|
1421
|
+
* @description Lists the LangWatchQL analytics views this key may query, with each column's type, description, the permissions that unlock it, and whether this caller holds them — plus each view's grain, join keys, partition-pruning time column, freshness and a runnable example query. It also lists, under `functions`, every function name a query may call.
|
|
1422
|
+
*
|
|
1423
|
+
* Under `appFunctions` it lists the app functions a projection may call, each with its signature, the type and encoding of the value it returns, how many distinct keys one run may read, and the permissions it needs.
|
|
1424
|
+
*
|
|
1425
|
+
* Scoped to the projects the credential can read and their permissions: a column or app function this key cannot read in every one of them is listed with `available: false` rather than hidden, so a caller can see what a wider key would unlock.
|
|
1418
1426
|
*
|
|
1419
|
-
*
|
|
1427
|
+
* Any LangWatch API key — project, organization or personal — reaches every project it can read `analytics:view` on: an organization or personal key spans its projects, a project key its one. Rows from more than one project come back flagged with the `MULTI_PROJECT_RESULT` diagnostic — to read a single project, filter inside the statement with `WHERE TenantId = '<project id>'`.
|
|
1420
1428
|
*/
|
|
1421
1429
|
get: operations["getApiV1QuerySchema"];
|
|
1422
1430
|
put?: never;
|
|
@@ -1427,6 +1435,38 @@ interface paths {
|
|
|
1427
1435
|
patch?: never;
|
|
1428
1436
|
trace?: never;
|
|
1429
1437
|
};
|
|
1438
|
+
"/api/v1/query/reference": {
|
|
1439
|
+
parameters: {
|
|
1440
|
+
query?: never;
|
|
1441
|
+
header?: never;
|
|
1442
|
+
path?: never;
|
|
1443
|
+
cookie?: never;
|
|
1444
|
+
};
|
|
1445
|
+
/**
|
|
1446
|
+
* Discover both query languages
|
|
1447
|
+
* @description Describes both query languages in one payload: LangWatchQL (SQL over the analytics views) with its schema, limits and endpoints, and the trace filter (a Lucene-flavored string over the trace list) with its syntax, its fields and their static value vocabularies, and the open-ended attribute namespaces.
|
|
1448
|
+
*
|
|
1449
|
+
* It also carries worked examples in both languages and a table saying which language answers which kind of question. Every example is checked against the real validator and the real translator before it ships, so a published example parses and compiles; whether THIS key can run one is its own `available` flag.
|
|
1450
|
+
*
|
|
1451
|
+
* Pure: it reads the catalogs and this key's own permissions, never the project's traces, so it answers from memory rather than from the database.
|
|
1452
|
+
*
|
|
1453
|
+
* It answers `Cache-Control: private, no-store`, because the document is shaped by the calling credential: `available`, the embedded schema and the gated columns all differ between keys, and a cache keyed on the URL or the project would replay one key's document to another. Ask for it again rather than storing it.
|
|
1454
|
+
*
|
|
1455
|
+
* The values a field actually holds change under you and are a separate call — `GET /api/traces/facets`.
|
|
1456
|
+
*
|
|
1457
|
+
* An example this key cannot run is listed with `available: false` and keeps its `requires.gates`, so a caller can see which permission it needs.
|
|
1458
|
+
*
|
|
1459
|
+
* Any credential for the project may read it. The trace filter half is the traces family's vocabulary, so a key scoped to `traces:view` alone is answered rather than refused; for that key the LangWatchQL half arrives with `lwql.enabled: false` and an empty schema. `GET /api/v1/query/schema` is stricter and refuses that key outright, which is why this document withholds the catalog rather than repeating it.
|
|
1460
|
+
*/
|
|
1461
|
+
get: operations["getApiV1QueryReference"];
|
|
1462
|
+
put?: never;
|
|
1463
|
+
post?: never;
|
|
1464
|
+
delete?: never;
|
|
1465
|
+
options?: never;
|
|
1466
|
+
head?: never;
|
|
1467
|
+
patch?: never;
|
|
1468
|
+
trace?: never;
|
|
1469
|
+
};
|
|
1430
1470
|
"/api/coding-agent/sessions/{sessionId}/events": {
|
|
1431
1471
|
parameters: {
|
|
1432
1472
|
query?: never;
|
|
@@ -2303,7 +2343,7 @@ interface paths {
|
|
|
2303
2343
|
put?: never;
|
|
2304
2344
|
/**
|
|
2305
2345
|
* Create virtual key
|
|
2306
|
-
* @description Mints a new virtual key and returns the secret exactly once. The caller MUST persist the `secret` value, because LangWatch stores only a hash. `scopes` defaults to the caller's project; org- and team-scoped keys require a scoped API key holding `virtualKeys:manage` at each requested scope. An org- or team-scoped key also needs a place for its traces and spend to land, and must say where: pass `trace_project_id` (needs `virtualKeys:manage` on that project). Without it, and without exactly one project scope to take it from, creation refuses with `gateway_trace_project_ambiguous`, because the spend would be attributed to the organization's hidden governance project and counted by no budget on the project you had in mind. An organization whose only project is the governance one is exempt, since there is nothing else to name; one with no governance project either refuses with `trace_project_required`. Send `Idempotency-Key` to make a retry safe: a replay returns the original response including its `secret`, which is the only way to recover a secret whose response was lost in transit.
|
|
2346
|
+
* @description Mints a new virtual key and returns the secret exactly once. The caller MUST persist the `secret` value, because LangWatch stores only a hash. With `reveal_once` the response withholds the secret and carries `reveal_id` and `preview` instead: the secret is parked for 24 hours and served once, to the person the key is for, through the LangWatch app, so a caller that only relays the key (an agent printing a snippet) never holds it. `scopes` defaults to the caller's project, where `virtualKeys:create` is enough; org- and team-scoped keys, or a key for another project, require a scoped API key holding `virtualKeys:manage` at each requested scope. An org- or team-scoped key also needs a place for its traces and spend to land, and must say where: pass `trace_project_id` (needs `virtualKeys:manage` on that project). Without it, and without exactly one project scope to take it from, creation refuses with `gateway_trace_project_ambiguous`, because the spend would be attributed to the organization's hidden governance project and counted by no budget on the project you had in mind. An organization whose only project is the governance one is exempt, since there is nothing else to name; one with no governance project either refuses with `trace_project_required`. Send `Idempotency-Key` to make a retry safe: a replay returns the original response including its `secret`, which is the only way to recover a secret whose response was lost in transit.
|
|
2307
2347
|
*/
|
|
2308
2348
|
post: operations["postApiGatewayV1VirtualKeys"];
|
|
2309
2349
|
delete?: never;
|
|
@@ -2808,7 +2848,7 @@ interface paths {
|
|
|
2808
2848
|
path?: never;
|
|
2809
2849
|
cookie?: never;
|
|
2810
2850
|
};
|
|
2811
|
-
/** @description List the open requests Langy made for a folder of mine
|
|
2851
|
+
/** @description List the open requests Langy made for a folder of mine, on every project I can read. Only the person Langy asked ever sees a request, and each one expires fifteen minutes after it was made. */
|
|
2812
2852
|
get: operations["listLangyControlRequests"];
|
|
2813
2853
|
put?: never;
|
|
2814
2854
|
post?: never;
|
|
@@ -3738,7 +3778,7 @@ interface paths {
|
|
|
3738
3778
|
};
|
|
3739
3779
|
/**
|
|
3740
3780
|
* Get a project
|
|
3741
|
-
* @description Get a project by ID
|
|
3781
|
+
* @description Get a project by ID without secret keys. Requires project:view permission.
|
|
3742
3782
|
*/
|
|
3743
3783
|
get: operations["getProject"];
|
|
3744
3784
|
put?: never;
|
|
@@ -3766,7 +3806,8 @@ interface paths {
|
|
|
3766
3806
|
};
|
|
3767
3807
|
/**
|
|
3768
3808
|
* Get the project API key
|
|
3769
|
-
* @
|
|
3809
|
+
* @deprecated
|
|
3810
|
+
* @description Deprecated. Project base keys can be revealed only by a signed-in project administrator in the browser or an approved device flow. Organization API keys are always refused with 403.
|
|
3770
3811
|
*/
|
|
3771
3812
|
get: operations["getProjectApiKey"];
|
|
3772
3813
|
put?: never;
|
|
@@ -3788,7 +3829,8 @@ interface paths {
|
|
|
3788
3829
|
put?: never;
|
|
3789
3830
|
/**
|
|
3790
3831
|
* Regenerate the project API key
|
|
3791
|
-
* @
|
|
3832
|
+
* @deprecated
|
|
3833
|
+
* @description Deprecated. Project base keys can be rotated only by a signed-in project administrator in the browser. Organization API keys are always refused with 403.
|
|
3792
3834
|
*/
|
|
3793
3835
|
post: operations["regenerateProjectApiKey"];
|
|
3794
3836
|
delete?: never;
|
|
@@ -3994,6 +4036,121 @@ interface paths {
|
|
|
3994
4036
|
patch?: never;
|
|
3995
4037
|
trace?: never;
|
|
3996
4038
|
};
|
|
4039
|
+
"/api/v1/instant-evals": {
|
|
4040
|
+
parameters: {
|
|
4041
|
+
query?: never;
|
|
4042
|
+
header?: never;
|
|
4043
|
+
path?: never;
|
|
4044
|
+
cookie?: never;
|
|
4045
|
+
};
|
|
4046
|
+
/** @description List the project's runs, newest first. The project comes from the credential, so a run of another project is never listed. Page through them with before, which takes the created time of the oldest run the previous page carried. */
|
|
4047
|
+
get: operations["listInstantEvalRuns"];
|
|
4048
|
+
put?: never;
|
|
4049
|
+
/** @description Start a run. The statement is accepted, its questions are derived from the eval functions it projects, and the judging happens on the queue: the answer is the queued run, and its progress is read back from the run endpoint. A statement the query policy refuses, one that projects no TraceId, one that projects no eval function, and a row limit past what the plan allows are all refused before anything is judged. Instead of a statement you may send a target and your questions, and the statement is written for you and handed back on the run; sending both is refused. */
|
|
4050
|
+
post: operations["createInstantEvalRun"];
|
|
4051
|
+
delete?: never;
|
|
4052
|
+
options?: never;
|
|
4053
|
+
head?: never;
|
|
4054
|
+
patch?: never;
|
|
4055
|
+
trace?: never;
|
|
4056
|
+
};
|
|
4057
|
+
"/api/v1/instant-evals/estimate": {
|
|
4058
|
+
parameters: {
|
|
4059
|
+
query?: never;
|
|
4060
|
+
header?: never;
|
|
4061
|
+
path?: never;
|
|
4062
|
+
cookie?: never;
|
|
4063
|
+
};
|
|
4064
|
+
get?: never;
|
|
4065
|
+
put?: never;
|
|
4066
|
+
/**
|
|
4067
|
+
* Estimate a run
|
|
4068
|
+
* @description Price a run without starting it. The rows are counted, a sample of their texts is measured, and the cost is worked out from that. Nothing is judged and nothing is charged. Takes the same body a run does, a statement or a target with questions.
|
|
4069
|
+
*/
|
|
4070
|
+
post: operations["estimateInstantEvalRun"];
|
|
4071
|
+
delete?: never;
|
|
4072
|
+
options?: never;
|
|
4073
|
+
head?: never;
|
|
4074
|
+
patch?: never;
|
|
4075
|
+
trace?: never;
|
|
4076
|
+
};
|
|
4077
|
+
"/api/v1/instant-evals/{id}": {
|
|
4078
|
+
parameters: {
|
|
4079
|
+
query?: never;
|
|
4080
|
+
header?: never;
|
|
4081
|
+
path?: never;
|
|
4082
|
+
cookie?: never;
|
|
4083
|
+
};
|
|
4084
|
+
/** @description Read one run: its status, how many rows it found and judged, how many matched in total and per question, what it could not answer, and the tokens, cost and price the judging came to. An id this project does not hold answers 404 instant_eval_not_found. */
|
|
4085
|
+
get: operations["getInstantEvalRun"];
|
|
4086
|
+
put?: never;
|
|
4087
|
+
post?: never;
|
|
4088
|
+
delete?: never;
|
|
4089
|
+
options?: never;
|
|
4090
|
+
head?: never;
|
|
4091
|
+
patch?: never;
|
|
4092
|
+
trace?: never;
|
|
4093
|
+
};
|
|
4094
|
+
"/api/v1/instant-evals/{id}/cancel": {
|
|
4095
|
+
parameters: {
|
|
4096
|
+
query?: never;
|
|
4097
|
+
header?: never;
|
|
4098
|
+
path?: never;
|
|
4099
|
+
cookie?: never;
|
|
4100
|
+
};
|
|
4101
|
+
get?: never;
|
|
4102
|
+
put?: never;
|
|
4103
|
+
/**
|
|
4104
|
+
* Cancel a run
|
|
4105
|
+
* @description Ask a run to stop. The run stops before its next page, so the pages it already judged keep their judgements and are still readable. A run that has already finished, failed or been cancelled answers 409 instant_eval_already_finished.
|
|
4106
|
+
*/
|
|
4107
|
+
post: operations["cancelInstantEvalRun"];
|
|
4108
|
+
delete?: never;
|
|
4109
|
+
options?: never;
|
|
4110
|
+
head?: never;
|
|
4111
|
+
patch?: never;
|
|
4112
|
+
trace?: never;
|
|
4113
|
+
};
|
|
4114
|
+
"/api/v1/instant-evals/{id}/results": {
|
|
4115
|
+
parameters: {
|
|
4116
|
+
query?: never;
|
|
4117
|
+
header?: never;
|
|
4118
|
+
path?: never;
|
|
4119
|
+
cookie?: never;
|
|
4120
|
+
};
|
|
4121
|
+
/**
|
|
4122
|
+
* Read a run's results
|
|
4123
|
+
* @description Read the run's judgements, one page at a time. Pass the cursor a page answers with to read the page after it; the last page carries no cursor, and no judgement is ever carried by two pages. Narrow the page with questionId, matched and status.
|
|
4124
|
+
*/
|
|
4125
|
+
get: operations["listInstantEvalRunResults"];
|
|
4126
|
+
put?: never;
|
|
4127
|
+
post?: never;
|
|
4128
|
+
delete?: never;
|
|
4129
|
+
options?: never;
|
|
4130
|
+
head?: never;
|
|
4131
|
+
patch?: never;
|
|
4132
|
+
trace?: never;
|
|
4133
|
+
};
|
|
4134
|
+
"/api/v1/instant-evals/{id}/sample": {
|
|
4135
|
+
parameters: {
|
|
4136
|
+
query?: never;
|
|
4137
|
+
header?: never;
|
|
4138
|
+
path?: never;
|
|
4139
|
+
cookie?: never;
|
|
4140
|
+
};
|
|
4141
|
+
/**
|
|
4142
|
+
* Sample a run
|
|
4143
|
+
* @description Read a few of the run's rows with the text that was judged beside the verdict it received. The text is re-read through the statement's own extraction functions, so nothing is judged again and reading a sample is free.
|
|
4144
|
+
*/
|
|
4145
|
+
get: operations["sampleInstantEvalRun"];
|
|
4146
|
+
put?: never;
|
|
4147
|
+
post?: never;
|
|
4148
|
+
delete?: never;
|
|
4149
|
+
options?: never;
|
|
4150
|
+
head?: never;
|
|
4151
|
+
patch?: never;
|
|
4152
|
+
trace?: never;
|
|
4153
|
+
};
|
|
3997
4154
|
"/api/v1/run-plans": {
|
|
3998
4155
|
parameters: {
|
|
3999
4156
|
query?: never;
|
|
@@ -4339,6 +4496,32 @@ interface paths {
|
|
|
4339
4496
|
patch?: never;
|
|
4340
4497
|
trace?: never;
|
|
4341
4498
|
};
|
|
4499
|
+
"/api/traces/facets": {
|
|
4500
|
+
parameters: {
|
|
4501
|
+
query?: never;
|
|
4502
|
+
header?: never;
|
|
4503
|
+
path?: never;
|
|
4504
|
+
cookie?: never;
|
|
4505
|
+
};
|
|
4506
|
+
/**
|
|
4507
|
+
* Discover what the trace filter fields hold
|
|
4508
|
+
* @description What the trace filter fields actually hold in THIS project, which the filter language's own reference deliberately does not carry: values are tenant data, they move under you, and reading them all costs about thirty aggregate queries.
|
|
4509
|
+
*
|
|
4510
|
+
* Two answers from one door. Without `field` you get the discovery payload: every facet this project has, each with its top values and counts, plus the range bounds for the numeric ones. With `field` you get one field's values, paged, filtered by `prefix`.
|
|
4511
|
+
*
|
|
4512
|
+
* The values are cached and refreshed in the background, so a cold project answers `pending: true` with the payload it has; call again shortly for the computed one.
|
|
4513
|
+
*
|
|
4514
|
+
* Use it whenever you are unsure how a value is spelled. `GET /api/v1/query/reference` lists the fields and their fixed vocabularies; only this endpoint knows the open ones.
|
|
4515
|
+
*/
|
|
4516
|
+
get: operations["getApiTracesFacets"];
|
|
4517
|
+
put?: never;
|
|
4518
|
+
post?: never;
|
|
4519
|
+
delete?: never;
|
|
4520
|
+
options?: never;
|
|
4521
|
+
head?: never;
|
|
4522
|
+
patch?: never;
|
|
4523
|
+
trace?: never;
|
|
4524
|
+
};
|
|
4342
4525
|
"/api/traces/{traceId}/transcript": {
|
|
4343
4526
|
parameters: {
|
|
4344
4527
|
query?: never;
|
|
@@ -5859,7 +6042,7 @@ interface operations {
|
|
|
5859
6042
|
* @description The kind of agent. A connected agent is registered from code by the SDK and cannot be created or reconfigured through this API.
|
|
5860
6043
|
* @enum {string}
|
|
5861
6044
|
*/
|
|
5862
|
-
type: "signature" | "code" | "workflow" | "http" | "connected";
|
|
6045
|
+
type: "signature" | "code" | "workflow" | "http" | "connected" | "voice";
|
|
5863
6046
|
config: {
|
|
5864
6047
|
[key: string]: unknown;
|
|
5865
6048
|
} | null;
|
|
@@ -5946,7 +6129,7 @@ interface operations {
|
|
|
5946
6129
|
* @description The kind of agent to write. A connected agent is registered from code by the SDK, so "connected" is refused with agent_register_only.
|
|
5947
6130
|
* @enum {string}
|
|
5948
6131
|
*/
|
|
5949
|
-
type: "signature" | "code" | "workflow" | "http" | "connected";
|
|
6132
|
+
type: "signature" | "code" | "workflow" | "http" | "connected" | "voice";
|
|
5950
6133
|
config: {
|
|
5951
6134
|
[key: string]: unknown;
|
|
5952
6135
|
};
|
|
@@ -5968,7 +6151,7 @@ interface operations {
|
|
|
5968
6151
|
* @description The kind of agent. A connected agent is registered from code by the SDK and cannot be created or reconfigured through this API.
|
|
5969
6152
|
* @enum {string}
|
|
5970
6153
|
*/
|
|
5971
|
-
type: "signature" | "code" | "workflow" | "http" | "connected";
|
|
6154
|
+
type: "signature" | "code" | "workflow" | "http" | "connected" | "voice";
|
|
5972
6155
|
config: {
|
|
5973
6156
|
[key: string]: unknown;
|
|
5974
6157
|
} | null;
|
|
@@ -6058,7 +6241,7 @@ interface operations {
|
|
|
6058
6241
|
* @description The kind of agent. A connected agent is registered from code by the SDK and cannot be created or reconfigured through this API.
|
|
6059
6242
|
* @enum {string}
|
|
6060
6243
|
*/
|
|
6061
|
-
type: "signature" | "code" | "workflow" | "http" | "connected";
|
|
6244
|
+
type: "signature" | "code" | "workflow" | "http" | "connected" | "voice";
|
|
6062
6245
|
config: {
|
|
6063
6246
|
[key: string]: unknown;
|
|
6064
6247
|
} | null;
|
|
@@ -6141,7 +6324,7 @@ interface operations {
|
|
|
6141
6324
|
* @description The kind of agent to write. A connected agent is registered from code by the SDK, so "connected" is refused with agent_register_only.
|
|
6142
6325
|
* @enum {string}
|
|
6143
6326
|
*/
|
|
6144
|
-
type?: "signature" | "code" | "workflow" | "http" | "connected";
|
|
6327
|
+
type?: "signature" | "code" | "workflow" | "http" | "connected" | "voice";
|
|
6145
6328
|
config?: {
|
|
6146
6329
|
[key: string]: unknown;
|
|
6147
6330
|
};
|
|
@@ -6163,7 +6346,7 @@ interface operations {
|
|
|
6163
6346
|
* @description The kind of agent. A connected agent is registered from code by the SDK and cannot be created or reconfigured through this API.
|
|
6164
6347
|
* @enum {string}
|
|
6165
6348
|
*/
|
|
6166
|
-
type: "signature" | "code" | "workflow" | "http" | "connected";
|
|
6349
|
+
type: "signature" | "code" | "workflow" | "http" | "connected" | "voice";
|
|
6167
6350
|
config: {
|
|
6168
6351
|
[key: string]: unknown;
|
|
6169
6352
|
} | null;
|
|
@@ -6250,7 +6433,7 @@ interface operations {
|
|
|
6250
6433
|
id: string;
|
|
6251
6434
|
name: string;
|
|
6252
6435
|
/** @enum {string} */
|
|
6253
|
-
type: "signature" | "code" | "workflow" | "http" | "connected";
|
|
6436
|
+
type: "signature" | "code" | "workflow" | "http" | "connected" | "voice";
|
|
6254
6437
|
archivedAt: string | null;
|
|
6255
6438
|
};
|
|
6256
6439
|
};
|
|
@@ -6275,7 +6458,7 @@ interface operations {
|
|
|
6275
6458
|
* @description The kind of agent to write. A connected agent is registered from code by the SDK, so "connected" is refused with agent_register_only.
|
|
6276
6459
|
* @enum {string}
|
|
6277
6460
|
*/
|
|
6278
|
-
type?: "signature" | "code" | "workflow" | "http" | "connected";
|
|
6461
|
+
type?: "signature" | "code" | "workflow" | "http" | "connected" | "voice";
|
|
6279
6462
|
config?: {
|
|
6280
6463
|
[key: string]: unknown;
|
|
6281
6464
|
};
|
|
@@ -6297,7 +6480,7 @@ interface operations {
|
|
|
6297
6480
|
* @description The kind of agent. A connected agent is registered from code by the SDK and cannot be created or reconfigured through this API.
|
|
6298
6481
|
* @enum {string}
|
|
6299
6482
|
*/
|
|
6300
|
-
type: "signature" | "code" | "workflow" | "http" | "connected";
|
|
6483
|
+
type: "signature" | "code" | "workflow" | "http" | "connected" | "voice";
|
|
6301
6484
|
config: {
|
|
6302
6485
|
[key: string]: unknown;
|
|
6303
6486
|
} | null;
|
|
@@ -7108,6 +7291,7 @@ interface operations {
|
|
|
7108
7291
|
};
|
|
7109
7292
|
traceIds?: string[];
|
|
7110
7293
|
negateFilters?: boolean;
|
|
7294
|
+
excludeOrigins?: string[];
|
|
7111
7295
|
series: {
|
|
7112
7296
|
/** @enum {string} */
|
|
7113
7297
|
metric: "metadata.trace_id" | "metadata.user_id" | "metadata.thread_id" | "metadata.span_type" | "sentiment.thumbs_up_down" | "performance.completion_time" | "performance.first_token" | "performance.total_cost" | "performance.cost_billed" | "performance.cost_non_billed" | "performance.prompt_tokens" | "performance.completion_tokens" | "performance.cache_read_tokens" | "performance.cache_write_tokens" | "performance.reasoning_tokens" | "performance.total_processed_tokens" | "performance.total_tokens" | "performance.tokens_per_second" | "events.event_type" | "events.event_score" | "events.event_details" | "evaluations.evaluation_score" | "evaluations.evaluation_pass_rate" | "evaluations.evaluation_runs" | "threads.average_duration_per_thread";
|
|
@@ -9226,7 +9410,7 @@ interface operations {
|
|
|
9226
9410
|
};
|
|
9227
9411
|
};
|
|
9228
9412
|
responses: {
|
|
9229
|
-
/** @description The query ran. Columns, rows, execution statistics
|
|
9413
|
+
/** @description The query ran. Columns, rows, execution statistics and diagnostics, scoped to the projects the key can read. */
|
|
9230
9414
|
200: {
|
|
9231
9415
|
headers: {
|
|
9232
9416
|
[name: string]: unknown;
|
|
@@ -9246,14 +9430,13 @@ interface operations {
|
|
|
9246
9430
|
bytesRead: number;
|
|
9247
9431
|
rowsReturned: number;
|
|
9248
9432
|
};
|
|
9249
|
-
truncated: boolean;
|
|
9250
9433
|
followsTimeWindow: boolean;
|
|
9251
9434
|
followsGranularity: boolean;
|
|
9252
9435
|
granularitySeconds?: number;
|
|
9253
9436
|
coarsenedFromSeconds?: number;
|
|
9254
9437
|
diagnostics: {
|
|
9255
9438
|
/** @enum {string} */
|
|
9256
|
-
code: "
|
|
9439
|
+
code: "MULTI_PROJECT_RESULT" | "POSSIBLE_FANOUT" | "UNBOUNDED_TIME_RANGE" | "MISSING_TIME_BUCKETS" | "INCOMPLETE_COMPARISON_PERIOD" | "APP_FUNCTION_VALUE_TRUNCATED" | "APP_FUNCTION_UNRESOLVED_KEYS" | "APP_FUNCTION_RESULT_TRUNCATED" | "INSTANT_EVAL_SKIPPED";
|
|
9257
9440
|
message: string;
|
|
9258
9441
|
meta?: {
|
|
9259
9442
|
[key: string]: unknown;
|
|
@@ -9373,7 +9556,7 @@ interface operations {
|
|
|
9373
9556
|
};
|
|
9374
9557
|
requestBody?: never;
|
|
9375
9558
|
responses: {
|
|
9376
|
-
/** @description The
|
|
9559
|
+
/** @description The views and columns this key may query, with the permissions that unlock each one. */
|
|
9377
9560
|
200: {
|
|
9378
9561
|
headers: {
|
|
9379
9562
|
[name: string]: unknown;
|
|
@@ -9381,7 +9564,7 @@ interface operations {
|
|
|
9381
9564
|
content: {
|
|
9382
9565
|
"application/json": {
|
|
9383
9566
|
database: string;
|
|
9384
|
-
|
|
9567
|
+
views: {
|
|
9385
9568
|
name: string;
|
|
9386
9569
|
description: string;
|
|
9387
9570
|
grain: string;
|
|
@@ -9399,6 +9582,23 @@ interface operations {
|
|
|
9399
9582
|
}[];
|
|
9400
9583
|
exampleSql: string;
|
|
9401
9584
|
}[];
|
|
9585
|
+
functions: string[];
|
|
9586
|
+
appFunctions: {
|
|
9587
|
+
name: string;
|
|
9588
|
+
signature: string;
|
|
9589
|
+
description: string;
|
|
9590
|
+
/** @enum {string} */
|
|
9591
|
+
kind: "extraction" | "eval";
|
|
9592
|
+
returns: string;
|
|
9593
|
+
/** @enum {string} */
|
|
9594
|
+
encoding: "text" | "json";
|
|
9595
|
+
/** @enum {string} */
|
|
9596
|
+
keyKind: "trace" | "thread" | "span" | "text";
|
|
9597
|
+
cap: number;
|
|
9598
|
+
gates: ("input" | "output" | "costs")[];
|
|
9599
|
+
available: boolean;
|
|
9600
|
+
exampleSql: string;
|
|
9601
|
+
}[];
|
|
9402
9602
|
};
|
|
9403
9603
|
};
|
|
9404
9604
|
};
|
|
@@ -9484,78 +9684,127 @@ interface operations {
|
|
|
9484
9684
|
};
|
|
9485
9685
|
};
|
|
9486
9686
|
};
|
|
9487
|
-
|
|
9687
|
+
getApiV1QueryReference: {
|
|
9488
9688
|
parameters: {
|
|
9489
|
-
query?:
|
|
9490
|
-
/** @description Comma-separated event kinds to include. Known kinds: model_call, compaction, rate_limit, api_error, retries_exhausted, tool_result, tool_decision, user_prompt, subagent_completed. */
|
|
9491
|
-
kinds?: string;
|
|
9492
|
-
/** @description Opaque keyset cursor from the previous response's nextCursor. */
|
|
9493
|
-
cursor?: string;
|
|
9494
|
-
limit?: number;
|
|
9495
|
-
/** @description Epoch ms lower bound on event time; with `to`, prunes storage partitions for faster reads. */
|
|
9496
|
-
from?: number;
|
|
9497
|
-
/** @description Epoch ms upper bound on event time. */
|
|
9498
|
-
to?: number;
|
|
9499
|
-
};
|
|
9689
|
+
query?: never;
|
|
9500
9690
|
header?: never;
|
|
9501
|
-
path
|
|
9502
|
-
/** @description The agent's own session id (session.id / conversation id). */
|
|
9503
|
-
sessionId: string;
|
|
9504
|
-
};
|
|
9691
|
+
path?: never;
|
|
9505
9692
|
cookie?: never;
|
|
9506
9693
|
};
|
|
9507
9694
|
requestBody?: never;
|
|
9508
9695
|
responses: {
|
|
9509
|
-
/** @description
|
|
9696
|
+
/** @description The LangWatchQL schema and limits, the trace filter's syntax and fields, worked examples in both languages, and which language answers which kind of question. */
|
|
9510
9697
|
200: {
|
|
9511
9698
|
headers: {
|
|
9512
9699
|
[name: string]: unknown;
|
|
9513
9700
|
};
|
|
9514
9701
|
content: {
|
|
9515
9702
|
"application/json": {
|
|
9516
|
-
|
|
9517
|
-
|
|
9518
|
-
|
|
9519
|
-
|
|
9520
|
-
|
|
9521
|
-
|
|
9522
|
-
|
|
9523
|
-
|
|
9524
|
-
|
|
9525
|
-
|
|
9526
|
-
|
|
9527
|
-
|
|
9528
|
-
|
|
9529
|
-
|
|
9530
|
-
|
|
9531
|
-
|
|
9532
|
-
|
|
9533
|
-
|
|
9534
|
-
|
|
9535
|
-
|
|
9536
|
-
|
|
9537
|
-
|
|
9538
|
-
|
|
9539
|
-
|
|
9540
|
-
|
|
9541
|
-
|
|
9542
|
-
|
|
9543
|
-
|
|
9544
|
-
|
|
9545
|
-
|
|
9546
|
-
|
|
9547
|
-
|
|
9548
|
-
|
|
9549
|
-
|
|
9550
|
-
|
|
9551
|
-
|
|
9552
|
-
|
|
9553
|
-
|
|
9554
|
-
|
|
9555
|
-
|
|
9556
|
-
|
|
9703
|
+
version: string;
|
|
9704
|
+
lwql: {
|
|
9705
|
+
enabled: boolean;
|
|
9706
|
+
schema: {
|
|
9707
|
+
database: string;
|
|
9708
|
+
views: {
|
|
9709
|
+
name: string;
|
|
9710
|
+
description: string;
|
|
9711
|
+
grain: string;
|
|
9712
|
+
joinKeys: string[];
|
|
9713
|
+
timeColumn: string;
|
|
9714
|
+
freshness: string;
|
|
9715
|
+
columns: {
|
|
9716
|
+
name: string;
|
|
9717
|
+
type: string;
|
|
9718
|
+
description: string;
|
|
9719
|
+
/** @enum {string|null} */
|
|
9720
|
+
unit: "ms" | "USD" | "tokens" | "tokens/s" | null;
|
|
9721
|
+
gates: ("input" | "output" | "costs")[];
|
|
9722
|
+
available: boolean;
|
|
9723
|
+
}[];
|
|
9724
|
+
exampleSql: string;
|
|
9725
|
+
}[];
|
|
9726
|
+
functions: string[];
|
|
9727
|
+
appFunctions: {
|
|
9728
|
+
name: string;
|
|
9729
|
+
signature: string;
|
|
9730
|
+
description: string;
|
|
9731
|
+
/** @enum {string} */
|
|
9732
|
+
kind: "extraction" | "eval";
|
|
9733
|
+
returns: string;
|
|
9734
|
+
/** @enum {string} */
|
|
9735
|
+
encoding: "text" | "json";
|
|
9736
|
+
/** @enum {string} */
|
|
9737
|
+
keyKind: "trace" | "thread" | "span" | "text";
|
|
9738
|
+
cap: number;
|
|
9739
|
+
gates: ("input" | "output" | "costs")[];
|
|
9740
|
+
available: boolean;
|
|
9741
|
+
exampleSql: string;
|
|
9742
|
+
}[];
|
|
9743
|
+
};
|
|
9744
|
+
limits: {
|
|
9745
|
+
maxStatementLength: number;
|
|
9746
|
+
maxRowsReturned: number;
|
|
9747
|
+
maxResultBytes: number;
|
|
9748
|
+
maxExecutionTimeSeconds: number;
|
|
9749
|
+
pagination: string;
|
|
9750
|
+
};
|
|
9751
|
+
endpoints: {
|
|
9752
|
+
/** @enum {string} */
|
|
9753
|
+
method: "GET" | "POST";
|
|
9754
|
+
path: string;
|
|
9755
|
+
description: string;
|
|
9756
|
+
}[];
|
|
9757
|
+
};
|
|
9758
|
+
traceFilter: {
|
|
9759
|
+
syntax: string;
|
|
9760
|
+
fields: {
|
|
9761
|
+
name: string;
|
|
9762
|
+
label: string;
|
|
9763
|
+
/** @enum {string} */
|
|
9764
|
+
valueType: "categorical" | "range" | "text" | "existence";
|
|
9765
|
+
group: string | null;
|
|
9766
|
+
facetable: boolean;
|
|
9767
|
+
knownValues: string[];
|
|
9768
|
+
}[];
|
|
9769
|
+
dynamicPrefixes: {
|
|
9770
|
+
prefix: string;
|
|
9771
|
+
label: string;
|
|
9772
|
+
description: string;
|
|
9773
|
+
aliases: string[];
|
|
9774
|
+
}[];
|
|
9775
|
+
endpoints: {
|
|
9776
|
+
/** @enum {string} */
|
|
9777
|
+
method: "GET" | "POST";
|
|
9778
|
+
path: string;
|
|
9779
|
+
description: string;
|
|
9780
|
+
}[];
|
|
9781
|
+
};
|
|
9782
|
+
examples: {
|
|
9783
|
+
id: string;
|
|
9784
|
+
title: string;
|
|
9785
|
+
/** @enum {string} */
|
|
9786
|
+
intent: "triage" | "cost" | "latency" | "quality" | "conversations" | "discovery" | "export";
|
|
9787
|
+
/** @enum {string} */
|
|
9788
|
+
language: "lwql" | "trace-filter";
|
|
9789
|
+
tags: string[];
|
|
9790
|
+
text: string;
|
|
9791
|
+
parameters: {
|
|
9792
|
+
name: string;
|
|
9793
|
+
type: string;
|
|
9794
|
+
description: string;
|
|
9795
|
+
}[];
|
|
9796
|
+
requires: {
|
|
9797
|
+
gates: ("input" | "output" | "costs")[];
|
|
9798
|
+
functions: string[];
|
|
9799
|
+
};
|
|
9800
|
+
available: boolean;
|
|
9801
|
+
notes?: string;
|
|
9802
|
+
}[];
|
|
9803
|
+
decisionTable: {
|
|
9804
|
+
when: string;
|
|
9805
|
+
use: string;
|
|
9806
|
+
why: string;
|
|
9557
9807
|
}[];
|
|
9558
|
-
nextCursor: string | null;
|
|
9559
9808
|
};
|
|
9560
9809
|
};
|
|
9561
9810
|
};
|
|
@@ -9566,14 +9815,171 @@ interface operations {
|
|
|
9566
9815
|
};
|
|
9567
9816
|
content: {
|
|
9568
9817
|
"application/json": {
|
|
9569
|
-
error:
|
|
9570
|
-
|
|
9571
|
-
|
|
9572
|
-
|
|
9573
|
-
|
|
9574
|
-
|
|
9575
|
-
|
|
9576
|
-
|
|
9818
|
+
error: {
|
|
9819
|
+
type: string;
|
|
9820
|
+
code: string;
|
|
9821
|
+
message: string;
|
|
9822
|
+
meta?: {
|
|
9823
|
+
[key: string]: unknown;
|
|
9824
|
+
};
|
|
9825
|
+
trace_id?: string;
|
|
9826
|
+
span_id?: string;
|
|
9827
|
+
};
|
|
9828
|
+
};
|
|
9829
|
+
};
|
|
9830
|
+
};
|
|
9831
|
+
/** @description Unauthorized */
|
|
9832
|
+
401: {
|
|
9833
|
+
headers: {
|
|
9834
|
+
[name: string]: unknown;
|
|
9835
|
+
};
|
|
9836
|
+
content: {
|
|
9837
|
+
"application/json": {
|
|
9838
|
+
error: {
|
|
9839
|
+
type: string;
|
|
9840
|
+
code: string;
|
|
9841
|
+
message: string;
|
|
9842
|
+
meta?: {
|
|
9843
|
+
[key: string]: unknown;
|
|
9844
|
+
};
|
|
9845
|
+
trace_id?: string;
|
|
9846
|
+
span_id?: string;
|
|
9847
|
+
};
|
|
9848
|
+
};
|
|
9849
|
+
};
|
|
9850
|
+
};
|
|
9851
|
+
/** @description Forbidden */
|
|
9852
|
+
403: {
|
|
9853
|
+
headers: {
|
|
9854
|
+
[name: string]: unknown;
|
|
9855
|
+
};
|
|
9856
|
+
content: {
|
|
9857
|
+
"application/json": {
|
|
9858
|
+
error: {
|
|
9859
|
+
type: string;
|
|
9860
|
+
code: string;
|
|
9861
|
+
message: string;
|
|
9862
|
+
meta?: {
|
|
9863
|
+
[key: string]: unknown;
|
|
9864
|
+
};
|
|
9865
|
+
trace_id?: string;
|
|
9866
|
+
span_id?: string;
|
|
9867
|
+
};
|
|
9868
|
+
};
|
|
9869
|
+
};
|
|
9870
|
+
};
|
|
9871
|
+
/** @description Internal Server Error */
|
|
9872
|
+
500: {
|
|
9873
|
+
headers: {
|
|
9874
|
+
[name: string]: unknown;
|
|
9875
|
+
};
|
|
9876
|
+
content: {
|
|
9877
|
+
"application/json": {
|
|
9878
|
+
error: {
|
|
9879
|
+
type: string;
|
|
9880
|
+
code: string;
|
|
9881
|
+
message: string;
|
|
9882
|
+
meta?: {
|
|
9883
|
+
[key: string]: unknown;
|
|
9884
|
+
};
|
|
9885
|
+
trace_id?: string;
|
|
9886
|
+
span_id?: string;
|
|
9887
|
+
};
|
|
9888
|
+
};
|
|
9889
|
+
};
|
|
9890
|
+
};
|
|
9891
|
+
};
|
|
9892
|
+
};
|
|
9893
|
+
getApiCodingAgentSessionsBySessionIdEvents: {
|
|
9894
|
+
parameters: {
|
|
9895
|
+
query?: {
|
|
9896
|
+
/** @description Comma-separated event kinds to include. Known kinds: model_call, compaction, rate_limit, api_error, retries_exhausted, tool_result, tool_decision, user_prompt, subagent_completed. */
|
|
9897
|
+
kinds?: string;
|
|
9898
|
+
/** @description Opaque keyset cursor from the previous response's nextCursor. */
|
|
9899
|
+
cursor?: string;
|
|
9900
|
+
limit?: number;
|
|
9901
|
+
/** @description Epoch ms lower bound on event time; with `to`, prunes storage partitions for faster reads. */
|
|
9902
|
+
from?: number;
|
|
9903
|
+
/** @description Epoch ms upper bound on event time. */
|
|
9904
|
+
to?: number;
|
|
9905
|
+
};
|
|
9906
|
+
header?: never;
|
|
9907
|
+
path: {
|
|
9908
|
+
/** @description The agent's own session id (session.id / conversation id). */
|
|
9909
|
+
sessionId: string;
|
|
9910
|
+
};
|
|
9911
|
+
cookie?: never;
|
|
9912
|
+
};
|
|
9913
|
+
requestBody?: never;
|
|
9914
|
+
responses: {
|
|
9915
|
+
/** @description One page of session events plus the cursor for the next page */
|
|
9916
|
+
200: {
|
|
9917
|
+
headers: {
|
|
9918
|
+
[name: string]: unknown;
|
|
9919
|
+
};
|
|
9920
|
+
content: {
|
|
9921
|
+
"application/json": {
|
|
9922
|
+
events: {
|
|
9923
|
+
sessionId: string;
|
|
9924
|
+
timeUnixMs: number;
|
|
9925
|
+
recordId: string;
|
|
9926
|
+
eventKind: string;
|
|
9927
|
+
agent: string;
|
|
9928
|
+
sessionKeySource: string;
|
|
9929
|
+
traceId: string;
|
|
9930
|
+
spanId: string;
|
|
9931
|
+
promptId: string;
|
|
9932
|
+
querySource: string;
|
|
9933
|
+
agentType: string;
|
|
9934
|
+
eventSequence: number;
|
|
9935
|
+
requestId: string;
|
|
9936
|
+
model: string;
|
|
9937
|
+
inputTokens: number;
|
|
9938
|
+
outputTokens: number;
|
|
9939
|
+
cacheReadTokens: number;
|
|
9940
|
+
cacheCreationTokens: number;
|
|
9941
|
+
costUsd: number;
|
|
9942
|
+
durationMs: number;
|
|
9943
|
+
ttftMs: number;
|
|
9944
|
+
attempt: number;
|
|
9945
|
+
speed: string;
|
|
9946
|
+
stopReason: string;
|
|
9947
|
+
preTokens: number;
|
|
9948
|
+
postTokens: number;
|
|
9949
|
+
compactionTrigger: string;
|
|
9950
|
+
precomputeReuse: string;
|
|
9951
|
+
statusCode: string;
|
|
9952
|
+
errorType: string;
|
|
9953
|
+
rateLimitCarrier: string;
|
|
9954
|
+
retryDurationMs: number;
|
|
9955
|
+
toolName: string;
|
|
9956
|
+
success: string;
|
|
9957
|
+
decision: string;
|
|
9958
|
+
decisionSource: string;
|
|
9959
|
+
toolInputBytes: number;
|
|
9960
|
+
toolResultBytes: number;
|
|
9961
|
+
promptChars: number;
|
|
9962
|
+
totalTokens: number;
|
|
9963
|
+
}[];
|
|
9964
|
+
nextCursor: string | null;
|
|
9965
|
+
};
|
|
9966
|
+
};
|
|
9967
|
+
};
|
|
9968
|
+
/** @description Bad Request */
|
|
9969
|
+
400: {
|
|
9970
|
+
headers: {
|
|
9971
|
+
[name: string]: unknown;
|
|
9972
|
+
};
|
|
9973
|
+
content: {
|
|
9974
|
+
"application/json": {
|
|
9975
|
+
error: string;
|
|
9976
|
+
message?: string;
|
|
9977
|
+
};
|
|
9978
|
+
};
|
|
9979
|
+
};
|
|
9980
|
+
/** @description Unauthorized */
|
|
9981
|
+
401: {
|
|
9982
|
+
headers: {
|
|
9577
9983
|
[name: string]: unknown;
|
|
9578
9984
|
};
|
|
9579
9985
|
content: {
|
|
@@ -13031,6 +13437,7 @@ interface operations {
|
|
|
13031
13437
|
};
|
|
13032
13438
|
traceIds?: string[];
|
|
13033
13439
|
negateFilters?: boolean;
|
|
13440
|
+
excludeOrigins?: string[];
|
|
13034
13441
|
series: {
|
|
13035
13442
|
/** @enum {string} */
|
|
13036
13443
|
metric: "metadata.trace_id" | "metadata.user_id" | "metadata.thread_id" | "metadata.span_type" | "sentiment.thumbs_up_down" | "performance.completion_time" | "performance.first_token" | "performance.total_cost" | "performance.cost_billed" | "performance.cost_non_billed" | "performance.prompt_tokens" | "performance.completion_tokens" | "performance.cache_read_tokens" | "performance.cache_write_tokens" | "performance.reasoning_tokens" | "performance.total_processed_tokens" | "performance.total_tokens" | "performance.tokens_per_second" | "events.event_type" | "events.event_score" | "events.event_details" | "evaluations.evaluation_score" | "evaluations.evaluation_pass_rate" | "evaluations.evaluation_runs" | "threads.average_duration_per_thread";
|
|
@@ -14391,6 +14798,7 @@ interface operations {
|
|
|
14391
14798
|
name: string;
|
|
14392
14799
|
description?: string;
|
|
14393
14800
|
principal_user_id?: string | null;
|
|
14801
|
+
reveal_once?: boolean;
|
|
14394
14802
|
scopes?: {
|
|
14395
14803
|
/** @enum {string} */
|
|
14396
14804
|
scope_type: "organization" | "team" | "project";
|
|
@@ -14543,7 +14951,12 @@ interface operations {
|
|
|
14543
14951
|
*/
|
|
14544
14952
|
expires_at: string | null;
|
|
14545
14953
|
};
|
|
14546
|
-
secret
|
|
14954
|
+
/** @description The secret, absent when `reveal_once` was set. */
|
|
14955
|
+
secret?: string;
|
|
14956
|
+
/** @description With `reveal_once`: the id that serves the secret once, through the app. */
|
|
14957
|
+
reveal_id?: string;
|
|
14958
|
+
/** @description With `reveal_once`: the key's display prefix, safe to show in place of the secret. */
|
|
14959
|
+
preview?: string;
|
|
14547
14960
|
};
|
|
14548
14961
|
};
|
|
14549
14962
|
};
|
|
@@ -14587,7 +15000,7 @@ interface operations {
|
|
|
14587
15000
|
};
|
|
14588
15001
|
};
|
|
14589
15002
|
};
|
|
14590
|
-
/** @description Caller lacks virtualKeys:manage at a
|
|
15003
|
+
/** @description Caller lacks virtualKeys:create on its own project, or virtualKeys:manage at a scope beyond it */
|
|
14591
15004
|
403: {
|
|
14592
15005
|
headers: {
|
|
14593
15006
|
[name: string]: unknown;
|
|
@@ -19144,6 +19557,7 @@ interface operations {
|
|
|
19144
19557
|
workspace: {
|
|
19145
19558
|
root: string;
|
|
19146
19559
|
name: string;
|
|
19560
|
+
gitRepository?: boolean;
|
|
19147
19561
|
gitBranch?: string;
|
|
19148
19562
|
gitRemote?: string;
|
|
19149
19563
|
gitDirty?: boolean;
|
|
@@ -19287,10 +19701,12 @@ interface operations {
|
|
|
19287
19701
|
tool: "local_edit";
|
|
19288
19702
|
params: {
|
|
19289
19703
|
path: string;
|
|
19290
|
-
edits: {
|
|
19704
|
+
edits: ({
|
|
19291
19705
|
oldText: string;
|
|
19292
19706
|
newText: string;
|
|
19293
|
-
}
|
|
19707
|
+
} | {
|
|
19708
|
+
append: string;
|
|
19709
|
+
})[];
|
|
19294
19710
|
};
|
|
19295
19711
|
} | {
|
|
19296
19712
|
/** @constant */
|
|
@@ -19327,6 +19743,12 @@ interface operations {
|
|
|
19327
19743
|
path?: string;
|
|
19328
19744
|
limit?: number;
|
|
19329
19745
|
};
|
|
19746
|
+
} | {
|
|
19747
|
+
/** @constant */
|
|
19748
|
+
tool: "local_langwatch_env";
|
|
19749
|
+
params: {
|
|
19750
|
+
path?: string;
|
|
19751
|
+
};
|
|
19330
19752
|
});
|
|
19331
19753
|
} | {
|
|
19332
19754
|
/** @constant */
|
|
@@ -19399,6 +19821,7 @@ interface operations {
|
|
|
19399
19821
|
workspace: {
|
|
19400
19822
|
root: string;
|
|
19401
19823
|
name: string;
|
|
19824
|
+
gitRepository?: boolean;
|
|
19402
19825
|
gitBranch?: string;
|
|
19403
19826
|
gitRemote?: string;
|
|
19404
19827
|
gitDirty?: boolean;
|
|
@@ -19433,7 +19856,7 @@ interface operations {
|
|
|
19433
19856
|
};
|
|
19434
19857
|
error?: {
|
|
19435
19858
|
/** @enum {string} */
|
|
19436
|
-
code: "path_refused" | "command_refused" | "permission_denied" | "permission_expired" | "cancelled" | "timeout" | "exec_failed" | "not_found";
|
|
19859
|
+
code: "path_refused" | "command_refused" | "permission_denied" | "permission_expired" | "cancelled" | "timeout" | "exec_failed" | "not_found" | "key_refused";
|
|
19437
19860
|
message: string;
|
|
19438
19861
|
};
|
|
19439
19862
|
} | {
|
|
@@ -19546,6 +19969,7 @@ interface operations {
|
|
|
19546
19969
|
workspace: {
|
|
19547
19970
|
root: string;
|
|
19548
19971
|
name: string;
|
|
19972
|
+
gitRepository?: boolean;
|
|
19549
19973
|
gitBranch?: string;
|
|
19550
19974
|
gitRemote?: string;
|
|
19551
19975
|
gitDirty?: boolean;
|
|
@@ -25292,7 +25716,7 @@ interface operations {
|
|
|
25292
25716
|
langwatch?: {
|
|
25293
25717
|
targetReferenceId: string;
|
|
25294
25718
|
/** @enum {string} */
|
|
25295
|
-
targetType: "prompt" | "http" | "code" | "workflow" | "connected";
|
|
25719
|
+
targetType: "prompt" | "http" | "code" | "workflow" | "connected" | "voice";
|
|
25296
25720
|
targetKey?: string;
|
|
25297
25721
|
targetParameters?: {
|
|
25298
25722
|
[key: string]: string | number | boolean;
|
|
@@ -25310,6 +25734,14 @@ interface operations {
|
|
|
25310
25734
|
hostname: string;
|
|
25311
25735
|
label: string | null;
|
|
25312
25736
|
};
|
|
25737
|
+
/** @enum {string} */
|
|
25738
|
+
callerKind?: "simulated" | "human";
|
|
25739
|
+
caller?: {
|
|
25740
|
+
voice: string;
|
|
25741
|
+
interruptProbability: number;
|
|
25742
|
+
effects: string;
|
|
25743
|
+
};
|
|
25744
|
+
isCutAtLimit?: boolean;
|
|
25313
25745
|
};
|
|
25314
25746
|
} & {
|
|
25315
25747
|
[key: string]: unknown;
|
|
@@ -25325,7 +25757,7 @@ interface operations {
|
|
|
25325
25757
|
/** @default default */
|
|
25326
25758
|
scenarioSetId?: string;
|
|
25327
25759
|
/** @enum {string} */
|
|
25328
|
-
status: "SUCCESS" | "ERROR" | "CANCELLED" | "IN_PROGRESS" | "PENDING" | "FAILED" | "STALLED" | "QUEUED" | "RUNNING";
|
|
25760
|
+
status: "SUCCESS" | "ERROR" | "CANCELLED" | "IN_PROGRESS" | "PENDING" | "FAILED" | "STALLED" | "QUEUED" | "RUNNING" | "PENDING_EVALUATION";
|
|
25329
25761
|
results?: {
|
|
25330
25762
|
/** @enum {string} */
|
|
25331
25763
|
verdict: "success" | "failure" | "inconclusive";
|
|
@@ -27253,18 +27685,6 @@ interface operations {
|
|
|
27253
27685
|
};
|
|
27254
27686
|
requestBody?: never;
|
|
27255
27687
|
responses: {
|
|
27256
|
-
/** @description The project's API key */
|
|
27257
|
-
200: {
|
|
27258
|
-
headers: {
|
|
27259
|
-
[name: string]: unknown;
|
|
27260
|
-
};
|
|
27261
|
-
content: {
|
|
27262
|
-
"application/json": {
|
|
27263
|
-
/** @description Send as X-Auth-Token, Bearer, or Basic */
|
|
27264
|
-
apiKey: string;
|
|
27265
|
-
};
|
|
27266
|
-
};
|
|
27267
|
-
};
|
|
27268
27688
|
/** @description Invalid or missing API key token */
|
|
27269
27689
|
401: {
|
|
27270
27690
|
headers: {
|
|
@@ -27272,20 +27692,13 @@ interface operations {
|
|
|
27272
27692
|
};
|
|
27273
27693
|
content?: never;
|
|
27274
27694
|
};
|
|
27275
|
-
/** @description
|
|
27695
|
+
/** @description A signed-in project administrator is required; API-key principals cannot reveal base keys */
|
|
27276
27696
|
403: {
|
|
27277
27697
|
headers: {
|
|
27278
27698
|
[name: string]: unknown;
|
|
27279
27699
|
};
|
|
27280
27700
|
content?: never;
|
|
27281
27701
|
};
|
|
27282
|
-
/** @description No project with that id in this organization */
|
|
27283
|
-
404: {
|
|
27284
|
-
headers: {
|
|
27285
|
-
[name: string]: unknown;
|
|
27286
|
-
};
|
|
27287
|
-
content?: never;
|
|
27288
|
-
};
|
|
27289
27702
|
};
|
|
27290
27703
|
};
|
|
27291
27704
|
regenerateProjectApiKey: {
|
|
@@ -27300,18 +27713,6 @@ interface operations {
|
|
|
27300
27713
|
};
|
|
27301
27714
|
requestBody?: never;
|
|
27302
27715
|
responses: {
|
|
27303
|
-
/** @description The new API key. The previous one no longer authenticates. */
|
|
27304
|
-
200: {
|
|
27305
|
-
headers: {
|
|
27306
|
-
[name: string]: unknown;
|
|
27307
|
-
};
|
|
27308
|
-
content: {
|
|
27309
|
-
"application/json": {
|
|
27310
|
-
/** @description Send as X-Auth-Token, Bearer, or Basic */
|
|
27311
|
-
apiKey: string;
|
|
27312
|
-
};
|
|
27313
|
-
};
|
|
27314
|
-
};
|
|
27315
27716
|
/** @description Invalid or missing API key token */
|
|
27316
27717
|
401: {
|
|
27317
27718
|
headers: {
|
|
@@ -27319,20 +27720,13 @@ interface operations {
|
|
|
27319
27720
|
};
|
|
27320
27721
|
content?: never;
|
|
27321
27722
|
};
|
|
27322
|
-
/** @description
|
|
27723
|
+
/** @description A signed-in project administrator is required; API-key principals cannot rotate base keys */
|
|
27323
27724
|
403: {
|
|
27324
27725
|
headers: {
|
|
27325
27726
|
[name: string]: unknown;
|
|
27326
27727
|
};
|
|
27327
27728
|
content?: never;
|
|
27328
27729
|
};
|
|
27329
|
-
/** @description No project with that id in this organization */
|
|
27330
|
-
404: {
|
|
27331
|
-
headers: {
|
|
27332
|
-
[name: string]: unknown;
|
|
27333
|
-
};
|
|
27334
|
-
content?: never;
|
|
27335
|
-
};
|
|
27336
27730
|
};
|
|
27337
27731
|
};
|
|
27338
27732
|
getApiSecrets: {
|
|
@@ -27798,6 +28192,7 @@ interface operations {
|
|
|
27798
28192
|
scenarioRunId: string;
|
|
27799
28193
|
name: string | null;
|
|
27800
28194
|
description: string | null;
|
|
28195
|
+
/** @description Where the run stands. PENDING_EVALUATION means the conversation is over and the judge has decided, but the evaluators the run's suite and plan attach have not been recorded yet, so a required one may still fail the run. Wait for another status before reading the verdict as final. */
|
|
27801
28196
|
status: string;
|
|
27802
28197
|
results: {
|
|
27803
28198
|
verdict?: string | null;
|
|
@@ -27920,6 +28315,7 @@ interface operations {
|
|
|
27920
28315
|
scenarioRunId: string;
|
|
27921
28316
|
name: string | null;
|
|
27922
28317
|
description: string | null;
|
|
28318
|
+
/** @description Where the run stands. PENDING_EVALUATION means the conversation is over and the judge has decided, but the evaluators the run's suite and plan attach have not been recorded yet, so a required one may still fail the run. Wait for another status before reading the verdict as final. */
|
|
27923
28319
|
status: string;
|
|
27924
28320
|
results: {
|
|
27925
28321
|
verdict?: string | null;
|
|
@@ -28276,7 +28672,7 @@ interface operations {
|
|
|
28276
28672
|
* @description What kind of thing the scenarios run against. A connected agent is one registered from code with the SDK.
|
|
28277
28673
|
* @enum {string}
|
|
28278
28674
|
*/
|
|
28279
|
-
type: "prompt" | "http" | "code" | "workflow" | "connected";
|
|
28675
|
+
type: "prompt" | "http" | "code" | "workflow" | "connected" | "voice";
|
|
28280
28676
|
/** @description The id of the prompt, agent or workflow to run against. A connected target may also say <name>@<environment>, for example support-agent@production, which resolves to the agent id. */
|
|
28281
28677
|
referenceId: string;
|
|
28282
28678
|
/** @description Parameter values this target alone runs with, by name. They are merged over the run-level parameters and the target wins, so two targets may name the same agent with different values: that is how one run compares one agent on two models, and the results show one column for each target. */
|
|
@@ -28385,7 +28781,7 @@ interface operations {
|
|
|
28385
28781
|
* @description What kind of thing the scenarios run against. A connected agent is one registered from code with the SDK.
|
|
28386
28782
|
* @enum {string}
|
|
28387
28783
|
*/
|
|
28388
|
-
type: "prompt" | "http" | "code" | "workflow" | "connected";
|
|
28784
|
+
type: "prompt" | "http" | "code" | "workflow" | "connected" | "voice";
|
|
28389
28785
|
/** @description The id of the prompt, agent or workflow to run against. A connected target may also say <name>@<environment>, for example support-agent@production, which resolves to the agent id. */
|
|
28390
28786
|
referenceId: string;
|
|
28391
28787
|
/** @description Parameter values this target alone runs with, by name. They are merged over the run-level parameters and the target wins, so two targets may name the same agent with different values: that is how one run compares one agent on two models, and the results show one column for each target. */
|
|
@@ -28439,7 +28835,7 @@ interface operations {
|
|
|
28439
28835
|
* @description What kind of thing the scenarios run against. A connected agent is one registered from code with the SDK.
|
|
28440
28836
|
* @enum {string}
|
|
28441
28837
|
*/
|
|
28442
|
-
type: "prompt" | "http" | "code" | "workflow" | "connected";
|
|
28838
|
+
type: "prompt" | "http" | "code" | "workflow" | "connected" | "voice";
|
|
28443
28839
|
/** @description The id of the prompt, agent or workflow to run against. A connected target may also say <name>@<environment>, for example support-agent@production, which resolves to the agent id. */
|
|
28444
28840
|
referenceId: string;
|
|
28445
28841
|
/** @description Parameter values this target alone runs with, by name. They are merged over the run-level parameters and the target wins, so two targets may name the same agent with different values: that is how one run compares one agent on two models, and the results show one column for each target. */
|
|
@@ -28555,7 +28951,7 @@ interface operations {
|
|
|
28555
28951
|
* @description What kind of thing the scenarios run against. A connected agent is one registered from code with the SDK.
|
|
28556
28952
|
* @enum {string}
|
|
28557
28953
|
*/
|
|
28558
|
-
type: "prompt" | "http" | "code" | "workflow" | "connected";
|
|
28954
|
+
type: "prompt" | "http" | "code" | "workflow" | "connected" | "voice";
|
|
28559
28955
|
/** @description The id of the prompt, agent or workflow to run against. A connected target may also say <name>@<environment>, for example support-agent@production, which resolves to the agent id. */
|
|
28560
28956
|
referenceId: string;
|
|
28561
28957
|
/** @description Parameter values this target alone runs with, by name. They are merged over the run-level parameters and the target wins, so two targets may name the same agent with different values: that is how one run compares one agent on two models, and the results show one column for each target. */
|
|
@@ -28759,7 +29155,7 @@ interface operations {
|
|
|
28759
29155
|
* @description What kind of thing the scenarios run against. A connected agent is one registered from code with the SDK.
|
|
28760
29156
|
* @enum {string}
|
|
28761
29157
|
*/
|
|
28762
|
-
type: "prompt" | "http" | "code" | "workflow" | "connected";
|
|
29158
|
+
type: "prompt" | "http" | "code" | "workflow" | "connected" | "voice";
|
|
28763
29159
|
/** @description The id of the prompt, agent or workflow to run against. A connected target may also say <name>@<environment>, for example support-agent@production, which resolves to the agent id. */
|
|
28764
29160
|
referenceId: string;
|
|
28765
29161
|
/** @description Parameter values this target alone runs with, by name. They are merged over the run-level parameters and the target wins, so two targets may name the same agent with different values: that is how one run compares one agent on two models, and the results show one column for each target. */
|
|
@@ -28811,7 +29207,7 @@ interface operations {
|
|
|
28811
29207
|
* @description What kind of thing the scenarios run against. A connected agent is one registered from code with the SDK.
|
|
28812
29208
|
* @enum {string}
|
|
28813
29209
|
*/
|
|
28814
|
-
type: "prompt" | "http" | "code" | "workflow" | "connected";
|
|
29210
|
+
type: "prompt" | "http" | "code" | "workflow" | "connected" | "voice";
|
|
28815
29211
|
/** @description The id of the prompt, agent or workflow to run against. A connected target may also say <name>@<environment>, for example support-agent@production, which resolves to the agent id. */
|
|
28816
29212
|
referenceId: string;
|
|
28817
29213
|
/** @description Parameter values this target alone runs with, by name. They are merged over the run-level parameters and the target wins, so two targets may name the same agent with different values: that is how one run compares one agent on two models, and the results show one column for each target. */
|
|
@@ -28941,7 +29337,7 @@ interface operations {
|
|
|
28941
29337
|
* @description What kind of thing the scenarios run against. A connected agent is one registered from code with the SDK.
|
|
28942
29338
|
* @enum {string}
|
|
28943
29339
|
*/
|
|
28944
|
-
type: "prompt" | "http" | "code" | "workflow" | "connected";
|
|
29340
|
+
type: "prompt" | "http" | "code" | "workflow" | "connected" | "voice";
|
|
28945
29341
|
/** @description The id of the prompt, agent or workflow to run against. A connected target may also say <name>@<environment>, for example support-agent@production, which resolves to the agent id. */
|
|
28946
29342
|
referenceId: string;
|
|
28947
29343
|
/** @description Parameter values this target alone runs with, by name. They are merged over the run-level parameters and the target wins, so two targets may name the same agent with different values: that is how one run compares one agent on two models, and the results show one column for each target. */
|
|
@@ -29043,7 +29439,7 @@ interface operations {
|
|
|
29043
29439
|
* @description What kind of thing the scenarios run against. A connected agent is one registered from code with the SDK.
|
|
29044
29440
|
* @enum {string}
|
|
29045
29441
|
*/
|
|
29046
|
-
type: "prompt" | "http" | "code" | "workflow" | "connected";
|
|
29442
|
+
type: "prompt" | "http" | "code" | "workflow" | "connected" | "voice";
|
|
29047
29443
|
/** @description The id of the prompt, agent or workflow to run against. A connected target may also say <name>@<environment>, for example support-agent@production, which resolves to the agent id. */
|
|
29048
29444
|
referenceId: string;
|
|
29049
29445
|
/** @description Parameter values this target alone runs with, by name. They are merged over the run-level parameters and the target wins, so two targets may name the same agent with different values: that is how one run compares one agent on two models, and the results show one column for each target. */
|
|
@@ -29090,7 +29486,7 @@ interface operations {
|
|
|
29090
29486
|
* @description What kind of thing the scenarios run against. A connected agent is one registered from code with the SDK.
|
|
29091
29487
|
* @enum {string}
|
|
29092
29488
|
*/
|
|
29093
|
-
type: "prompt" | "http" | "code" | "workflow" | "connected";
|
|
29489
|
+
type: "prompt" | "http" | "code" | "workflow" | "connected" | "voice";
|
|
29094
29490
|
/** @description The id of the prompt, agent or workflow to run against. A connected target may also say <name>@<environment>, for example support-agent@production, which resolves to the agent id. */
|
|
29095
29491
|
referenceId: string;
|
|
29096
29492
|
/** @description Parameter values this target alone runs with, by name. They are merged over the run-level parameters and the target wins, so two targets may name the same agent with different values: that is how one run compares one agent on two models, and the results show one column for each target. */
|
|
@@ -29167,11 +29563,15 @@ interface operations {
|
|
|
29167
29563
|
};
|
|
29168
29564
|
};
|
|
29169
29565
|
};
|
|
29170
|
-
|
|
29566
|
+
listInstantEvalRuns: {
|
|
29171
29567
|
parameters: {
|
|
29172
29568
|
query?: {
|
|
29173
|
-
/** @description
|
|
29174
|
-
|
|
29569
|
+
/** @description Runs to list, at most one hundred. */
|
|
29570
|
+
limit?: number;
|
|
29571
|
+
/** @description List runs accepted strictly before this instant, as an ISO 8601 timestamp. Half of the list's cursor: pass `beforeId` with it. */
|
|
29572
|
+
before?: string;
|
|
29573
|
+
/** @description The id of the last run of the previous page. Two runs can share an instant, so this is what keeps a page from skipping the others written in the same millisecond. */
|
|
29574
|
+
beforeId?: string;
|
|
29175
29575
|
};
|
|
29176
29576
|
header?: never;
|
|
29177
29577
|
path?: never;
|
|
@@ -29186,44 +29586,741 @@ interface operations {
|
|
|
29186
29586
|
};
|
|
29187
29587
|
content: {
|
|
29188
29588
|
"application/json": {
|
|
29189
|
-
/** @description The
|
|
29190
|
-
|
|
29191
|
-
|
|
29192
|
-
|
|
29193
|
-
|
|
29194
|
-
|
|
29195
|
-
|
|
29196
|
-
|
|
29197
|
-
/** @
|
|
29198
|
-
|
|
29199
|
-
|
|
29200
|
-
|
|
29201
|
-
|
|
29202
|
-
|
|
29203
|
-
|
|
29204
|
-
|
|
29205
|
-
|
|
29206
|
-
|
|
29207
|
-
|
|
29208
|
-
|
|
29209
|
-
|
|
29210
|
-
|
|
29211
|
-
|
|
29212
|
-
|
|
29213
|
-
|
|
29214
|
-
|
|
29589
|
+
/** @description The project's runs, newest first. */
|
|
29590
|
+
runs: {
|
|
29591
|
+
/** @description The run id. */
|
|
29592
|
+
id: string;
|
|
29593
|
+
/** @description What the run was called, if anything. */
|
|
29594
|
+
name: string | null;
|
|
29595
|
+
/** @description The statement, exactly as submitted. */
|
|
29596
|
+
sql: string;
|
|
29597
|
+
/** @description The values the statement's parameters were filled with. */
|
|
29598
|
+
parameters: {
|
|
29599
|
+
[key: string]: string | number | boolean | null;
|
|
29600
|
+
};
|
|
29601
|
+
/** @description One entry per eval function the statement projects, derived from it when the run was accepted. */
|
|
29602
|
+
questions: {
|
|
29603
|
+
/** @description The statement's own output column, which is the name this question is addressed by everywhere else. */
|
|
29604
|
+
id: string;
|
|
29605
|
+
/** @description The eval function that asked it. */
|
|
29606
|
+
function: string;
|
|
29607
|
+
/**
|
|
29608
|
+
* @description What kind of answer the question takes.
|
|
29609
|
+
* @enum {string}
|
|
29610
|
+
*/
|
|
29611
|
+
kind: "boolean" | "score" | "category";
|
|
29612
|
+
/** @description Which part of the verdict the statement's column carries. */
|
|
29613
|
+
reads: string;
|
|
29614
|
+
/** @description Where a boolean question's probability becomes a pass. Null for a question that is not a boolean. */
|
|
29615
|
+
threshold: number | null;
|
|
29616
|
+
}[];
|
|
29617
|
+
/** @description Rows this run may judge. */
|
|
29618
|
+
limit: number;
|
|
29215
29619
|
/**
|
|
29216
|
-
* @description
|
|
29620
|
+
* @description Where the run is in its life.
|
|
29217
29621
|
* @enum {string}
|
|
29218
29622
|
*/
|
|
29219
|
-
|
|
29220
|
-
/** @description
|
|
29221
|
-
|
|
29222
|
-
/** @description
|
|
29223
|
-
|
|
29224
|
-
|
|
29225
|
-
|
|
29226
|
-
|
|
29623
|
+
status: "queued" | "planning" | "running" | "finished" | "failed" | "cancelled";
|
|
29624
|
+
/** @description Rows the run found, bounded by its limit. Null until it has looked. */
|
|
29625
|
+
total: number | null;
|
|
29626
|
+
/** @description Rows judged so far. */
|
|
29627
|
+
progress: number;
|
|
29628
|
+
/** @description Judgements that matched, across this run's boolean questions. Null when the run asked none: a score or a category question has no match to count. */
|
|
29629
|
+
matched: number | null;
|
|
29630
|
+
/** @description Per question: matches for a boolean question, judged rows for a score or a category one. */
|
|
29631
|
+
matchedByQuestion: {
|
|
29632
|
+
[key: string]: number;
|
|
29633
|
+
};
|
|
29634
|
+
/** @description Rows the judge could not answer. */
|
|
29635
|
+
failed: number;
|
|
29636
|
+
/** @description Rows the judge declined to answer. */
|
|
29637
|
+
skipped: number;
|
|
29638
|
+
/** @description Input tokens the judge billed for. */
|
|
29639
|
+
tokens: number;
|
|
29640
|
+
/** @description What the judging costs you, in United States dollars. */
|
|
29641
|
+
priceUsd: number;
|
|
29642
|
+
/** @description The code of the failure that ended the run, when one did. */
|
|
29643
|
+
error: string | null;
|
|
29644
|
+
/** @description When the run was accepted. */
|
|
29645
|
+
createdAt: string;
|
|
29646
|
+
/** @description When the run was last written to. */
|
|
29647
|
+
updatedAt: string;
|
|
29648
|
+
/** @description When the run began reading rows. */
|
|
29649
|
+
startedAt: string | null;
|
|
29650
|
+
/** @description When the run ended. */
|
|
29651
|
+
finishedAt: string | null;
|
|
29652
|
+
}[];
|
|
29653
|
+
};
|
|
29654
|
+
};
|
|
29655
|
+
};
|
|
29656
|
+
};
|
|
29657
|
+
};
|
|
29658
|
+
createInstantEvalRun: {
|
|
29659
|
+
parameters: {
|
|
29660
|
+
query?: never;
|
|
29661
|
+
header?: never;
|
|
29662
|
+
path?: never;
|
|
29663
|
+
cookie?: never;
|
|
29664
|
+
};
|
|
29665
|
+
requestBody: {
|
|
29666
|
+
content: {
|
|
29667
|
+
"application/json": {
|
|
29668
|
+
/** @description The LangWatchQL statement to judge. It must project TraceId and at least one eval function column. Send this or target, never both. */
|
|
29669
|
+
sql?: string;
|
|
29670
|
+
/** @description Values for the parameters the statement declares. */
|
|
29671
|
+
parameters?: {
|
|
29672
|
+
[key: string]: string | number | boolean | null;
|
|
29673
|
+
};
|
|
29674
|
+
/**
|
|
29675
|
+
* @description What one judged row is, in place of a statement: a trace, a conversation, or one model call. The statement is written for you from this and the questions, and handed back on the run so you can edit it and resubmit.
|
|
29676
|
+
* @enum {string}
|
|
29677
|
+
*/
|
|
29678
|
+
target?: "traces" | "threads" | "llm_spans";
|
|
29679
|
+
/** @description With target: a trace filter, in the language the trace explorer's search bar speaks, narrowing which rows are judged. */
|
|
29680
|
+
filter?: string;
|
|
29681
|
+
/**
|
|
29682
|
+
* Format: date-time
|
|
29683
|
+
* @description With target: the oldest instant to judge, as an ISO 8601 timestamp. Defaults to seven days ago.
|
|
29684
|
+
*/
|
|
29685
|
+
start?: string;
|
|
29686
|
+
/**
|
|
29687
|
+
* Format: date-time
|
|
29688
|
+
* @description With target: the newest instant to judge. Defaults to now.
|
|
29689
|
+
*/
|
|
29690
|
+
end?: string;
|
|
29691
|
+
/** @description With target: what to ask of each row. One classification asks them all per row, and the question text is part of what that classification is priced on; the estimate endpoint prices the exact set. */
|
|
29692
|
+
questions?: {
|
|
29693
|
+
/** @description What to call this question. It becomes the statement's output column and the name every judgement is filed under. Defaults to q1, q2 and so on. */
|
|
29694
|
+
id?: string;
|
|
29695
|
+
/**
|
|
29696
|
+
* @description What kind of answer you want: a yes or no, a rating on a scale, or one of a list of options.
|
|
29697
|
+
* @default boolean
|
|
29698
|
+
* @enum {string}
|
|
29699
|
+
*/
|
|
29700
|
+
kind?: "boolean" | "score" | "category";
|
|
29701
|
+
/** @description The question, in your own words, as you would write it for a human reader. */
|
|
29702
|
+
instructions: string;
|
|
29703
|
+
/** @description For a yes or no question: what counts as yes, then what counts as no. Cannot be combined with a threshold. */
|
|
29704
|
+
criteria?: string[];
|
|
29705
|
+
/** @description For a yes or no question: the probability at or above which the answer counts as yes. Without one the column carries the probability itself and a run draws the line at an even chance. */
|
|
29706
|
+
threshold?: number;
|
|
29707
|
+
/** @description For a rating: the two ends of the scale. */
|
|
29708
|
+
range?: {
|
|
29709
|
+
/** @description The lowest level of the scale. */
|
|
29710
|
+
min: number;
|
|
29711
|
+
/** @description The highest level of the scale. */
|
|
29712
|
+
max: number;
|
|
29713
|
+
};
|
|
29714
|
+
/** @description For a choice: the options to pick between. */
|
|
29715
|
+
options?: {
|
|
29716
|
+
/** @description What the column holds when this option is the answer. */
|
|
29717
|
+
name: string;
|
|
29718
|
+
/** @description What this option means, in your own words. */
|
|
29719
|
+
description: string;
|
|
29720
|
+
}[];
|
|
29721
|
+
}[];
|
|
29722
|
+
/** @description What to call the run. Yours to choose. */
|
|
29723
|
+
name?: string;
|
|
29724
|
+
/** @description Rows the run may judge. Ten thousand by default on every plan, up to one hundred thousand on a plan that lifts the cap. */
|
|
29725
|
+
limit?: number;
|
|
29726
|
+
};
|
|
29727
|
+
};
|
|
29728
|
+
};
|
|
29729
|
+
responses: {
|
|
29730
|
+
/** @description Success */
|
|
29731
|
+
202: {
|
|
29732
|
+
headers: {
|
|
29733
|
+
[name: string]: unknown;
|
|
29734
|
+
};
|
|
29735
|
+
content: {
|
|
29736
|
+
"application/json": {
|
|
29737
|
+
/** @description The run id. */
|
|
29738
|
+
id: string;
|
|
29739
|
+
/** @description What the run was called, if anything. */
|
|
29740
|
+
name: string | null;
|
|
29741
|
+
/** @description The statement, exactly as submitted. */
|
|
29742
|
+
sql: string;
|
|
29743
|
+
/** @description The values the statement's parameters were filled with. */
|
|
29744
|
+
parameters: {
|
|
29745
|
+
[key: string]: string | number | boolean | null;
|
|
29746
|
+
};
|
|
29747
|
+
/** @description One entry per eval function the statement projects, derived from it when the run was accepted. */
|
|
29748
|
+
questions: {
|
|
29749
|
+
/** @description The statement's own output column, which is the name this question is addressed by everywhere else. */
|
|
29750
|
+
id: string;
|
|
29751
|
+
/** @description The eval function that asked it. */
|
|
29752
|
+
function: string;
|
|
29753
|
+
/**
|
|
29754
|
+
* @description What kind of answer the question takes.
|
|
29755
|
+
* @enum {string}
|
|
29756
|
+
*/
|
|
29757
|
+
kind: "boolean" | "score" | "category";
|
|
29758
|
+
/** @description Which part of the verdict the statement's column carries. */
|
|
29759
|
+
reads: string;
|
|
29760
|
+
/** @description Where a boolean question's probability becomes a pass. Null for a question that is not a boolean. */
|
|
29761
|
+
threshold: number | null;
|
|
29762
|
+
}[];
|
|
29763
|
+
/** @description Rows this run may judge. */
|
|
29764
|
+
limit: number;
|
|
29765
|
+
/**
|
|
29766
|
+
* @description Where the run is in its life.
|
|
29767
|
+
* @enum {string}
|
|
29768
|
+
*/
|
|
29769
|
+
status: "queued" | "planning" | "running" | "finished" | "failed" | "cancelled";
|
|
29770
|
+
/** @description Rows the run found, bounded by its limit. Null until it has looked. */
|
|
29771
|
+
total: number | null;
|
|
29772
|
+
/** @description Rows judged so far. */
|
|
29773
|
+
progress: number;
|
|
29774
|
+
/** @description Judgements that matched, across this run's boolean questions. Null when the run asked none: a score or a category question has no match to count. */
|
|
29775
|
+
matched: number | null;
|
|
29776
|
+
/** @description Per question: matches for a boolean question, judged rows for a score or a category one. */
|
|
29777
|
+
matchedByQuestion: {
|
|
29778
|
+
[key: string]: number;
|
|
29779
|
+
};
|
|
29780
|
+
/** @description Rows the judge could not answer. */
|
|
29781
|
+
failed: number;
|
|
29782
|
+
/** @description Rows the judge declined to answer. */
|
|
29783
|
+
skipped: number;
|
|
29784
|
+
/** @description Input tokens the judge billed for. */
|
|
29785
|
+
tokens: number;
|
|
29786
|
+
/** @description What the judging costs you, in United States dollars. */
|
|
29787
|
+
priceUsd: number;
|
|
29788
|
+
/** @description The code of the failure that ended the run, when one did. */
|
|
29789
|
+
error: string | null;
|
|
29790
|
+
/** @description When the run was accepted. */
|
|
29791
|
+
createdAt: string;
|
|
29792
|
+
/** @description When the run was last written to. */
|
|
29793
|
+
updatedAt: string;
|
|
29794
|
+
/** @description When the run began reading rows. */
|
|
29795
|
+
startedAt: string | null;
|
|
29796
|
+
/** @description When the run ended. */
|
|
29797
|
+
finishedAt: string | null;
|
|
29798
|
+
};
|
|
29799
|
+
};
|
|
29800
|
+
};
|
|
29801
|
+
};
|
|
29802
|
+
};
|
|
29803
|
+
estimateInstantEvalRun: {
|
|
29804
|
+
parameters: {
|
|
29805
|
+
query?: never;
|
|
29806
|
+
header?: never;
|
|
29807
|
+
path?: never;
|
|
29808
|
+
cookie?: never;
|
|
29809
|
+
};
|
|
29810
|
+
requestBody: {
|
|
29811
|
+
content: {
|
|
29812
|
+
"application/json": {
|
|
29813
|
+
/** @description The LangWatchQL statement to judge. It must project TraceId and at least one eval function column. Send this or target, never both. */
|
|
29814
|
+
sql?: string;
|
|
29815
|
+
/** @description Values for the parameters the statement declares. */
|
|
29816
|
+
parameters?: {
|
|
29817
|
+
[key: string]: string | number | boolean | null;
|
|
29818
|
+
};
|
|
29819
|
+
/**
|
|
29820
|
+
* @description What one judged row is, in place of a statement: a trace, a conversation, or one model call. The statement is written for you from this and the questions, and handed back on the run so you can edit it and resubmit.
|
|
29821
|
+
* @enum {string}
|
|
29822
|
+
*/
|
|
29823
|
+
target?: "traces" | "threads" | "llm_spans";
|
|
29824
|
+
/** @description With target: a trace filter, in the language the trace explorer's search bar speaks, narrowing which rows are judged. */
|
|
29825
|
+
filter?: string;
|
|
29826
|
+
/**
|
|
29827
|
+
* Format: date-time
|
|
29828
|
+
* @description With target: the oldest instant to judge, as an ISO 8601 timestamp. Defaults to seven days ago.
|
|
29829
|
+
*/
|
|
29830
|
+
start?: string;
|
|
29831
|
+
/**
|
|
29832
|
+
* Format: date-time
|
|
29833
|
+
* @description With target: the newest instant to judge. Defaults to now.
|
|
29834
|
+
*/
|
|
29835
|
+
end?: string;
|
|
29836
|
+
/** @description With target: what to ask of each row. One classification asks them all per row, and the question text is part of what that classification is priced on; the estimate endpoint prices the exact set. */
|
|
29837
|
+
questions?: {
|
|
29838
|
+
/** @description What to call this question. It becomes the statement's output column and the name every judgement is filed under. Defaults to q1, q2 and so on. */
|
|
29839
|
+
id?: string;
|
|
29840
|
+
/**
|
|
29841
|
+
* @description What kind of answer you want: a yes or no, a rating on a scale, or one of a list of options.
|
|
29842
|
+
* @default boolean
|
|
29843
|
+
* @enum {string}
|
|
29844
|
+
*/
|
|
29845
|
+
kind?: "boolean" | "score" | "category";
|
|
29846
|
+
/** @description The question, in your own words, as you would write it for a human reader. */
|
|
29847
|
+
instructions: string;
|
|
29848
|
+
/** @description For a yes or no question: what counts as yes, then what counts as no. Cannot be combined with a threshold. */
|
|
29849
|
+
criteria?: string[];
|
|
29850
|
+
/** @description For a yes or no question: the probability at or above which the answer counts as yes. Without one the column carries the probability itself and a run draws the line at an even chance. */
|
|
29851
|
+
threshold?: number;
|
|
29852
|
+
/** @description For a rating: the two ends of the scale. */
|
|
29853
|
+
range?: {
|
|
29854
|
+
/** @description The lowest level of the scale. */
|
|
29855
|
+
min: number;
|
|
29856
|
+
/** @description The highest level of the scale. */
|
|
29857
|
+
max: number;
|
|
29858
|
+
};
|
|
29859
|
+
/** @description For a choice: the options to pick between. */
|
|
29860
|
+
options?: {
|
|
29861
|
+
/** @description What the column holds when this option is the answer. */
|
|
29862
|
+
name: string;
|
|
29863
|
+
/** @description What this option means, in your own words. */
|
|
29864
|
+
description: string;
|
|
29865
|
+
}[];
|
|
29866
|
+
}[];
|
|
29867
|
+
/** @description What to call the run. Yours to choose. */
|
|
29868
|
+
name?: string;
|
|
29869
|
+
/** @description Rows the run may judge. Ten thousand by default on every plan, up to one hundred thousand on a plan that lifts the cap. */
|
|
29870
|
+
limit?: number;
|
|
29871
|
+
};
|
|
29872
|
+
};
|
|
29873
|
+
};
|
|
29874
|
+
responses: {
|
|
29875
|
+
/** @description Success */
|
|
29876
|
+
200: {
|
|
29877
|
+
headers: {
|
|
29878
|
+
[name: string]: unknown;
|
|
29879
|
+
};
|
|
29880
|
+
content: {
|
|
29881
|
+
"application/json": {
|
|
29882
|
+
/** @description Rows the statement matches, bounded by the run's limit. */
|
|
29883
|
+
rows: number;
|
|
29884
|
+
/** @description Whether the statement matches more rows than the run may judge. */
|
|
29885
|
+
isRowsCapped: boolean;
|
|
29886
|
+
/** @description Input tokens one judged row sends, measured from a sample. */
|
|
29887
|
+
avgTokens: number;
|
|
29888
|
+
/** @description Input tokens the whole run would send. */
|
|
29889
|
+
totalTokens: number;
|
|
29890
|
+
/** @description Classifications the run would make, one per judged row. */
|
|
29891
|
+
requests: number;
|
|
29892
|
+
/** @description What the run would cost you, in United States dollars. */
|
|
29893
|
+
priceUsd: number;
|
|
29894
|
+
/** @description What is left of the free Instant Evals budget, in United States dollars. Only present for an organization without a paid plan. */
|
|
29895
|
+
freeBudgetRemainingUsd?: number;
|
|
29896
|
+
};
|
|
29897
|
+
};
|
|
29898
|
+
};
|
|
29899
|
+
};
|
|
29900
|
+
};
|
|
29901
|
+
getInstantEvalRun: {
|
|
29902
|
+
parameters: {
|
|
29903
|
+
query?: never;
|
|
29904
|
+
header?: never;
|
|
29905
|
+
path: {
|
|
29906
|
+
/** @description The run id. */
|
|
29907
|
+
id: string;
|
|
29908
|
+
};
|
|
29909
|
+
cookie?: never;
|
|
29910
|
+
};
|
|
29911
|
+
requestBody?: never;
|
|
29912
|
+
responses: {
|
|
29913
|
+
/** @description Success */
|
|
29914
|
+
200: {
|
|
29915
|
+
headers: {
|
|
29916
|
+
[name: string]: unknown;
|
|
29917
|
+
};
|
|
29918
|
+
content: {
|
|
29919
|
+
"application/json": {
|
|
29920
|
+
/** @description The run id. */
|
|
29921
|
+
id: string;
|
|
29922
|
+
/** @description What the run was called, if anything. */
|
|
29923
|
+
name: string | null;
|
|
29924
|
+
/** @description The statement, exactly as submitted. */
|
|
29925
|
+
sql: string;
|
|
29926
|
+
/** @description The values the statement's parameters were filled with. */
|
|
29927
|
+
parameters: {
|
|
29928
|
+
[key: string]: string | number | boolean | null;
|
|
29929
|
+
};
|
|
29930
|
+
/** @description One entry per eval function the statement projects, derived from it when the run was accepted. */
|
|
29931
|
+
questions: {
|
|
29932
|
+
/** @description The statement's own output column, which is the name this question is addressed by everywhere else. */
|
|
29933
|
+
id: string;
|
|
29934
|
+
/** @description The eval function that asked it. */
|
|
29935
|
+
function: string;
|
|
29936
|
+
/**
|
|
29937
|
+
* @description What kind of answer the question takes.
|
|
29938
|
+
* @enum {string}
|
|
29939
|
+
*/
|
|
29940
|
+
kind: "boolean" | "score" | "category";
|
|
29941
|
+
/** @description Which part of the verdict the statement's column carries. */
|
|
29942
|
+
reads: string;
|
|
29943
|
+
/** @description Where a boolean question's probability becomes a pass. Null for a question that is not a boolean. */
|
|
29944
|
+
threshold: number | null;
|
|
29945
|
+
}[];
|
|
29946
|
+
/** @description Rows this run may judge. */
|
|
29947
|
+
limit: number;
|
|
29948
|
+
/**
|
|
29949
|
+
* @description Where the run is in its life.
|
|
29950
|
+
* @enum {string}
|
|
29951
|
+
*/
|
|
29952
|
+
status: "queued" | "planning" | "running" | "finished" | "failed" | "cancelled";
|
|
29953
|
+
/** @description Rows the run found, bounded by its limit. Null until it has looked. */
|
|
29954
|
+
total: number | null;
|
|
29955
|
+
/** @description Rows judged so far. */
|
|
29956
|
+
progress: number;
|
|
29957
|
+
/** @description Judgements that matched, across this run's boolean questions. Null when the run asked none: a score or a category question has no match to count. */
|
|
29958
|
+
matched: number | null;
|
|
29959
|
+
/** @description Per question: matches for a boolean question, judged rows for a score or a category one. */
|
|
29960
|
+
matchedByQuestion: {
|
|
29961
|
+
[key: string]: number;
|
|
29962
|
+
};
|
|
29963
|
+
/** @description Rows the judge could not answer. */
|
|
29964
|
+
failed: number;
|
|
29965
|
+
/** @description Rows the judge declined to answer. */
|
|
29966
|
+
skipped: number;
|
|
29967
|
+
/** @description Input tokens the judge billed for. */
|
|
29968
|
+
tokens: number;
|
|
29969
|
+
/** @description What the judging costs you, in United States dollars. */
|
|
29970
|
+
priceUsd: number;
|
|
29971
|
+
/** @description The code of the failure that ended the run, when one did. */
|
|
29972
|
+
error: string | null;
|
|
29973
|
+
/** @description When the run was accepted. */
|
|
29974
|
+
createdAt: string;
|
|
29975
|
+
/** @description When the run was last written to. */
|
|
29976
|
+
updatedAt: string;
|
|
29977
|
+
/** @description When the run began reading rows. */
|
|
29978
|
+
startedAt: string | null;
|
|
29979
|
+
/** @description When the run ended. */
|
|
29980
|
+
finishedAt: string | null;
|
|
29981
|
+
};
|
|
29982
|
+
};
|
|
29983
|
+
};
|
|
29984
|
+
/** @description Not Found */
|
|
29985
|
+
404: {
|
|
29986
|
+
headers: {
|
|
29987
|
+
[name: string]: unknown;
|
|
29988
|
+
};
|
|
29989
|
+
content: {
|
|
29990
|
+
"application/json": {
|
|
29991
|
+
error: {
|
|
29992
|
+
type: string;
|
|
29993
|
+
code: string;
|
|
29994
|
+
message: string;
|
|
29995
|
+
meta?: {
|
|
29996
|
+
[key: string]: unknown;
|
|
29997
|
+
};
|
|
29998
|
+
trace_id?: string;
|
|
29999
|
+
span_id?: string;
|
|
30000
|
+
};
|
|
30001
|
+
};
|
|
30002
|
+
};
|
|
30003
|
+
};
|
|
30004
|
+
};
|
|
30005
|
+
};
|
|
30006
|
+
cancelInstantEvalRun: {
|
|
30007
|
+
parameters: {
|
|
30008
|
+
query?: never;
|
|
30009
|
+
header?: never;
|
|
30010
|
+
path: {
|
|
30011
|
+
/** @description The run id. */
|
|
30012
|
+
id: string;
|
|
30013
|
+
};
|
|
30014
|
+
cookie?: never;
|
|
30015
|
+
};
|
|
30016
|
+
requestBody?: never;
|
|
30017
|
+
responses: {
|
|
30018
|
+
/** @description Success */
|
|
30019
|
+
200: {
|
|
30020
|
+
headers: {
|
|
30021
|
+
[name: string]: unknown;
|
|
30022
|
+
};
|
|
30023
|
+
content: {
|
|
30024
|
+
"application/json": {
|
|
30025
|
+
/** @description The run id. */
|
|
30026
|
+
id: string;
|
|
30027
|
+
/** @description What the run was called, if anything. */
|
|
30028
|
+
name: string | null;
|
|
30029
|
+
/** @description The statement, exactly as submitted. */
|
|
30030
|
+
sql: string;
|
|
30031
|
+
/** @description The values the statement's parameters were filled with. */
|
|
30032
|
+
parameters: {
|
|
30033
|
+
[key: string]: string | number | boolean | null;
|
|
30034
|
+
};
|
|
30035
|
+
/** @description One entry per eval function the statement projects, derived from it when the run was accepted. */
|
|
30036
|
+
questions: {
|
|
30037
|
+
/** @description The statement's own output column, which is the name this question is addressed by everywhere else. */
|
|
30038
|
+
id: string;
|
|
30039
|
+
/** @description The eval function that asked it. */
|
|
30040
|
+
function: string;
|
|
30041
|
+
/**
|
|
30042
|
+
* @description What kind of answer the question takes.
|
|
30043
|
+
* @enum {string}
|
|
30044
|
+
*/
|
|
30045
|
+
kind: "boolean" | "score" | "category";
|
|
30046
|
+
/** @description Which part of the verdict the statement's column carries. */
|
|
30047
|
+
reads: string;
|
|
30048
|
+
/** @description Where a boolean question's probability becomes a pass. Null for a question that is not a boolean. */
|
|
30049
|
+
threshold: number | null;
|
|
30050
|
+
}[];
|
|
30051
|
+
/** @description Rows this run may judge. */
|
|
30052
|
+
limit: number;
|
|
30053
|
+
/**
|
|
30054
|
+
* @description Where the run is in its life.
|
|
30055
|
+
* @enum {string}
|
|
30056
|
+
*/
|
|
30057
|
+
status: "queued" | "planning" | "running" | "finished" | "failed" | "cancelled";
|
|
30058
|
+
/** @description Rows the run found, bounded by its limit. Null until it has looked. */
|
|
30059
|
+
total: number | null;
|
|
30060
|
+
/** @description Rows judged so far. */
|
|
30061
|
+
progress: number;
|
|
30062
|
+
/** @description Judgements that matched, across this run's boolean questions. Null when the run asked none: a score or a category question has no match to count. */
|
|
30063
|
+
matched: number | null;
|
|
30064
|
+
/** @description Per question: matches for a boolean question, judged rows for a score or a category one. */
|
|
30065
|
+
matchedByQuestion: {
|
|
30066
|
+
[key: string]: number;
|
|
30067
|
+
};
|
|
30068
|
+
/** @description Rows the judge could not answer. */
|
|
30069
|
+
failed: number;
|
|
30070
|
+
/** @description Rows the judge declined to answer. */
|
|
30071
|
+
skipped: number;
|
|
30072
|
+
/** @description Input tokens the judge billed for. */
|
|
30073
|
+
tokens: number;
|
|
30074
|
+
/** @description What the judging costs you, in United States dollars. */
|
|
30075
|
+
priceUsd: number;
|
|
30076
|
+
/** @description The code of the failure that ended the run, when one did. */
|
|
30077
|
+
error: string | null;
|
|
30078
|
+
/** @description When the run was accepted. */
|
|
30079
|
+
createdAt: string;
|
|
30080
|
+
/** @description When the run was last written to. */
|
|
30081
|
+
updatedAt: string;
|
|
30082
|
+
/** @description When the run began reading rows. */
|
|
30083
|
+
startedAt: string | null;
|
|
30084
|
+
/** @description When the run ended. */
|
|
30085
|
+
finishedAt: string | null;
|
|
30086
|
+
};
|
|
30087
|
+
};
|
|
30088
|
+
};
|
|
30089
|
+
/** @description Not Found */
|
|
30090
|
+
404: {
|
|
30091
|
+
headers: {
|
|
30092
|
+
[name: string]: unknown;
|
|
30093
|
+
};
|
|
30094
|
+
content: {
|
|
30095
|
+
"application/json": {
|
|
30096
|
+
error: {
|
|
30097
|
+
type: string;
|
|
30098
|
+
code: string;
|
|
30099
|
+
message: string;
|
|
30100
|
+
meta?: {
|
|
30101
|
+
[key: string]: unknown;
|
|
30102
|
+
};
|
|
30103
|
+
trace_id?: string;
|
|
30104
|
+
span_id?: string;
|
|
30105
|
+
};
|
|
30106
|
+
};
|
|
30107
|
+
};
|
|
30108
|
+
};
|
|
30109
|
+
/** @description Conflict */
|
|
30110
|
+
409: {
|
|
30111
|
+
headers: {
|
|
30112
|
+
[name: string]: unknown;
|
|
30113
|
+
};
|
|
30114
|
+
content: {
|
|
30115
|
+
"application/json": {
|
|
30116
|
+
error: {
|
|
30117
|
+
type: string;
|
|
30118
|
+
code: string;
|
|
30119
|
+
message: string;
|
|
30120
|
+
meta?: {
|
|
30121
|
+
[key: string]: unknown;
|
|
30122
|
+
};
|
|
30123
|
+
trace_id?: string;
|
|
30124
|
+
span_id?: string;
|
|
30125
|
+
};
|
|
30126
|
+
};
|
|
30127
|
+
};
|
|
30128
|
+
};
|
|
30129
|
+
};
|
|
30130
|
+
};
|
|
30131
|
+
listInstantEvalRunResults: {
|
|
30132
|
+
parameters: {
|
|
30133
|
+
query?: {
|
|
30134
|
+
/** @description Only this question's judgements. */
|
|
30135
|
+
questionId?: string;
|
|
30136
|
+
/** @description Only judgements that matched, or only those that did not. Omit for both. */
|
|
30137
|
+
matched?: "true" | "1" | "yes" | "false" | "0" | "no";
|
|
30138
|
+
/** @description Only judgements in this state. */
|
|
30139
|
+
status?: "judged" | "skipped" | "failed";
|
|
30140
|
+
/** @description Judgements per page, at most one thousand. */
|
|
30141
|
+
limit?: number;
|
|
30142
|
+
/** @description The cursor the previous page answered with. */
|
|
30143
|
+
cursor?: string;
|
|
30144
|
+
};
|
|
30145
|
+
header?: never;
|
|
30146
|
+
path: {
|
|
30147
|
+
/** @description The run id. */
|
|
30148
|
+
id: string;
|
|
30149
|
+
};
|
|
30150
|
+
cookie?: never;
|
|
30151
|
+
};
|
|
30152
|
+
requestBody?: never;
|
|
30153
|
+
responses: {
|
|
30154
|
+
/** @description Success */
|
|
30155
|
+
200: {
|
|
30156
|
+
headers: {
|
|
30157
|
+
[name: string]: unknown;
|
|
30158
|
+
};
|
|
30159
|
+
content: {
|
|
30160
|
+
"application/json": {
|
|
30161
|
+
/** @description One page of the run's judgements. */
|
|
30162
|
+
judgments: {
|
|
30163
|
+
/** @description The trace the judgement is about. */
|
|
30164
|
+
traceId: string;
|
|
30165
|
+
/** @description The question it answers, named by its output column. */
|
|
30166
|
+
questionId: string;
|
|
30167
|
+
/** @description The conversation the trace belongs to. */
|
|
30168
|
+
threadId: string;
|
|
30169
|
+
/** @description The span the judged text was read from. */
|
|
30170
|
+
spanId: string;
|
|
30171
|
+
/** @description What kind of question was asked. */
|
|
30172
|
+
kind: string;
|
|
30173
|
+
/**
|
|
30174
|
+
* @description Whether the judge answered, declined, or could not answer.
|
|
30175
|
+
* @enum {string}
|
|
30176
|
+
*/
|
|
30177
|
+
status: "judged" | "skipped" | "failed";
|
|
30178
|
+
/** @description Whether a boolean question passed its threshold. */
|
|
30179
|
+
passed: boolean | null;
|
|
30180
|
+
/** @description A score question's answer. */
|
|
30181
|
+
score: number | null;
|
|
30182
|
+
/** @description A category question's answer. */
|
|
30183
|
+
label: string | null;
|
|
30184
|
+
/** @description How likely the judge found a boolean question's answer to be true. */
|
|
30185
|
+
probability: number | null;
|
|
30186
|
+
/** @description The full distribution behind a category answer. */
|
|
30187
|
+
probabilities: {
|
|
30188
|
+
[key: string]: number;
|
|
30189
|
+
} | null;
|
|
30190
|
+
/** @description Why the judge could not answer, when it could not. */
|
|
30191
|
+
error: string | null;
|
|
30192
|
+
/** @description When the judgement was made. */
|
|
30193
|
+
occurredAt: string;
|
|
30194
|
+
}[];
|
|
30195
|
+
/** @description Pass as cursor to read the page after this one. Absent on the last page. */
|
|
30196
|
+
nextCursor?: string;
|
|
30197
|
+
};
|
|
30198
|
+
};
|
|
30199
|
+
};
|
|
30200
|
+
};
|
|
30201
|
+
};
|
|
30202
|
+
sampleInstantEvalRun: {
|
|
30203
|
+
parameters: {
|
|
30204
|
+
query?: {
|
|
30205
|
+
/** @description Rows to re-read, at most twenty five. */
|
|
30206
|
+
n?: number;
|
|
30207
|
+
};
|
|
30208
|
+
header?: never;
|
|
30209
|
+
path: {
|
|
30210
|
+
/** @description The run id. */
|
|
30211
|
+
id: string;
|
|
30212
|
+
};
|
|
30213
|
+
cookie?: never;
|
|
30214
|
+
};
|
|
30215
|
+
requestBody?: never;
|
|
30216
|
+
responses: {
|
|
30217
|
+
/** @description Success */
|
|
30218
|
+
200: {
|
|
30219
|
+
headers: {
|
|
30220
|
+
[name: string]: unknown;
|
|
30221
|
+
};
|
|
30222
|
+
content: {
|
|
30223
|
+
"application/json": {
|
|
30224
|
+
/** @description The statement's own rows, with each judged column holding the text that was judged rather than the verdict. */
|
|
30225
|
+
rows: {
|
|
30226
|
+
[key: string]: unknown;
|
|
30227
|
+
}[];
|
|
30228
|
+
/** @description The verdicts those rows received. */
|
|
30229
|
+
judgments: {
|
|
30230
|
+
/** @description The trace the judgement is about. */
|
|
30231
|
+
traceId: string;
|
|
30232
|
+
/** @description The question it answers, named by its output column. */
|
|
30233
|
+
questionId: string;
|
|
30234
|
+
/** @description The conversation the trace belongs to. */
|
|
30235
|
+
threadId: string;
|
|
30236
|
+
/** @description The span the judged text was read from. */
|
|
30237
|
+
spanId: string;
|
|
30238
|
+
/** @description What kind of question was asked. */
|
|
30239
|
+
kind: string;
|
|
30240
|
+
/**
|
|
30241
|
+
* @description Whether the judge answered, declined, or could not answer.
|
|
30242
|
+
* @enum {string}
|
|
30243
|
+
*/
|
|
30244
|
+
status: "judged" | "skipped" | "failed";
|
|
30245
|
+
/** @description Whether a boolean question passed its threshold. */
|
|
30246
|
+
passed: boolean | null;
|
|
30247
|
+
/** @description A score question's answer. */
|
|
30248
|
+
score: number | null;
|
|
30249
|
+
/** @description A category question's answer. */
|
|
30250
|
+
label: string | null;
|
|
30251
|
+
/** @description How likely the judge found a boolean question's answer to be true. */
|
|
30252
|
+
probability: number | null;
|
|
30253
|
+
/** @description The full distribution behind a category answer. */
|
|
30254
|
+
probabilities: {
|
|
30255
|
+
[key: string]: number;
|
|
30256
|
+
} | null;
|
|
30257
|
+
/** @description Why the judge could not answer, when it could not. */
|
|
30258
|
+
error: string | null;
|
|
30259
|
+
/** @description When the judgement was made. */
|
|
30260
|
+
occurredAt: string;
|
|
30261
|
+
}[];
|
|
30262
|
+
};
|
|
30263
|
+
};
|
|
30264
|
+
};
|
|
30265
|
+
};
|
|
30266
|
+
};
|
|
30267
|
+
listRunPlans: {
|
|
30268
|
+
parameters: {
|
|
30269
|
+
query?: {
|
|
30270
|
+
/** @description Include archived run plans in the list. true, 1, yes for yes; false, 0, no or omitted for no. */
|
|
30271
|
+
includeArchived?: string;
|
|
30272
|
+
};
|
|
30273
|
+
header?: never;
|
|
30274
|
+
path?: never;
|
|
30275
|
+
cookie?: never;
|
|
30276
|
+
};
|
|
30277
|
+
requestBody?: never;
|
|
30278
|
+
responses: {
|
|
30279
|
+
/** @description Success */
|
|
30280
|
+
200: {
|
|
30281
|
+
headers: {
|
|
30282
|
+
[name: string]: unknown;
|
|
30283
|
+
};
|
|
30284
|
+
content: {
|
|
30285
|
+
"application/json": {
|
|
30286
|
+
/** @description The run plan id. */
|
|
30287
|
+
id: string;
|
|
30288
|
+
/** @description The run plan name. This is the plan's identity: a run started under this name joins this plan. */
|
|
30289
|
+
name: string;
|
|
30290
|
+
/** @description The plan's address in the platform. It is kept when the plan is renamed, so run history never moves. */
|
|
30291
|
+
slug: string;
|
|
30292
|
+
/** @description What the run plan covers: all (every active scenario), test_suites (the scenarios filed in the named test suites), labels (the scenarios carrying any of the labels), or scenarios (the scenarioIds sent with the configuration). A dynamic scope is resolved again at every run, so a scenario written later runs without editing the plan. */
|
|
30293
|
+
scope: {
|
|
30294
|
+
/** @constant */
|
|
30295
|
+
mode: "all";
|
|
30296
|
+
} | {
|
|
30297
|
+
/** @constant */
|
|
30298
|
+
mode: "test_suites";
|
|
30299
|
+
testSuiteIds: string[];
|
|
30300
|
+
} | {
|
|
30301
|
+
/** @constant */
|
|
30302
|
+
mode: "labels";
|
|
30303
|
+
labels: string[];
|
|
30304
|
+
} | {
|
|
30305
|
+
/** @constant */
|
|
30306
|
+
mode: "scenarios";
|
|
30307
|
+
};
|
|
30308
|
+
/** @description The scenarios the last run of this plan covered. */
|
|
30309
|
+
scenarioIds: string[];
|
|
30310
|
+
/** @description What the plan runs against, in the order the results show. A target carrying runParameters runs with those values. */
|
|
30311
|
+
targets: {
|
|
30312
|
+
/**
|
|
30313
|
+
* @description What kind of thing the scenarios run against. A connected agent is one registered from code with the SDK.
|
|
30314
|
+
* @enum {string}
|
|
30315
|
+
*/
|
|
30316
|
+
type: "prompt" | "http" | "code" | "workflow" | "connected" | "voice";
|
|
30317
|
+
/** @description The id of the prompt, agent or workflow to run against. A connected target may also say <name>@<environment>, for example support-agent@production, which resolves to the agent id. */
|
|
30318
|
+
referenceId: string;
|
|
30319
|
+
/** @description Parameter values this target alone runs with, by name. They are merged over the run-level parameters and the target wins, so two targets may name the same agent with different values: that is how one run compares one agent on two models, and the results show one column for each target. */
|
|
30320
|
+
runParameters?: {
|
|
30321
|
+
[key: string]: string | number | boolean;
|
|
30322
|
+
};
|
|
30323
|
+
}[];
|
|
29227
30324
|
/** @description How many times each scenario and target pairing runs. */
|
|
29228
30325
|
repeatCount: number;
|
|
29229
30326
|
/** @description The model that plays the user, or null for the scenario or project default. */
|
|
@@ -29234,11 +30331,8 @@ interface operations {
|
|
|
29234
30331
|
labels: string[];
|
|
29235
30332
|
/** @description The plan's own evaluators. Absent on servers that predate evaluators on this family. */
|
|
29236
30333
|
evaluators?: {
|
|
29237
|
-
/** @description The attachment id. Stable across edits of the attachment. */
|
|
29238
30334
|
id: string;
|
|
29239
|
-
/** @description The id of the saved evaluator this attachment runs. */
|
|
29240
30335
|
evaluatorId: string;
|
|
29241
|
-
/** @description Whether a failing result fails the scenario. A score-only evaluator reports and never gates. */
|
|
29242
30336
|
required: boolean;
|
|
29243
30337
|
/** @description Where each evaluator input reads its value, keyed by input name. Inputs left out are unmapped; a required input left unmapped refuses the run. */
|
|
29244
30338
|
mappings: {
|
|
@@ -29307,7 +30401,7 @@ interface operations {
|
|
|
29307
30401
|
* @description What kind of thing the scenarios run against. A connected agent is one registered from code with the SDK.
|
|
29308
30402
|
* @enum {string}
|
|
29309
30403
|
*/
|
|
29310
|
-
type: "prompt" | "http" | "code" | "workflow" | "connected";
|
|
30404
|
+
type: "prompt" | "http" | "code" | "workflow" | "connected" | "voice";
|
|
29311
30405
|
/** @description The id of the prompt, agent or workflow to run against. A connected target may also say <name>@<environment>, for example support-agent@production, which resolves to the agent id. */
|
|
29312
30406
|
referenceId: string;
|
|
29313
30407
|
/** @description Parameter values this target alone runs with, by name. They are merged over the run-level parameters and the target wins, so two targets may name the same agent with different values: that is how one run compares one agent on two models, and the results show one column for each target. */
|
|
@@ -29325,11 +30419,8 @@ interface operations {
|
|
|
29325
30419
|
scenarioIds?: string[];
|
|
29326
30420
|
/** @description The plan's own evaluators, run beside the ones attached to the test suites its scenarios belong to. A plan evaluator reads the conversation and the trace, never a scenario field. Leave it out to keep what the plan already holds. */
|
|
29327
30421
|
evaluators?: {
|
|
29328
|
-
/** @description The attachment id. Stable across edits of the attachment. */
|
|
29329
30422
|
id: string;
|
|
29330
|
-
/** @description The id of the saved evaluator this attachment runs. */
|
|
29331
30423
|
evaluatorId: string;
|
|
29332
|
-
/** @description Whether a failing result fails the scenario. A score-only evaluator reports and never gates. */
|
|
29333
30424
|
required: boolean;
|
|
29334
30425
|
/** @description Where each evaluator input reads its value, keyed by input name. Inputs left out are unmapped; a required input left unmapped refuses the run. */
|
|
29335
30426
|
mappings: {
|
|
@@ -29393,7 +30484,7 @@ interface operations {
|
|
|
29393
30484
|
* @description What kind of thing the scenarios run against. A connected agent is one registered from code with the SDK.
|
|
29394
30485
|
* @enum {string}
|
|
29395
30486
|
*/
|
|
29396
|
-
type: "prompt" | "http" | "code" | "workflow" | "connected";
|
|
30487
|
+
type: "prompt" | "http" | "code" | "workflow" | "connected" | "voice";
|
|
29397
30488
|
/** @description The id of the prompt, agent or workflow to run against. A connected target may also say <name>@<environment>, for example support-agent@production, which resolves to the agent id. */
|
|
29398
30489
|
referenceId: string;
|
|
29399
30490
|
/** @description Parameter values this target alone runs with, by name. They are merged over the run-level parameters and the target wins, so two targets may name the same agent with different values: that is how one run compares one agent on two models, and the results show one column for each target. */
|
|
@@ -29469,7 +30560,7 @@ interface operations {
|
|
|
29469
30560
|
* @description What kind of thing the scenarios run against. A connected agent is one registered from code with the SDK.
|
|
29470
30561
|
* @enum {string}
|
|
29471
30562
|
*/
|
|
29472
|
-
type: "prompt" | "http" | "code" | "workflow" | "connected";
|
|
30563
|
+
type: "prompt" | "http" | "code" | "workflow" | "connected" | "voice";
|
|
29473
30564
|
/** @description The id of the prompt, agent or workflow to run against. A connected target may also say <name>@<environment>, for example support-agent@production, which resolves to the agent id. */
|
|
29474
30565
|
referenceId: string;
|
|
29475
30566
|
/** @description Parameter values this target alone runs with, by name. They are merged over the run-level parameters and the target wins, so two targets may name the same agent with different values: that is how one run compares one agent on two models, and the results show one column for each target. */
|
|
@@ -29487,11 +30578,8 @@ interface operations {
|
|
|
29487
30578
|
labels: string[];
|
|
29488
30579
|
/** @description The plan's own evaluators. Absent on servers that predate evaluators on this family. */
|
|
29489
30580
|
evaluators?: {
|
|
29490
|
-
/** @description The attachment id. Stable across edits of the attachment. */
|
|
29491
30581
|
id: string;
|
|
29492
|
-
/** @description The id of the saved evaluator this attachment runs. */
|
|
29493
30582
|
evaluatorId: string;
|
|
29494
|
-
/** @description Whether a failing result fails the scenario. A score-only evaluator reports and never gates. */
|
|
29495
30583
|
required: boolean;
|
|
29496
30584
|
/** @description Where each evaluator input reads its value, keyed by input name. Inputs left out are unmapped; a required input left unmapped refuses the run. */
|
|
29497
30585
|
mappings: {
|
|
@@ -29614,7 +30702,7 @@ interface operations {
|
|
|
29614
30702
|
* @description What kind of thing the scenarios run against. A connected agent is one registered from code with the SDK.
|
|
29615
30703
|
* @enum {string}
|
|
29616
30704
|
*/
|
|
29617
|
-
type: "prompt" | "http" | "code" | "workflow" | "connected";
|
|
30705
|
+
type: "prompt" | "http" | "code" | "workflow" | "connected" | "voice";
|
|
29618
30706
|
/** @description The id of the prompt, agent or workflow to run against. A connected target may also say <name>@<environment>, for example support-agent@production, which resolves to the agent id. */
|
|
29619
30707
|
referenceId: string;
|
|
29620
30708
|
/** @description Parameter values this target alone runs with, by name. They are merged over the run-level parameters and the target wins, so two targets may name the same agent with different values: that is how one run compares one agent on two models, and the results show one column for each target. */
|
|
@@ -29672,21 +30760,14 @@ interface operations {
|
|
|
29672
30760
|
scenarioCount: number;
|
|
29673
30761
|
/** @description The fields the test suite declares. Absent on servers that predate fields on this family. */
|
|
29674
30762
|
fields?: {
|
|
29675
|
-
/** @description The field name, as scenarios and evaluator mappings address it. Lowercase letters, digits and underscores, starting with a letter. */
|
|
29676
30763
|
identifier: string;
|
|
29677
|
-
/**
|
|
29678
|
-
* @description The value type every scenario carries for this field.
|
|
29679
|
-
* @enum {string}
|
|
29680
|
-
*/
|
|
30764
|
+
/** @enum {string} */
|
|
29681
30765
|
type: "text" | "number" | "boolean";
|
|
29682
30766
|
}[];
|
|
29683
30767
|
/** @description The evaluators attached to the test suite. Absent on servers that predate evaluators on this family. */
|
|
29684
30768
|
evaluators?: {
|
|
29685
|
-
/** @description The attachment id. Stable across edits of the attachment. */
|
|
29686
30769
|
id: string;
|
|
29687
|
-
/** @description The id of the saved evaluator this attachment runs. */
|
|
29688
30770
|
evaluatorId: string;
|
|
29689
|
-
/** @description Whether a failing result fails the scenario. A score-only evaluator reports and never gates. */
|
|
29690
30771
|
required: boolean;
|
|
29691
30772
|
/** @description Where each evaluator input reads its value, keyed by input name. Inputs left out are unmapped; a required input left unmapped refuses the run. */
|
|
29692
30773
|
mappings: {
|
|
@@ -29733,21 +30814,14 @@ interface operations {
|
|
|
29733
30814
|
name: string;
|
|
29734
30815
|
/** @description The fields the test suite declares, in the order the platform shows them. Up to 30. An identifier is lowercase letters, digits and underscores, starting with a letter; the type is text, number or boolean. */
|
|
29735
30816
|
fields?: {
|
|
29736
|
-
/** @description The field name, as scenarios and evaluator mappings address it. Lowercase letters, digits and underscores, starting with a letter. */
|
|
29737
30817
|
identifier: string;
|
|
29738
|
-
/**
|
|
29739
|
-
* @description The value type every scenario carries for this field.
|
|
29740
|
-
* @enum {string}
|
|
29741
|
-
*/
|
|
30818
|
+
/** @enum {string} */
|
|
29742
30819
|
type: "text" | "number" | "boolean";
|
|
29743
30820
|
}[];
|
|
29744
30821
|
/** @description The evaluators that run after every scenario run. Up to 20. A required evaluator that fails fails the scenario; a score-only evaluator reports and never gates. */
|
|
29745
30822
|
evaluators?: {
|
|
29746
|
-
/** @description The attachment id. Stable across edits of the attachment. */
|
|
29747
30823
|
id: string;
|
|
29748
|
-
/** @description The id of the saved evaluator this attachment runs. */
|
|
29749
30824
|
evaluatorId: string;
|
|
29750
|
-
/** @description Whether a failing result fails the scenario. A score-only evaluator reports and never gates. */
|
|
29751
30825
|
required: boolean;
|
|
29752
30826
|
/** @description Where each evaluator input reads its value, keyed by input name. Inputs left out are unmapped; a required input left unmapped refuses the run. */
|
|
29753
30827
|
mappings: {
|
|
@@ -29787,21 +30861,14 @@ interface operations {
|
|
|
29787
30861
|
scenarioCount: number;
|
|
29788
30862
|
/** @description The fields the test suite declares. Absent on servers that predate fields on this family. */
|
|
29789
30863
|
fields?: {
|
|
29790
|
-
/** @description The field name, as scenarios and evaluator mappings address it. Lowercase letters, digits and underscores, starting with a letter. */
|
|
29791
30864
|
identifier: string;
|
|
29792
|
-
/**
|
|
29793
|
-
* @description The value type every scenario carries for this field.
|
|
29794
|
-
* @enum {string}
|
|
29795
|
-
*/
|
|
30865
|
+
/** @enum {string} */
|
|
29796
30866
|
type: "text" | "number" | "boolean";
|
|
29797
30867
|
}[];
|
|
29798
30868
|
/** @description The evaluators attached to the test suite. Absent on servers that predate evaluators on this family. */
|
|
29799
30869
|
evaluators?: {
|
|
29800
|
-
/** @description The attachment id. Stable across edits of the attachment. */
|
|
29801
30870
|
id: string;
|
|
29802
|
-
/** @description The id of the saved evaluator this attachment runs. */
|
|
29803
30871
|
evaluatorId: string;
|
|
29804
|
-
/** @description Whether a failing result fails the scenario. A score-only evaluator reports and never gates. */
|
|
29805
30872
|
required: boolean;
|
|
29806
30873
|
/** @description Where each evaluator input reads its value, keyed by input name. Inputs left out are unmapped; a required input left unmapped refuses the run. */
|
|
29807
30874
|
mappings: {
|
|
@@ -29865,21 +30932,14 @@ interface operations {
|
|
|
29865
30932
|
scenarioCount: number;
|
|
29866
30933
|
/** @description The fields the test suite declares. Absent on servers that predate fields on this family. */
|
|
29867
30934
|
fields?: {
|
|
29868
|
-
/** @description The field name, as scenarios and evaluator mappings address it. Lowercase letters, digits and underscores, starting with a letter. */
|
|
29869
30935
|
identifier: string;
|
|
29870
|
-
/**
|
|
29871
|
-
* @description The value type every scenario carries for this field.
|
|
29872
|
-
* @enum {string}
|
|
29873
|
-
*/
|
|
30936
|
+
/** @enum {string} */
|
|
29874
30937
|
type: "text" | "number" | "boolean";
|
|
29875
30938
|
}[];
|
|
29876
30939
|
/** @description The evaluators attached to the test suite. Absent on servers that predate evaluators on this family. */
|
|
29877
30940
|
evaluators?: {
|
|
29878
|
-
/** @description The attachment id. Stable across edits of the attachment. */
|
|
29879
30941
|
id: string;
|
|
29880
|
-
/** @description The id of the saved evaluator this attachment runs. */
|
|
29881
30942
|
evaluatorId: string;
|
|
29882
|
-
/** @description Whether a failing result fails the scenario. A score-only evaluator reports and never gates. */
|
|
29883
30943
|
required: boolean;
|
|
29884
30944
|
/** @description Where each evaluator input reads its value, keyed by input name. Inputs left out are unmapped; a required input left unmapped refuses the run. */
|
|
29885
30945
|
mappings: {
|
|
@@ -29967,21 +31027,14 @@ interface operations {
|
|
|
29967
31027
|
name?: string;
|
|
29968
31028
|
/** @description The full list of fields the suite declares. A field an attached evaluator still reads cannot be removed: answers 422 suite_field_in_use. */
|
|
29969
31029
|
fields?: {
|
|
29970
|
-
/** @description The field name, as scenarios and evaluator mappings address it. Lowercase letters, digits and underscores, starting with a letter. */
|
|
29971
31030
|
identifier: string;
|
|
29972
|
-
/**
|
|
29973
|
-
* @description The value type every scenario carries for this field.
|
|
29974
|
-
* @enum {string}
|
|
29975
|
-
*/
|
|
31031
|
+
/** @enum {string} */
|
|
29976
31032
|
type: "text" | "number" | "boolean";
|
|
29977
31033
|
}[];
|
|
29978
31034
|
/** @description The full list of evaluators attached to the suite. An evaluator the project does not hold answers 422 suite_evaluator_not_found; a mapping the run cannot read answers 422 suite_evaluator_mapping_invalid. */
|
|
29979
31035
|
evaluators?: {
|
|
29980
|
-
/** @description The attachment id. Stable across edits of the attachment. */
|
|
29981
31036
|
id: string;
|
|
29982
|
-
/** @description The id of the saved evaluator this attachment runs. */
|
|
29983
31037
|
evaluatorId: string;
|
|
29984
|
-
/** @description Whether a failing result fails the scenario. A score-only evaluator reports and never gates. */
|
|
29985
31038
|
required: boolean;
|
|
29986
31039
|
/** @description Where each evaluator input reads its value, keyed by input name. Inputs left out are unmapped; a required input left unmapped refuses the run. */
|
|
29987
31040
|
mappings: {
|
|
@@ -30021,21 +31074,14 @@ interface operations {
|
|
|
30021
31074
|
scenarioCount: number;
|
|
30022
31075
|
/** @description The fields the test suite declares. Absent on servers that predate fields on this family. */
|
|
30023
31076
|
fields?: {
|
|
30024
|
-
/** @description The field name, as scenarios and evaluator mappings address it. Lowercase letters, digits and underscores, starting with a letter. */
|
|
30025
31077
|
identifier: string;
|
|
30026
|
-
/**
|
|
30027
|
-
* @description The value type every scenario carries for this field.
|
|
30028
|
-
* @enum {string}
|
|
30029
|
-
*/
|
|
31078
|
+
/** @enum {string} */
|
|
30030
31079
|
type: "text" | "number" | "boolean";
|
|
30031
31080
|
}[];
|
|
30032
31081
|
/** @description The evaluators attached to the test suite. Absent on servers that predate evaluators on this family. */
|
|
30033
31082
|
evaluators?: {
|
|
30034
|
-
/** @description The attachment id. Stable across edits of the attachment. */
|
|
30035
31083
|
id: string;
|
|
30036
|
-
/** @description The id of the saved evaluator this attachment runs. */
|
|
30037
31084
|
evaluatorId: string;
|
|
30038
|
-
/** @description Whether a failing result fails the scenario. A score-only evaluator reports and never gates. */
|
|
30039
31085
|
required: boolean;
|
|
30040
31086
|
/** @description Where each evaluator input reads its value, keyed by input name. Inputs left out are unmapped; a required input left unmapped refuses the run. */
|
|
30041
31087
|
mappings: {
|
|
@@ -30087,7 +31133,7 @@ interface operations {
|
|
|
30087
31133
|
* @description What kind of thing the scenarios run against. A connected agent is one registered from code with the SDK.
|
|
30088
31134
|
* @enum {string}
|
|
30089
31135
|
*/
|
|
30090
|
-
type: "prompt" | "http" | "code" | "workflow" | "connected";
|
|
31136
|
+
type: "prompt" | "http" | "code" | "workflow" | "connected" | "voice";
|
|
30091
31137
|
/** @description The id of the prompt, agent or workflow to run against. A connected target may also say <name>@<environment>, for example support-agent@production, which resolves to the agent id. */
|
|
30092
31138
|
referenceId: string;
|
|
30093
31139
|
/** @description Parameter values this target alone runs with, by name. They are merged over the run-level parameters and the target wins, so two targets may name the same agent with different values: that is how one run compares one agent on two models, and the results show one column for each target. */
|
|
@@ -30149,7 +31195,7 @@ interface operations {
|
|
|
30149
31195
|
* @description What kind of thing the scenarios run against. A connected agent is one registered from code with the SDK.
|
|
30150
31196
|
* @enum {string}
|
|
30151
31197
|
*/
|
|
30152
|
-
type: "prompt" | "http" | "code" | "workflow" | "connected";
|
|
31198
|
+
type: "prompt" | "http" | "code" | "workflow" | "connected" | "voice";
|
|
30153
31199
|
/** @description The id of the prompt, agent or workflow to run against. A connected target may also say <name>@<environment>, for example support-agent@production, which resolves to the agent id. */
|
|
30154
31200
|
referenceId: string;
|
|
30155
31201
|
/** @description Parameter values this target alone runs with, by name. They are merged over the run-level parameters and the target wins, so two targets may name the same agent with different values: that is how one run compares one agent on two models, and the results show one column for each target. */
|
|
@@ -30690,6 +31736,7 @@ interface operations {
|
|
|
30690
31736
|
};
|
|
30691
31737
|
traceIds?: string[];
|
|
30692
31738
|
negateFilters?: boolean;
|
|
31739
|
+
excludeOrigins?: string[];
|
|
30693
31740
|
/** @description Removed. Offset pagination is no longer supported and any value other than 0 is rejected. Page with the scrollId returned by the previous response instead. The field remains on the schema so that sending it produces an explanatory error rather than being silently discarded. */
|
|
30694
31741
|
pageOffset?: number;
|
|
30695
31742
|
pageSize?: number;
|
|
@@ -30708,6 +31755,8 @@ interface operations {
|
|
|
30708
31755
|
/** @description When true, fetches full span data for each trace. Useful for bulk export. Default false. */
|
|
30709
31756
|
includeSpans?: boolean;
|
|
30710
31757
|
llmMode?: boolean;
|
|
31758
|
+
/** @description A trace filter string in the same language the Trace Explorer's search bar speaks — `status:error AND model:gpt-*`, `trace.attribute.langwatch.user_id:alice`, `evaluatorVerdict:fail`, a quoted phrase for free text. It is combined with `filters`, `query` and `traceIds` rather than replacing any of them, so every condition you send must hold. `GET /api/v1/query/reference` lists every field and the syntax; `GET /api/traces/facets` says what values a field actually holds. A malformed filter, or one naming a field the language does not have, is a 422 that names the field. */
|
|
31759
|
+
filter?: string;
|
|
30711
31760
|
/**
|
|
30712
31761
|
* @description Which timestamp the startDate/endDate window filters on. 'occurred' (default) selects traces by when they happened. 'updated' selects traces by when they were last modified — use this for incremental ETL ('give me everything changed since my last pull'), since a trace can occur long before it gains a later evaluation or annotation.
|
|
30713
31762
|
* @default occurred
|
|
@@ -30804,6 +31853,131 @@ interface operations {
|
|
|
30804
31853
|
};
|
|
30805
31854
|
};
|
|
30806
31855
|
};
|
|
31856
|
+
getApiTracesFacets: {
|
|
31857
|
+
parameters: {
|
|
31858
|
+
query?: {
|
|
31859
|
+
field?: string;
|
|
31860
|
+
prefix?: string;
|
|
31861
|
+
limit?: number;
|
|
31862
|
+
offset?: number;
|
|
31863
|
+
startDate?: string;
|
|
31864
|
+
endDate?: string;
|
|
31865
|
+
};
|
|
31866
|
+
header?: never;
|
|
31867
|
+
path?: never;
|
|
31868
|
+
cookie?: never;
|
|
31869
|
+
};
|
|
31870
|
+
requestBody?: never;
|
|
31871
|
+
responses: {
|
|
31872
|
+
/** @description Without `field`, every facet the project has with its top values and whether the payload is still being computed. With `field`, that field's values and counts plus the distinct total and whether more remain. */
|
|
31873
|
+
200: {
|
|
31874
|
+
headers: {
|
|
31875
|
+
[name: string]: unknown;
|
|
31876
|
+
};
|
|
31877
|
+
content: {
|
|
31878
|
+
"application/json": {
|
|
31879
|
+
facets: ({
|
|
31880
|
+
key: string;
|
|
31881
|
+
/** @enum {string} */
|
|
31882
|
+
kind: "categorical" | "range" | "dynamic_keys";
|
|
31883
|
+
label: string;
|
|
31884
|
+
group: string;
|
|
31885
|
+
} & {
|
|
31886
|
+
[key: string]: unknown;
|
|
31887
|
+
})[];
|
|
31888
|
+
/** @description True when the payload is still being computed and what you have is the last committed one, possibly empty. Call again shortly. */
|
|
31889
|
+
pending: boolean;
|
|
31890
|
+
} | {
|
|
31891
|
+
values: {
|
|
31892
|
+
value: string;
|
|
31893
|
+
label?: string;
|
|
31894
|
+
count: number;
|
|
31895
|
+
}[];
|
|
31896
|
+
/** @description Distinct values the field holds in the window, before paging. */
|
|
31897
|
+
total: number;
|
|
31898
|
+
hasMore: boolean;
|
|
31899
|
+
};
|
|
31900
|
+
};
|
|
31901
|
+
};
|
|
31902
|
+
/** @description Bad Request */
|
|
31903
|
+
400: {
|
|
31904
|
+
headers: {
|
|
31905
|
+
[name: string]: unknown;
|
|
31906
|
+
};
|
|
31907
|
+
content: {
|
|
31908
|
+
"application/json": {
|
|
31909
|
+
error: string;
|
|
31910
|
+
message?: string;
|
|
31911
|
+
};
|
|
31912
|
+
};
|
|
31913
|
+
};
|
|
31914
|
+
/** @description Unauthorized */
|
|
31915
|
+
401: {
|
|
31916
|
+
headers: {
|
|
31917
|
+
[name: string]: unknown;
|
|
31918
|
+
};
|
|
31919
|
+
content: {
|
|
31920
|
+
"application/json": {
|
|
31921
|
+
error: string;
|
|
31922
|
+
message?: string;
|
|
31923
|
+
};
|
|
31924
|
+
};
|
|
31925
|
+
};
|
|
31926
|
+
/** @description The field is an attribute key and this project hides captured input or output, so its values are not listed. */
|
|
31927
|
+
403: {
|
|
31928
|
+
headers: {
|
|
31929
|
+
[name: string]: unknown;
|
|
31930
|
+
};
|
|
31931
|
+
content: {
|
|
31932
|
+
"application/json": {
|
|
31933
|
+
/** @constant */
|
|
31934
|
+
error: "trace_attribute_values_withheld";
|
|
31935
|
+
message: string;
|
|
31936
|
+
trace?: string;
|
|
31937
|
+
};
|
|
31938
|
+
};
|
|
31939
|
+
};
|
|
31940
|
+
/** @description The query did not name a facet with values to list. `fields` names the offending parameter and each reason carries what was received and what exists. */
|
|
31941
|
+
422: {
|
|
31942
|
+
headers: {
|
|
31943
|
+
[name: string]: unknown;
|
|
31944
|
+
};
|
|
31945
|
+
content: {
|
|
31946
|
+
"application/json": {
|
|
31947
|
+
/** @constant */
|
|
31948
|
+
error: "validation_error";
|
|
31949
|
+
message: string;
|
|
31950
|
+
/** @constant */
|
|
31951
|
+
target: "query";
|
|
31952
|
+
fields: string[];
|
|
31953
|
+
reasons: {
|
|
31954
|
+
code: string;
|
|
31955
|
+
meta?: {
|
|
31956
|
+
field?: string;
|
|
31957
|
+
type?: string;
|
|
31958
|
+
message?: string;
|
|
31959
|
+
received?: string;
|
|
31960
|
+
expected?: string[];
|
|
31961
|
+
};
|
|
31962
|
+
}[];
|
|
31963
|
+
trace?: string;
|
|
31964
|
+
};
|
|
31965
|
+
};
|
|
31966
|
+
};
|
|
31967
|
+
/** @description Internal Server Error */
|
|
31968
|
+
500: {
|
|
31969
|
+
headers: {
|
|
31970
|
+
[name: string]: unknown;
|
|
31971
|
+
};
|
|
31972
|
+
content: {
|
|
31973
|
+
"application/json": {
|
|
31974
|
+
error: string;
|
|
31975
|
+
message?: string;
|
|
31976
|
+
};
|
|
31977
|
+
};
|
|
31978
|
+
};
|
|
31979
|
+
};
|
|
31980
|
+
};
|
|
30807
31981
|
getApiTracesByTraceIdTranscript: {
|
|
30808
31982
|
parameters: {
|
|
30809
31983
|
query?: never;
|
|
@@ -33422,6 +34596,12 @@ interface operations {
|
|
|
33422
34596
|
cache_read_input_tokens: number;
|
|
33423
34597
|
cache_creation_input_tokens: number;
|
|
33424
34598
|
reasoning_tokens: number;
|
|
34599
|
+
/** @description Image tokens billed on the input side, 0 when the request carried no image. Priced at its own rate and disjoint from input_tokens, which never includes it. */
|
|
34600
|
+
input_image_tokens: number;
|
|
34601
|
+
/** @description Image tokens the answer was billed for, 0 when the answer held no image. Priced at its own rate and disjoint from output_tokens: an image_generation row reports output_tokens 0 and its render here, so a reconciler reading output_tokens alone sees none of the image traffic. */
|
|
34602
|
+
output_image_tokens: number;
|
|
34603
|
+
/** @description Images the request carried, 0 when it carried none. Display only: no rate prices it, so it never belongs in a cost sum. */
|
|
34604
|
+
image_count: number;
|
|
33425
34605
|
};
|
|
33426
34606
|
cost: {
|
|
33427
34607
|
/** @description Display value. Decimal string, up to 9 fractional digits, trailing zeros trimmed, never exponent notation. Use nano_usd for arithmetic. */
|
|
@@ -33567,6 +34747,12 @@ interface operations {
|
|
|
33567
34747
|
cache_read_input_tokens: number;
|
|
33568
34748
|
cache_creation_input_tokens: number;
|
|
33569
34749
|
reasoning_tokens: number;
|
|
34750
|
+
/** @description Image tokens billed on the input side, 0 when the request carried no image. Priced at its own rate and disjoint from input_tokens, which never includes it. */
|
|
34751
|
+
input_image_tokens: number;
|
|
34752
|
+
/** @description Image tokens the answer was billed for, 0 when the answer held no image. Priced at its own rate and disjoint from output_tokens: an image_generation row reports output_tokens 0 and its render here, so a reconciler reading output_tokens alone sees none of the image traffic. */
|
|
34753
|
+
output_image_tokens: number;
|
|
34754
|
+
/** @description Images the request carried, 0 when it carried none. Display only: no rate prices it, so it never belongs in a cost sum. */
|
|
34755
|
+
image_count: number;
|
|
33570
34756
|
} | null;
|
|
33571
34757
|
cost: {
|
|
33572
34758
|
/** @description Display value. Decimal string, up to 9 fractional digits, trailing zeros trimmed, never exponent notation. Use nano_usd for arithmetic. */
|
|
@@ -33717,6 +34903,12 @@ interface operations {
|
|
|
33717
34903
|
cache_read_input_tokens: number;
|
|
33718
34904
|
cache_creation_input_tokens: number;
|
|
33719
34905
|
reasoning_tokens: number;
|
|
34906
|
+
/** @description Image tokens billed on the input side, 0 when the request carried no image. Priced at its own rate and disjoint from input_tokens, which never includes it. */
|
|
34907
|
+
input_image_tokens: number;
|
|
34908
|
+
/** @description Image tokens the answer was billed for, 0 when the answer held no image. Priced at its own rate and disjoint from output_tokens: an image_generation row reports output_tokens 0 and its render here, so a reconciler reading output_tokens alone sees none of the image traffic. */
|
|
34909
|
+
output_image_tokens: number;
|
|
34910
|
+
/** @description Images the request carried, 0 when it carried none. Display only: no rate prices it, so it never belongs in a cost sum. */
|
|
34911
|
+
image_count: number;
|
|
33720
34912
|
};
|
|
33721
34913
|
caps: {
|
|
33722
34914
|
budget_id: string;
|