@langwatch/mcp-server 1.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.python-version +1 -0
- package/CHANGELOG.md +110 -0
- package/CONTRIBUTING.md +1 -1
- package/README.md +86 -4
- package/dist/{archive-project-MK3673A7.js → archive-project-W3IKI7UL.js} +5 -5
- package/dist/archive-run-plan-474OKHHU.js +16 -0
- package/dist/{archive-scenario-KATSXRIN.js → archive-scenario-V5UNYK26.js} +5 -5
- package/dist/archive-test-suite-O7FLDMXK.js +16 -0
- package/dist/{assign-prompt-tag-E3C5W6PI.js → assign-prompt-tag-Q46WRB6P.js} +4 -4
- package/dist/chunk-56OGTS24.js +44 -0
- package/dist/chunk-6DJDEXET.js +63 -0
- package/dist/chunk-7E4FZIYJ.js +15 -0
- package/dist/{chunk-J45SVWPR.js → chunk-AN4OUA4D.js} +2 -2
- package/dist/chunk-CKNAH5TG.js +74 -0
- package/dist/{chunk-CE6QMZOP.js → chunk-EYXMLE5X.js} +86 -11
- package/dist/{chunk-XY4STOBE.js → chunk-G4ZAZTZW.js} +2 -2
- package/dist/{chunk-5UWY24RP.js → chunk-GELJ2MQR.js} +12 -3
- package/dist/chunk-HV4DWKBI.js +46 -0
- package/dist/chunk-JR6DJZOU.js +49 -0
- package/dist/{chunk-DAIQKS27.js → chunk-JTQJRNFJ.js} +2 -2
- package/dist/{chunk-WWMDHZZV.js → chunk-K37ZNPXK.js} +2 -2
- package/dist/{chunk-EU22L52K.js → chunk-OJLYYJWI.js} +2 -2
- package/dist/{chunk-Y3HDZ6OW.js → chunk-QAMP3CGR.js} +2 -2
- package/dist/{chunk-VYPSKRPN.js → chunk-QRO35SOM.js} +347 -146
- package/dist/{chunk-O3LWVN67.js → chunk-RVAE2MEA.js} +2 -2
- package/dist/{chunk-UXXA4D6K.js → chunk-S4GVX36W.js} +2 -2
- package/dist/{chunk-BJ7B4R26.js → chunk-SAS5OB7F.js} +18 -2
- package/dist/{chunk-RYRB2YVJ.js → chunk-SR6QPRIE.js} +2 -2
- package/dist/chunk-UF6XIGYR.js +58 -0
- package/dist/{chunk-T7WJVS2N.js → chunk-UFOYFYI7.js} +2 -2
- package/dist/chunk-UJPXFA26.js +267 -0
- package/dist/{chunk-KOO5733S.js → chunk-WT4PO4A3.js} +8 -2
- package/dist/config.js +2 -2
- package/dist/{create-agent-7W57J7MT.js → create-agent-JFJ2K6X3.js} +5 -5
- package/dist/{create-annotation-W2XELBCQ.js → create-annotation-EUXXQ6NS.js} +5 -5
- package/dist/{create-api-key-MKRXKFJM.js → create-api-key-BGBB76JX.js} +5 -5
- package/dist/{create-dashboard-QGL7IODC.js → create-dashboard-Z76AIPLS.js} +5 -5
- package/dist/{create-dataset-3PH7A4HG.js → create-dataset-FD5VAXU4.js} +5 -5
- package/dist/{create-dataset-records-4DZS3VRL.js → create-dataset-records-WQUXXOHI.js} +5 -5
- package/dist/{create-evaluator-YWH6NC4M.js → create-evaluator-6NV33Z52.js} +5 -5
- package/dist/create-mcp-server.js +9 -4
- package/dist/{create-project-BOBQ32C5.js → create-project-ZQ7TFIO2.js} +5 -5
- package/dist/{create-prompt-FLCFGITE.js → create-prompt-BS6QRCUU.js} +4 -4
- package/dist/{create-prompt-tag-EMPNIKEA.js → create-prompt-tag-FCQ5ELSQ.js} +4 -4
- package/dist/{create-scenario-BSBIHUL6.js → create-scenario-DRPTPXU3.js} +12 -5
- package/dist/create-test-suite-ZIZCCFJ4.js +37 -0
- package/dist/{delete-agent-7MSYNY5E.js → delete-agent-SA4JMUZF.js} +5 -5
- package/dist/{delete-annotation-ZT5IGN24.js → delete-annotation-X3KKK7QY.js} +5 -5
- package/dist/{delete-dashboard-BTIY5S3L.js → delete-dashboard-LYMRPF27.js} +5 -5
- package/dist/{delete-dataset-U45HQHPR.js → delete-dataset-W6EZ44HY.js} +5 -5
- package/dist/{delete-dataset-records-R43IMAWB.js → delete-dataset-records-7CNOTALB.js} +5 -5
- package/dist/{delete-evaluator-U6JMQ6C7.js → delete-evaluator-RKYC4TNB.js} +5 -5
- package/dist/{delete-prompt-tag-4EV4TDHB.js → delete-prompt-tag-QB3OCZUV.js} +4 -4
- package/dist/{delete-workflow-5QYALXEL.js → delete-workflow-NG2VXYYM.js} +5 -5
- package/dist/{discover-evaluator-schema-MRGHVNTT.js → discover-evaluator-schema-ESSHQGPY.js} +58 -321
- package/dist/{discover-scenario-schema-UPBTCWPG.js → discover-scenario-schema-W7KBXA4J.js} +54 -2
- package/dist/{discover-schema-ICGXPHUW.js → discover-schema-L6QDPON3.js} +2 -2
- package/dist/get-agent-7THF7HCU.js +66 -0
- package/dist/{get-analytics-PVERWOA5.js → get-analytics-R6AZD3AW.js} +4 -4
- package/dist/{get-dashboard-ZZPC3WMD.js → get-dashboard-P4WWVHFU.js} +5 -5
- package/dist/{get-dataset-25TB2LQH.js → get-dataset-AKTYSQDI.js} +5 -5
- package/dist/{get-evaluator-3FTRQKEC.js → get-evaluator-RZ44UIN6.js} +5 -5
- package/dist/{get-project-5OAXYC57.js → get-project-QSQEAB33.js} +5 -5
- package/dist/get-prompt-GVKXF5NL.js +100 -0
- package/dist/get-run-plan-NY7J5GRE.js +61 -0
- package/dist/{get-scenario-DSSTR6FI.js → get-scenario-Q3LLQE4T.js} +9 -5
- package/dist/{get-simulation-run-RQPMS2MC.js → get-simulation-run-WINEYDMK.js} +14 -5
- package/dist/get-test-suite-4HKA7OKS.js +49 -0
- package/dist/{get-trace-O5VT3NAC.js → get-trace-RCUV7N4O.js} +4 -4
- package/dist/{get-workflow-3EZDOEIF.js → get-workflow-EEOC7Y45.js} +5 -5
- package/dist/http-security-EU3CHTY7.js +26 -0
- package/dist/{http-server-2ACJ3CXW.js → http-server-WCDUARTH.js} +1038 -693
- package/dist/index.js +6437 -4020
- package/dist/{langwatch-api-annotations-E65MI3WZ.js → langwatch-api-annotations-6DHVZWUK.js} +5 -5
- package/dist/{langwatch-api-dashboards-QCDECRCY.js → langwatch-api-dashboards-2KJL2M42.js} +5 -5
- package/dist/{langwatch-api-monitors-UIO2KTDV.js → langwatch-api-monitors-6ISFJ7WX.js} +4 -4
- package/dist/{langwatch-api-secrets-4T2EKSJ7.js → langwatch-api-secrets-OEBD6GBU.js} +4 -4
- package/dist/{langwatch-api-triggers-MPOEKPUY.js → langwatch-api-triggers-5E6DDGX6.js} +4 -4
- package/dist/langwatch-api-workflows-6UJL5U4Z.js +16 -0
- package/dist/list-agents-AMVDQPQL.js +43 -0
- package/dist/{list-annotations-CGX5BT7E.js → list-annotations-6PR5NLIC.js} +5 -5
- package/dist/{list-api-keys-JLWJPFRX.js → list-api-keys-GVPNBIHQ.js} +5 -5
- package/dist/{list-dashboards-RNIWX3TX.js → list-dashboards-Z3BZKDGF.js} +5 -5
- package/dist/{list-dataset-records-B5MHZR66.js → list-dataset-records-O67HZ35Y.js} +5 -5
- package/dist/{list-datasets-BVCAMQCP.js → list-datasets-WXZL6UMP.js} +5 -5
- package/dist/{list-evaluators-65FBTL2U.js → list-evaluators-PAX72WCN.js} +5 -5
- package/dist/{list-model-providers-GRK3I3VV.js → list-model-providers-VVLZMLCX.js} +5 -5
- package/dist/{list-projects-JLMBFMW3.js → list-projects-AQVI6SUK.js} +5 -5
- package/dist/{list-prompt-tags-UHNWDGQI.js → list-prompt-tags-WZFBJOHG.js} +4 -4
- package/dist/{list-prompts-7254B3U7.js → list-prompts-BQOHRHYT.js} +4 -4
- package/dist/list-run-plans-UBAU2DGH.js +46 -0
- package/dist/{list-scenarios-46YNICKM.js → list-scenarios-MS4IUIVM.js} +10 -7
- package/dist/{list-simulation-runs-LGNEFNPN.js → list-simulation-runs-D6YF4ZOU.js} +13 -7
- package/dist/list-test-suites-L7R2O6DR.js +35 -0
- package/dist/{list-workflows-MLQDM6CP.js → list-workflows-6MV4XJHF.js} +5 -5
- package/dist/{rename-prompt-tag-RHQMNEQL.js → rename-prompt-tag-K5LQ4CY6.js} +4 -4
- package/dist/rename-test-suite-NTYHAO5X.js +21 -0
- package/dist/report-issue-RHZZ33IS.js +885 -0
- package/dist/rerun-run-plan-MUDWJBE4.js +22 -0
- package/dist/{revoke-api-key-3Y35EMBT.js → revoke-api-key-6LAZ3QZZ.js} +5 -5
- package/dist/run-agent-CAGWGTYT.js +72 -0
- package/dist/run-plan-T4ZYOZGA.js +39 -0
- package/dist/run-test-suite-L4MZ7G4D.js +26 -0
- package/dist/{run-workflow-5LVBZYLS.js → run-workflow-DFIS23AP.js} +5 -5
- package/dist/{search-traces-U4HLOCZ2.js → search-traces-4XTJYXIK.js} +4 -4
- package/dist/{set-model-provider-V3M5CDIY.js → set-model-provider-XRNMFOVR.js} +5 -5
- package/dist/{update-agent-47RZIHS7.js → update-agent-DUWNYBNA.js} +5 -5
- package/dist/{update-dataset-G6QJJGS5.js → update-dataset-YSOMDCOB.js} +5 -5
- package/dist/{update-dataset-record-RMWLSWCS.js → update-dataset-record-DEA2ITYR.js} +5 -5
- package/dist/{update-evaluator-SFRRI5Q3.js → update-evaluator-NH7Q5GWV.js} +5 -5
- package/dist/{update-project-PI5DWBVK.js → update-project-Q26FBSYH.js} +5 -5
- package/dist/update-prompt-ZQQ2C3KU.js +143 -0
- package/dist/{update-scenario-G3IU2UIB.js → update-scenario-WUX34JAR.js} +12 -5
- package/package.json +8 -9
- package/pyproject.toml +5 -5
- package/src/__tests__/agent-tools.unit.test.ts +257 -0
- package/src/__tests__/all-tools.integration.test.ts +321 -75
- package/src/__tests__/config.unit.test.ts +33 -0
- package/src/__tests__/docker.integration.test.ts +91 -7
- package/src/__tests__/documentation-fetch-security.integration.test.ts +8 -2
- package/src/__tests__/feature-map-tool-drift.unit.test.ts +93 -0
- package/src/__tests__/http-security-origin.unit.test.ts +97 -0
- package/src/__tests__/http-security-sessions.unit.test.ts +209 -0
- package/src/__tests__/http-security-verifier.unit.test.ts +248 -0
- package/src/__tests__/http-server-allocation.integration.test.ts +279 -0
- package/src/__tests__/http-server-origin.integration.test.ts +169 -0
- package/src/__tests__/http-server-session-auth.integration.test.ts +341 -0
- package/src/__tests__/http-transport.integration.test.ts +85 -87
- package/src/__tests__/integration.integration.test.ts +7 -11
- package/src/__tests__/prompt-response-types.unit.test.ts +51 -0
- package/src/__tests__/prompts.integration.test.ts +736 -0
- package/src/__tests__/run-plan-tools.unit.test.ts +565 -0
- package/src/__tests__/scenario-tools.unit.test.ts +19 -0
- package/src/__tests__/simulation-run-tools.unit.test.ts +116 -0
- package/src/__tests__/support/http-server-harness.ts +138 -0
- package/src/__tests__/test-suite-tools.unit.test.ts +615 -0
- package/src/__tests__/{tools.unit.test.ts → tools.integration.test.ts} +31 -59
- package/src/config.ts +17 -2
- package/src/create-mcp-server.ts +334 -55
- package/src/http-security.ts +492 -0
- package/src/http-server.ts +536 -152
- package/src/index.ts +49 -2
- package/src/langwatch-api-agents.ts +196 -12
- package/src/langwatch-api-run-plans.ts +157 -0
- package/src/langwatch-api-scenarios.ts +9 -0
- package/src/langwatch-api-simulation-runs.ts +5 -0
- package/src/langwatch-api-test-suites.ts +135 -0
- package/src/langwatch-api.ts +62 -12
- package/src/schemas/index.ts +5 -0
- package/src/schemas/run-plan.ts +92 -0
- package/src/schemas/suite-fields.ts +149 -0
- package/src/tools/archive-run-plan.ts +12 -0
- package/src/tools/archive-test-suite.ts +12 -0
- package/src/tools/create-scenario.ts +8 -0
- package/src/tools/create-test-suite.ts +43 -0
- package/src/tools/discover-evaluator-schema.ts +2 -2
- package/src/tools/discover-scenario-schema.ts +54 -0
- package/src/tools/format-run-plan.ts +68 -0
- package/src/tools/format-scenario.ts +14 -0
- package/src/tools/format-suite-details.ts +96 -0
- package/src/tools/get-agent.ts +45 -3
- package/src/tools/get-prompt.ts +108 -24
- package/src/tools/get-run-plan.ts +62 -0
- package/src/tools/get-scenario.ts +3 -0
- package/src/tools/get-simulation-run.ts +9 -0
- package/src/tools/get-test-suite.ts +49 -0
- package/src/tools/list-agents.ts +20 -3
- package/src/tools/list-run-plans.ts +47 -0
- package/src/tools/list-scenarios.ts +12 -2
- package/src/tools/list-simulation-runs.ts +6 -2
- package/src/tools/list-test-suites.ts +35 -0
- package/src/tools/rename-test-suite.ts +18 -0
- package/src/tools/rerun-run-plan.ts +21 -0
- package/src/tools/run-agent.ts +54 -6
- package/src/tools/run-plan.ts +49 -0
- package/src/tools/run-test-suite.ts +30 -0
- package/src/tools/test-agent.ts +18 -0
- package/src/tools/update-prompt.ts +195 -10
- package/src/tools/update-scenario.ts +8 -0
- package/src/tools/update-test-suite.ts +45 -0
- package/tsconfig.json +7 -2
- package/uv.lock +418 -298
- package/dist/archive-suite-C3KX6IIZ.js +0 -16
- package/dist/chunk-WDH5P5RF.js +0 -48
- package/dist/create-suite-TWF6BBR4.js +0 -39
- package/dist/get-agent-7QJVPH72.js +0 -29
- package/dist/get-prompt-I753LZKZ.js +0 -46
- package/dist/get-suite-GTOCY5PL.js +0 -45
- package/dist/langwatch-api-workflows-5HRZ7HR4.js +0 -16
- package/dist/list-agents-TIJTP566.js +0 -33
- package/dist/list-suites-5VK4HXEK.js +0 -43
- package/dist/report-issue-DXJ6TPTC.js +0 -412
- package/dist/run-agent-RBSIFKRT.js +0 -37
- package/dist/run-suite-K3DNQ3SE.js +0 -30
- package/dist/update-prompt-RQFTEO7Y.js +0 -25
- package/dist/update-suite-TRGGLNR7.js +0 -40
- package/pnpm-workspace.yaml +0 -34
- package/src/langwatch-api-suites.ts +0 -118
- package/src/tools/archive-suite.ts +0 -12
- package/src/tools/create-suite.ts +0 -42
- package/src/tools/get-suite.ts +0 -47
- package/src/tools/list-suites.ts +0 -43
- package/src/tools/run-suite.ts +0 -28
- package/src/tools/update-suite.ts +0 -44
package/src/langwatch-api.ts
CHANGED
|
@@ -82,25 +82,48 @@ export interface PromptSummary {
|
|
|
82
82
|
version?: number;
|
|
83
83
|
}
|
|
84
84
|
|
|
85
|
+
export interface PromptTag {
|
|
86
|
+
name: string;
|
|
87
|
+
versionId: string;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export type PromptFieldList = Array<{ identifier: string; type: string }>;
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* A single versioned prompt payload. `GET /api/prompts/:id` returns the
|
|
94
|
+
* requested version's data flattened to the top level (merged with the base
|
|
95
|
+
* prompt data), and `GET /api/prompts/:id/versions` returns an array of
|
|
96
|
+
* entries in this same shape — there is no nested `versions` array on the
|
|
97
|
+
* detail response.
|
|
98
|
+
*/
|
|
85
99
|
export interface PromptVersion {
|
|
100
|
+
versionId?: string;
|
|
86
101
|
version?: number;
|
|
87
|
-
commitMessage?: string;
|
|
102
|
+
commitMessage?: string | null;
|
|
88
103
|
model?: string;
|
|
104
|
+
/** Legacy single-text prompt body. */
|
|
105
|
+
prompt?: string;
|
|
89
106
|
messages?: Array<{ role: string; content: string }>;
|
|
107
|
+
temperature?: number;
|
|
108
|
+
maxTokens?: number;
|
|
109
|
+
responseFormat?: Record<string, unknown> | null;
|
|
110
|
+
/** Runtime parameters: an object map of name -> JSON value, not an array. */
|
|
111
|
+
parameters?: Record<string, unknown>;
|
|
112
|
+
inputs?: PromptFieldList;
|
|
113
|
+
outputs?: PromptFieldList;
|
|
114
|
+
/** All tags on the prompt; each names the version it currently points to. */
|
|
115
|
+
tags?: PromptTag[];
|
|
90
116
|
}
|
|
91
117
|
|
|
92
|
-
export interface PromptDetailResponse extends PromptSummary {
|
|
93
|
-
versions?: PromptVersion[];
|
|
94
|
-
model?: string;
|
|
95
|
-
messages?: Array<{ role: string; content: string }>;
|
|
96
|
-
prompt?: Array<{ role: string; content: string }>;
|
|
97
|
-
}
|
|
118
|
+
export interface PromptDetailResponse extends PromptSummary, PromptVersion {}
|
|
98
119
|
|
|
99
120
|
export interface PromptMutationResponse {
|
|
100
121
|
id?: string;
|
|
101
122
|
handle?: string;
|
|
102
123
|
name?: string;
|
|
103
124
|
latestVersionNumber?: number;
|
|
125
|
+
versionId?: string;
|
|
126
|
+
tags?: string[];
|
|
104
127
|
}
|
|
105
128
|
|
|
106
129
|
// --- HTTP client ---
|
|
@@ -153,8 +176,10 @@ interface ParsedErrorBody {
|
|
|
153
176
|
const VALID_FAULTS: readonly HandledErrorFault[] = ["customer", "platform", "provider"];
|
|
154
177
|
|
|
155
178
|
/**
|
|
156
|
-
* Parses an error response body as a handled-error envelope. Accepts
|
|
157
|
-
*
|
|
179
|
+
* Parses an error response body as a handled-error envelope. Accepts three
|
|
180
|
+
* shapes: the canonical v1 envelope
|
|
181
|
+
* (`{ error: { type, code, message, meta?, trace_id } }`), the legacy REST
|
|
182
|
+
* shape (`{ error: "<code>", message, tips?, docsUrl?, fault? }`) and the
|
|
158
183
|
* serialized tRPC shape (`{ code, message?, tips?, docsUrl?, fault?, ... }`).
|
|
159
184
|
* Returns an empty object when the body is not a recognizable error envelope.
|
|
160
185
|
*/
|
|
@@ -164,7 +189,16 @@ function parseErrorBody(responseBody: string): ParsedErrorBody {
|
|
|
164
189
|
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
165
190
|
return {};
|
|
166
191
|
}
|
|
167
|
-
const
|
|
192
|
+
const envelope = parsed as Record<string, unknown>;
|
|
193
|
+
// The canonical v1 envelope nests everything under `error`. Read the
|
|
194
|
+
// nested object as the body so one parser serves both generations.
|
|
195
|
+
const nested =
|
|
196
|
+
typeof envelope.error === "object" &&
|
|
197
|
+
envelope.error !== null &&
|
|
198
|
+
!Array.isArray(envelope.error)
|
|
199
|
+
? (envelope.error as Record<string, unknown>)
|
|
200
|
+
: null;
|
|
201
|
+
const body = nested ?? envelope;
|
|
168
202
|
// Prefer `code` (the domain discriminant) over `error` — the
|
|
169
203
|
// packages/api unversioned envelope uses `error` for the HTTP status
|
|
170
204
|
// text ("Not Found") while `code` holds the real code.
|
|
@@ -294,14 +328,20 @@ export async function makeRequest(
|
|
|
294
328
|
return response.json();
|
|
295
329
|
}
|
|
296
330
|
|
|
297
|
-
/**
|
|
331
|
+
/**
|
|
332
|
+
* Searches traces with optional filters and pagination.
|
|
333
|
+
*
|
|
334
|
+
* Paging is by `scrollId` only: pass the one the previous response returned,
|
|
335
|
+
* and stop when a response carries none. `pageOffset` used to be accepted here
|
|
336
|
+
* but was never read by the server, so an offset walk silently re-served the
|
|
337
|
+
* first page; it is rejected at the boundary now (#6808).
|
|
338
|
+
*/
|
|
298
339
|
export async function searchTraces(params: {
|
|
299
340
|
query?: string;
|
|
300
341
|
filters?: Record<string, string[]>;
|
|
301
342
|
startDate: number;
|
|
302
343
|
endDate: number;
|
|
303
344
|
pageSize?: number;
|
|
304
|
-
pageOffset?: number;
|
|
305
345
|
scrollId?: string;
|
|
306
346
|
format?: "digest" | "json";
|
|
307
347
|
}): Promise<SearchTracesResponse> {
|
|
@@ -365,6 +405,16 @@ export async function getPrompt(
|
|
|
365
405
|
) as Promise<PromptDetailResponse>;
|
|
366
406
|
}
|
|
367
407
|
|
|
408
|
+
/** Lists all versions of a prompt (versioned data only). */
|
|
409
|
+
export async function getPromptVersions(
|
|
410
|
+
idOrHandle: string
|
|
411
|
+
): Promise<PromptVersion[]> {
|
|
412
|
+
return makeRequest(
|
|
413
|
+
"GET",
|
|
414
|
+
`/api/prompts/${encodeURIComponent(idOrHandle)}/versions`
|
|
415
|
+
) as Promise<PromptVersion[]>;
|
|
416
|
+
}
|
|
417
|
+
|
|
368
418
|
/** Creates a new prompt. */
|
|
369
419
|
export async function createPrompt(data: {
|
|
370
420
|
handle: string;
|
package/src/schemas/index.ts
CHANGED
|
@@ -2,3 +2,8 @@ export { filterFields, type FilterFieldInfo } from "./filter-fields.js";
|
|
|
2
2
|
export { analyticsMetrics, type MetricInfo } from "./analytics-metrics.js";
|
|
3
3
|
export { analyticsGroups, type GroupByInfo } from "./analytics-groups.js";
|
|
4
4
|
export { createDatasetSchema } from "./create-dataset.js";
|
|
5
|
+
export {
|
|
6
|
+
runParametersSchema,
|
|
7
|
+
runPlanScopeSchema,
|
|
8
|
+
runPlanTargetSchema,
|
|
9
|
+
} from "./run-plan.js";
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Typed inputs for the run plan tools.
|
|
5
|
+
*
|
|
6
|
+
* Each one mirrors the zod the platform validates the REST body with, so an
|
|
7
|
+
* agent is told what is accepted before the request goes out rather than
|
|
8
|
+
* after: `suiteScopeSchema` and `suiteTargetSchema` on the server side.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
/** What a run plan covers. */
|
|
12
|
+
export const runPlanScopeSchema = z.discriminatedUnion("mode", [
|
|
13
|
+
z
|
|
14
|
+
.object({ mode: z.literal("all") })
|
|
15
|
+
.describe("Every non-archived scenario of the project"),
|
|
16
|
+
z
|
|
17
|
+
.object({
|
|
18
|
+
mode: z.literal("test_suites"),
|
|
19
|
+
testSuiteIds: z
|
|
20
|
+
.array(z.string())
|
|
21
|
+
.describe("The test suite IDs whose scenarios to run"),
|
|
22
|
+
})
|
|
23
|
+
.describe("The non-archived scenarios filed in the given test suites"),
|
|
24
|
+
z
|
|
25
|
+
.object({
|
|
26
|
+
mode: z.literal("labels"),
|
|
27
|
+
labels: z.array(z.string()).describe("The labels to select scenarios by"),
|
|
28
|
+
})
|
|
29
|
+
.describe(
|
|
30
|
+
"The non-archived scenarios carrying at least one of the given labels",
|
|
31
|
+
),
|
|
32
|
+
z
|
|
33
|
+
.object({ mode: z.literal("scenarios") })
|
|
34
|
+
.describe("The scenarios named in scenarioIds, picked by hand"),
|
|
35
|
+
]);
|
|
36
|
+
|
|
37
|
+
/** The values a run supplies for the parameters its scenarios declare. */
|
|
38
|
+
export const runParametersSchema = z.record(
|
|
39
|
+
z.string(),
|
|
40
|
+
z.union([z.string(), z.number(), z.boolean()]),
|
|
41
|
+
);
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* One thing a plan runs its scenarios against, and the values that target
|
|
45
|
+
* alone runs with.
|
|
46
|
+
*
|
|
47
|
+
* The parameters are what makes a comparison run: two targets may name the
|
|
48
|
+
* same agent with different values, and the platform runs and reports each one
|
|
49
|
+
* on its own.
|
|
50
|
+
*/
|
|
51
|
+
export const runPlanTargetSchema = z.object({
|
|
52
|
+
type: z
|
|
53
|
+
.enum(["connected", "prompt", "http", "code", "workflow"])
|
|
54
|
+
.describe(
|
|
55
|
+
"What kind of thing the target is. A connected agent is one that registered itself from code with connectAgent (TypeScript) or connect_agent (Python).",
|
|
56
|
+
),
|
|
57
|
+
referenceId: z
|
|
58
|
+
.string()
|
|
59
|
+
.describe(
|
|
60
|
+
"The ID of the prompt, agent or workflow to run against. A connected agent may also be named as <name>@<environment>, for example support-agent@production; the platform resolves it.",
|
|
61
|
+
),
|
|
62
|
+
parameters: runParametersSchema
|
|
63
|
+
.optional()
|
|
64
|
+
.describe(
|
|
65
|
+
"Parameter values this target alone runs with, by name. They override the run-level parameters for this target. Name the same agent twice with different values here to compare it on two models.",
|
|
66
|
+
),
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* One target as `/api/v1` carries it.
|
|
71
|
+
*
|
|
72
|
+
* The tool input spells the per-target values `parameters`, beside the
|
|
73
|
+
* run-level `parameters` an agent already knows. The REST body spells them
|
|
74
|
+
* `runParameters`, because a target holds the values it runs with rather than
|
|
75
|
+
* the values of a run.
|
|
76
|
+
*/
|
|
77
|
+
export interface RunPlanTargetWire {
|
|
78
|
+
type: z.infer<typeof runPlanTargetSchema>["type"];
|
|
79
|
+
referenceId: string;
|
|
80
|
+
runParameters?: z.infer<typeof runParametersSchema>;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** The targets of a tool call, as the REST body carries them. */
|
|
84
|
+
export function toWireTargets(
|
|
85
|
+
targets: z.infer<typeof runPlanTargetSchema>[],
|
|
86
|
+
): RunPlanTargetWire[] {
|
|
87
|
+
return targets.map((target) => ({
|
|
88
|
+
type: target.type,
|
|
89
|
+
referenceId: target.referenceId,
|
|
90
|
+
...(target.parameters ? { runParameters: target.parameters } : {}),
|
|
91
|
+
}));
|
|
92
|
+
}
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Typed inputs for the fields a test suite declares, the evaluators attached
|
|
5
|
+
* to a suite or a run plan, and the values a scenario carries per field.
|
|
6
|
+
*
|
|
7
|
+
* Each one mirrors the zod the platform validates the REST body with
|
|
8
|
+
* (`suiteFieldDefinitionsSchema`, `evaluatorAttachmentsSchema` and
|
|
9
|
+
* `scenarioFieldValuesSchema` on the server side), so an agent is told what
|
|
10
|
+
* is accepted before the request goes out rather than after.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
/** The value types a field can hold. */
|
|
14
|
+
export const SUITE_FIELD_TYPES = ["text", "number", "boolean"] as const;
|
|
15
|
+
|
|
16
|
+
/** One field a test suite declares. */
|
|
17
|
+
export const suiteFieldSchema = z.object({
|
|
18
|
+
identifier: z
|
|
19
|
+
.string()
|
|
20
|
+
.min(1)
|
|
21
|
+
.max(64)
|
|
22
|
+
.regex(
|
|
23
|
+
/^[a-z][a-z0-9_]*$/,
|
|
24
|
+
"Lowercase letters, digits and underscores, starting with a letter",
|
|
25
|
+
)
|
|
26
|
+
.describe(
|
|
27
|
+
"The field name, as scenarios and evaluator mappings address it, for example golden_sql. Lowercase letters, digits and underscores, starting with a letter. situation, criteria, name, input and output are reserved.",
|
|
28
|
+
),
|
|
29
|
+
type: z
|
|
30
|
+
.enum(SUITE_FIELD_TYPES)
|
|
31
|
+
.describe("The value type every scenario carries for this field."),
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
/** The fields a test suite declares, in the order the platform shows them. */
|
|
35
|
+
export const suiteFieldsSchema = z
|
|
36
|
+
.array(suiteFieldSchema)
|
|
37
|
+
.max(30)
|
|
38
|
+
.describe(
|
|
39
|
+
"The fields the test suite declares, in order. Up to 30. Every scenario filed in the suite carries one value per field, and an evaluator attached to the suite reads them through its mappings.",
|
|
40
|
+
);
|
|
41
|
+
|
|
42
|
+
/** The sources an evaluator input can read from. */
|
|
43
|
+
export const SCENARIO_MAPPING_SOURCE_IDS = [
|
|
44
|
+
"conversation",
|
|
45
|
+
"scenario",
|
|
46
|
+
"trace",
|
|
47
|
+
] as const;
|
|
48
|
+
|
|
49
|
+
/** Where one evaluator input reads its value: a source path or a literal. */
|
|
50
|
+
export const scenarioMappingSchema = z
|
|
51
|
+
.discriminatedUnion("type", [
|
|
52
|
+
z.object({
|
|
53
|
+
type: z.literal("source"),
|
|
54
|
+
sourceId: z
|
|
55
|
+
.enum(SCENARIO_MAPPING_SOURCE_IDS)
|
|
56
|
+
.describe(
|
|
57
|
+
"conversation, scenario or trace. See discover_schema({ category: 'scenarios' }) for the paths each one offers.",
|
|
58
|
+
),
|
|
59
|
+
path: z
|
|
60
|
+
.array(z.string().min(1).max(256))
|
|
61
|
+
.min(1)
|
|
62
|
+
.max(3)
|
|
63
|
+
.describe(
|
|
64
|
+
"The path inside the source. conversation: [first_user_message] | [last_agent_message] | [transcript] | [messages]. scenario: [situation] | [criteria] | [fields, <identifier>]. trace: [contexts] | [tool_calls, <toolName>, input|output].",
|
|
65
|
+
),
|
|
66
|
+
}),
|
|
67
|
+
z.object({
|
|
68
|
+
type: z.literal("value"),
|
|
69
|
+
value: z.string().max(65_536).describe("A literal the input reads."),
|
|
70
|
+
}),
|
|
71
|
+
])
|
|
72
|
+
.describe(
|
|
73
|
+
"Where one evaluator input reads its value: a path into the conversation, the scenario or the trace, or a literal value.",
|
|
74
|
+
);
|
|
75
|
+
|
|
76
|
+
/** One evaluator attached to a test suite or a run plan. */
|
|
77
|
+
export const evaluatorAttachmentSchema = z.object({
|
|
78
|
+
id: z
|
|
79
|
+
.string()
|
|
80
|
+
.min(1)
|
|
81
|
+
.max(64)
|
|
82
|
+
.optional()
|
|
83
|
+
.describe(
|
|
84
|
+
"The attachment id, stable across edits. Leave it out on a new attachment and one is generated.",
|
|
85
|
+
),
|
|
86
|
+
evaluatorId: z
|
|
87
|
+
.string()
|
|
88
|
+
.min(1)
|
|
89
|
+
.describe(
|
|
90
|
+
"The id of the saved evaluator to run. Read it from platform_list_evaluators or platform_get_evaluator.",
|
|
91
|
+
),
|
|
92
|
+
required: z
|
|
93
|
+
.boolean()
|
|
94
|
+
.optional()
|
|
95
|
+
.describe(
|
|
96
|
+
"Whether a failing result fails the scenario. Defaults to true. A score-only evaluator reports and never gates, whatever this says.",
|
|
97
|
+
),
|
|
98
|
+
mappings: z
|
|
99
|
+
.record(z.string().min(1).max(128), scenarioMappingSchema)
|
|
100
|
+
.describe(
|
|
101
|
+
"Where each evaluator input reads its value, keyed by input name. Read the inputs from platform_get_evaluator. A required input with no mapping makes the platform refuse the run until it is set.",
|
|
102
|
+
),
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
/** The evaluators attached to a test suite or a run plan. */
|
|
106
|
+
export const evaluatorAttachmentsSchema = z
|
|
107
|
+
.array(evaluatorAttachmentSchema)
|
|
108
|
+
.max(20)
|
|
109
|
+
.describe(
|
|
110
|
+
"The evaluators that run after every scenario run, with their mappings. Up to 20. A required evaluator that fails fails the scenario; a score-only one reports beside the verdict.",
|
|
111
|
+
);
|
|
112
|
+
|
|
113
|
+
/** The values a scenario carries for the fields its suite declares. */
|
|
114
|
+
export const scenarioFieldValuesSchema = z
|
|
115
|
+
.record(z.string(), z.union([z.string(), z.number(), z.boolean()]))
|
|
116
|
+
.describe(
|
|
117
|
+
"The scenario's value for each field its test suite declares, by identifier, for example { golden_sql: 'SELECT ...' }. A text field takes a string, a number field a number and a boolean field true or false. A field the suite does not declare is refused. On update, an empty object clears the values.",
|
|
118
|
+
);
|
|
119
|
+
|
|
120
|
+
export type SuiteField = z.infer<typeof suiteFieldSchema>;
|
|
121
|
+
export type ScenarioMapping = z.infer<typeof scenarioMappingSchema>;
|
|
122
|
+
export type EvaluatorAttachmentInput = z.infer<typeof evaluatorAttachmentSchema>;
|
|
123
|
+
export type ScenarioFieldValues = z.infer<typeof scenarioFieldValuesSchema>;
|
|
124
|
+
|
|
125
|
+
/** One attachment as the REST surface carries it: id and gate settled. */
|
|
126
|
+
export interface EvaluatorAttachmentWire {
|
|
127
|
+
id: string;
|
|
128
|
+
evaluatorId: string;
|
|
129
|
+
required: boolean;
|
|
130
|
+
mappings: Record<string, ScenarioMapping>;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
const newAttachmentId = (): string =>
|
|
134
|
+
`att_${crypto.randomUUID().replace(/-/g, "").slice(0, 21)}`;
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* The attachments of a tool call, as the REST body carries them: a missing
|
|
138
|
+
* id is generated and a missing gate defaults to required.
|
|
139
|
+
*/
|
|
140
|
+
export function toWireAttachments(
|
|
141
|
+
attachments: EvaluatorAttachmentInput[],
|
|
142
|
+
): EvaluatorAttachmentWire[] {
|
|
143
|
+
return attachments.map((attachment) => ({
|
|
144
|
+
id: attachment.id ?? newAttachmentId(),
|
|
145
|
+
evaluatorId: attachment.evaluatorId,
|
|
146
|
+
required: attachment.required ?? true,
|
|
147
|
+
mappings: attachment.mappings,
|
|
148
|
+
}));
|
|
149
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { archiveRunPlan as apiArchiveRunPlan } from "../langwatch-api-run-plans.js";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Handles the platform_archive_run_plan MCP tool invocation.
|
|
5
|
+
*/
|
|
6
|
+
export async function handleArchiveRunPlan(params: {
|
|
7
|
+
id: string;
|
|
8
|
+
}): Promise<string> {
|
|
9
|
+
const result = await apiArchiveRunPlan(params.id);
|
|
10
|
+
|
|
11
|
+
return `Run plan ${result.id} is archived. Its past runs stay readable, and the name is free for a new plan.`;
|
|
12
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { archiveTestSuite as apiArchiveTestSuite } from "../langwatch-api-test-suites.js";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Handles the platform_archive_test_suite MCP tool invocation.
|
|
5
|
+
*/
|
|
6
|
+
export async function handleArchiveTestSuite(params: {
|
|
7
|
+
id: string;
|
|
8
|
+
}): Promise<string> {
|
|
9
|
+
const result = await apiArchiveTestSuite(params.id);
|
|
10
|
+
|
|
11
|
+
return `Test suite ${result.id} is archived, and the scenarios filed in it are archived with it.`;
|
|
12
|
+
}
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { createScenario as apiCreateScenario } from "../langwatch-api-scenarios.js";
|
|
2
|
+
import type { ScenarioFieldValues } from "../schemas/suite-fields.js";
|
|
3
|
+
import { formatScenarioFields } from "./format-scenario.js";
|
|
2
4
|
|
|
3
5
|
/**
|
|
4
6
|
* Handles the platform_create_scenario MCP tool invocation.
|
|
@@ -11,6 +13,8 @@ export async function handleCreateScenario(params: {
|
|
|
11
13
|
situation: string;
|
|
12
14
|
criteria?: string[];
|
|
13
15
|
labels?: string[];
|
|
16
|
+
testSuiteId?: string | null;
|
|
17
|
+
fields?: ScenarioFieldValues;
|
|
14
18
|
}): Promise<string> {
|
|
15
19
|
const result = await apiCreateScenario(params);
|
|
16
20
|
|
|
@@ -25,6 +29,10 @@ export async function handleCreateScenario(params: {
|
|
|
25
29
|
if (Array.isArray(result.labels) && result.labels.length > 0) {
|
|
26
30
|
lines.push(`**Labels**: ${result.labels.join(", ")}`);
|
|
27
31
|
}
|
|
32
|
+
if (result.testSuiteId) {
|
|
33
|
+
lines.push(`**Test suite**: ${result.testSuiteId}`);
|
|
34
|
+
}
|
|
35
|
+
lines.push(...formatScenarioFields(result.fields));
|
|
28
36
|
|
|
29
37
|
return lines.join("\n");
|
|
30
38
|
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { createTestSuite as apiCreateTestSuite } from "../langwatch-api-test-suites.js";
|
|
2
|
+
import {
|
|
3
|
+
type EvaluatorAttachmentInput,
|
|
4
|
+
type SuiteField,
|
|
5
|
+
toWireAttachments,
|
|
6
|
+
} from "../schemas/suite-fields.js";
|
|
7
|
+
import {
|
|
8
|
+
formatEvaluatorAttachments,
|
|
9
|
+
formatSuiteFields,
|
|
10
|
+
} from "./format-suite-details.js";
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Handles the platform_create_test_suite MCP tool invocation.
|
|
14
|
+
*/
|
|
15
|
+
export async function handleCreateTestSuite(params: {
|
|
16
|
+
name: string;
|
|
17
|
+
fields?: SuiteField[];
|
|
18
|
+
evaluators?: EvaluatorAttachmentInput[];
|
|
19
|
+
}): Promise<string> {
|
|
20
|
+
const suite = await apiCreateTestSuite({
|
|
21
|
+
name: params.name,
|
|
22
|
+
...(params.fields !== undefined ? { fields: params.fields } : {}),
|
|
23
|
+
...(params.evaluators !== undefined
|
|
24
|
+
? { evaluators: toWireAttachments(params.evaluators) }
|
|
25
|
+
: {}),
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
return [
|
|
29
|
+
`Test suite "${suite.name}" created.`,
|
|
30
|
+
"",
|
|
31
|
+
`**ID**: ${suite.id}`,
|
|
32
|
+
`**Slug**: ${suite.slug}`,
|
|
33
|
+
`**View**: ${suite.platformUrl}`,
|
|
34
|
+
...formatSuiteFields(suite.fields),
|
|
35
|
+
...formatEvaluatorAttachments(suite.evaluators),
|
|
36
|
+
"",
|
|
37
|
+
`> File scenarios in it with \`platform_create_scenario\` or \`platform_update_scenario\`, passing testSuiteId \`${suite.id}\`${
|
|
38
|
+
suite.fields && suite.fields.length > 0
|
|
39
|
+
? ` and a value per field under \`fields\``
|
|
40
|
+
: ""
|
|
41
|
+
}.`,
|
|
42
|
+
].join("\n");
|
|
43
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { AVAILABLE_EVALUATORS } from "
|
|
2
|
-
import type { EvaluatorDefinition, EvaluatorTypes } from "
|
|
1
|
+
import { AVAILABLE_EVALUATORS } from "../../../../services/langevals/ts-integration/evaluators.generated.js";
|
|
2
|
+
import type { EvaluatorDefinition, EvaluatorTypes } from "../../../../services/langevals/ts-integration/evaluators.generated.js";
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* Formats evaluator schema information for the discover_schema tool.
|
|
@@ -20,6 +20,60 @@ export function formatScenarioSchema(): string {
|
|
|
20
20
|
lines.push(
|
|
21
21
|
'- **labels** (array of strings): Tags for organizing scenarios (e.g., "auth", "happy-path", "edge-case")',
|
|
22
22
|
);
|
|
23
|
+
lines.push(
|
|
24
|
+
"- **testSuiteId** (string): The test suite the scenario is filed in. Every project has a Default suite.",
|
|
25
|
+
);
|
|
26
|
+
lines.push(
|
|
27
|
+
'- **fields** (object): The scenario\'s value for each field its test suite declares, by identifier (e.g. { "golden_sql": "SELECT ..." }). A text field takes a string, a number field a number, a boolean field true or false. A field the suite does not declare is refused. A blank value skips the evaluators that read it.',
|
|
28
|
+
);
|
|
29
|
+
|
|
30
|
+
lines.push("\n## Test Suite Fields\n");
|
|
31
|
+
lines.push(
|
|
32
|
+
"A test suite reads like a dataset: it declares typed fields beyond the situation and the criteria, and every scenario filed in it carries one value per field. Declare them with `platform_create_test_suite` or `platform_update_test_suite`:",
|
|
33
|
+
);
|
|
34
|
+
lines.push("```json");
|
|
35
|
+
lines.push('{ "fields": [{ "identifier": "golden_sql", "type": "text" }, { "identifier": "row_limit", "type": "number" }] }');
|
|
36
|
+
lines.push("```");
|
|
37
|
+
lines.push(
|
|
38
|
+
"- **identifier**: lowercase letters, digits and underscores, starting with a letter. situation, criteria, name, input and output are reserved.",
|
|
39
|
+
);
|
|
40
|
+
lines.push("- **type**: text, number or boolean.");
|
|
41
|
+
lines.push(
|
|
42
|
+
"- On update, the list replaces the one the suite holds. A field an attached evaluator still reads cannot be removed.",
|
|
43
|
+
);
|
|
44
|
+
|
|
45
|
+
lines.push("\n## Evaluator Attachments\n");
|
|
46
|
+
lines.push(
|
|
47
|
+
"A saved evaluator attaches to a test suite (and, for checks that read only the conversation or the trace, to a run plan). After every scenario run the platform runs it and stores one result per evaluator on the run. A **required** pass/fail evaluator that fails fails the scenario; a score-only evaluator reports beside the verdict and never gates.",
|
|
48
|
+
);
|
|
49
|
+
lines.push("```json");
|
|
50
|
+
lines.push('{ "evaluators": [{');
|
|
51
|
+
lines.push(' "evaluatorId": "<id from platform_list_evaluators>",');
|
|
52
|
+
lines.push(' "required": true,');
|
|
53
|
+
lines.push(' "mappings": {');
|
|
54
|
+
lines.push(' "output": { "type": "source", "sourceId": "trace", "path": ["tool_calls", "run_sql", "input"] },');
|
|
55
|
+
lines.push(' "expected_output": { "type": "source", "sourceId": "scenario", "path": ["fields", "golden_sql"] }');
|
|
56
|
+
lines.push(" }");
|
|
57
|
+
lines.push("}] }");
|
|
58
|
+
lines.push("```");
|
|
59
|
+
lines.push(
|
|
60
|
+
"Each mapping names where one evaluator input reads its value. Read the evaluator's inputs from `platform_get_evaluator`; every required input needs a mapping before a run. The paths:",
|
|
61
|
+
);
|
|
62
|
+
lines.push(
|
|
63
|
+
"- **conversation**: `[\"first_user_message\"]`, `[\"last_agent_message\"]`, `[\"transcript\"]` (role: content lines) or `[\"messages\"]` (JSON)",
|
|
64
|
+
);
|
|
65
|
+
lines.push(
|
|
66
|
+
"- **scenario**: `[\"situation\"]`, `[\"criteria\"]` or `[\"fields\", \"<identifier>\"]` (a field the suite declares; not on a run plan)",
|
|
67
|
+
);
|
|
68
|
+
lines.push(
|
|
69
|
+
"- **trace**: `[\"contexts\"]` (the retrieved contexts of the run's traces) or `[\"tool_calls\", \"<toolName>\", \"input\" | \"output\"]` (the last matching tool call)",
|
|
70
|
+
);
|
|
71
|
+
lines.push(
|
|
72
|
+
'- a literal: `{ "type": "value", "value": "..." }`',
|
|
73
|
+
);
|
|
74
|
+
lines.push(
|
|
75
|
+
"A scenario whose field is blank skips the evaluators that read it, with a reason. A trace with no such tool call or no contexts fails them, with a reason. The results come back under `results.evaluations` on `platform_get_simulation_run`.",
|
|
76
|
+
);
|
|
23
77
|
|
|
24
78
|
lines.push("\n## Writing a Good Situation\n");
|
|
25
79
|
lines.push(
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
RunPlanRunResult,
|
|
3
|
+
RunPlanScope,
|
|
4
|
+
} from "../langwatch-api-run-plans.js";
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* The digest every run of a plan returns, whichever tool started it.
|
|
8
|
+
*
|
|
9
|
+
* It says which plan ran and whether the run created it or joined one that
|
|
10
|
+
* already carried the name, because that is the difference between a new plan
|
|
11
|
+
* and a plan whose configuration this run just replaced.
|
|
12
|
+
*/
|
|
13
|
+
export function formatRunPlanRun(result: RunPlanRunResult): string {
|
|
14
|
+
const lines: string[] = [];
|
|
15
|
+
|
|
16
|
+
lines.push(
|
|
17
|
+
result.created
|
|
18
|
+
? `Run plan "${result.planName}" created and started.`
|
|
19
|
+
: `Run plan "${result.planName}" started with the configuration of this run.`,
|
|
20
|
+
);
|
|
21
|
+
lines.push("");
|
|
22
|
+
lines.push(`**Plan**: ${result.planName} (${result.runPlanId})`);
|
|
23
|
+
lines.push(`**Batch Run ID**: ${result.batchRunId}`);
|
|
24
|
+
lines.push(`**Set ID**: ${result.setId}`);
|
|
25
|
+
lines.push(`**Jobs**: ${result.jobCount}`);
|
|
26
|
+
|
|
27
|
+
if (result.skippedArchived.scenarios.length > 0) {
|
|
28
|
+
lines.push(
|
|
29
|
+
`**Skipped archived scenarios**: ${result.skippedArchived.scenarios.join(", ")}`,
|
|
30
|
+
);
|
|
31
|
+
}
|
|
32
|
+
if (result.skippedArchived.targets.length > 0) {
|
|
33
|
+
lines.push(
|
|
34
|
+
`**Skipped archived targets**: ${result.skippedArchived.targets.join(", ")}`,
|
|
35
|
+
);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
lines.push(`**View**: ${result.platformUrl}`);
|
|
39
|
+
lines.push("");
|
|
40
|
+
lines.push(
|
|
41
|
+
"> Use `platform_list_simulation_runs` with the batch run ID to read the results.",
|
|
42
|
+
);
|
|
43
|
+
|
|
44
|
+
return lines.join("\n");
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* One line saying what a plan covers. A plan stored before scopes existed
|
|
49
|
+
* carries none, and runs the scenario list it already held.
|
|
50
|
+
*/
|
|
51
|
+
export function describeRunPlanScope(
|
|
52
|
+
scope: RunPlanScope | null,
|
|
53
|
+
scenarioIds: string[],
|
|
54
|
+
): string {
|
|
55
|
+
if (scope === null) {
|
|
56
|
+
return `hand-picked scenarios (${scenarioIds.length})`;
|
|
57
|
+
}
|
|
58
|
+
switch (scope.mode) {
|
|
59
|
+
case "all":
|
|
60
|
+
return "every scenario in the project";
|
|
61
|
+
case "test_suites":
|
|
62
|
+
return `test suites: ${scope.testSuiteIds.join(", ")}`;
|
|
63
|
+
case "labels":
|
|
64
|
+
return `labels: ${scope.labels.join(", ")}`;
|
|
65
|
+
case "scenarios":
|
|
66
|
+
return `hand-picked scenarios (${scenarioIds.length})`;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { ScenarioFieldValues } from "../schemas/suite-fields.js";
|
|
2
|
+
|
|
3
|
+
/** The digest lines for the values a scenario carries per suite field. */
|
|
4
|
+
export function formatScenarioFields(
|
|
5
|
+
fields: ScenarioFieldValues | undefined,
|
|
6
|
+
): string[] {
|
|
7
|
+
if (!fields || Object.keys(fields).length === 0) return [];
|
|
8
|
+
return [
|
|
9
|
+
"\n## Fields",
|
|
10
|
+
...Object.entries(fields).map(
|
|
11
|
+
([identifier, value]) => `- ${identifier}: ${JSON.stringify(value)}`,
|
|
12
|
+
),
|
|
13
|
+
];
|
|
14
|
+
}
|