@langwatch/mcp-server 1.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.python-version +1 -0
- package/CHANGELOG.md +110 -0
- package/CONTRIBUTING.md +1 -1
- package/README.md +86 -4
- package/dist/{archive-project-MK3673A7.js → archive-project-W3IKI7UL.js} +5 -5
- package/dist/archive-run-plan-474OKHHU.js +16 -0
- package/dist/{archive-scenario-KATSXRIN.js → archive-scenario-V5UNYK26.js} +5 -5
- package/dist/archive-test-suite-O7FLDMXK.js +16 -0
- package/dist/{assign-prompt-tag-E3C5W6PI.js → assign-prompt-tag-Q46WRB6P.js} +4 -4
- package/dist/chunk-56OGTS24.js +44 -0
- package/dist/chunk-6DJDEXET.js +63 -0
- package/dist/chunk-7E4FZIYJ.js +15 -0
- package/dist/{chunk-J45SVWPR.js → chunk-AN4OUA4D.js} +2 -2
- package/dist/chunk-CKNAH5TG.js +74 -0
- package/dist/{chunk-CE6QMZOP.js → chunk-EYXMLE5X.js} +86 -11
- package/dist/{chunk-XY4STOBE.js → chunk-G4ZAZTZW.js} +2 -2
- package/dist/{chunk-5UWY24RP.js → chunk-GELJ2MQR.js} +12 -3
- package/dist/chunk-HV4DWKBI.js +46 -0
- package/dist/chunk-JR6DJZOU.js +49 -0
- package/dist/{chunk-DAIQKS27.js → chunk-JTQJRNFJ.js} +2 -2
- package/dist/{chunk-WWMDHZZV.js → chunk-K37ZNPXK.js} +2 -2
- package/dist/{chunk-EU22L52K.js → chunk-OJLYYJWI.js} +2 -2
- package/dist/{chunk-Y3HDZ6OW.js → chunk-QAMP3CGR.js} +2 -2
- package/dist/{chunk-VYPSKRPN.js → chunk-QRO35SOM.js} +347 -146
- package/dist/{chunk-O3LWVN67.js → chunk-RVAE2MEA.js} +2 -2
- package/dist/{chunk-UXXA4D6K.js → chunk-S4GVX36W.js} +2 -2
- package/dist/{chunk-BJ7B4R26.js → chunk-SAS5OB7F.js} +18 -2
- package/dist/{chunk-RYRB2YVJ.js → chunk-SR6QPRIE.js} +2 -2
- package/dist/chunk-UF6XIGYR.js +58 -0
- package/dist/{chunk-T7WJVS2N.js → chunk-UFOYFYI7.js} +2 -2
- package/dist/chunk-UJPXFA26.js +267 -0
- package/dist/{chunk-KOO5733S.js → chunk-WT4PO4A3.js} +8 -2
- package/dist/config.js +2 -2
- package/dist/{create-agent-7W57J7MT.js → create-agent-JFJ2K6X3.js} +5 -5
- package/dist/{create-annotation-W2XELBCQ.js → create-annotation-EUXXQ6NS.js} +5 -5
- package/dist/{create-api-key-MKRXKFJM.js → create-api-key-BGBB76JX.js} +5 -5
- package/dist/{create-dashboard-QGL7IODC.js → create-dashboard-Z76AIPLS.js} +5 -5
- package/dist/{create-dataset-3PH7A4HG.js → create-dataset-FD5VAXU4.js} +5 -5
- package/dist/{create-dataset-records-4DZS3VRL.js → create-dataset-records-WQUXXOHI.js} +5 -5
- package/dist/{create-evaluator-YWH6NC4M.js → create-evaluator-6NV33Z52.js} +5 -5
- package/dist/create-mcp-server.js +9 -4
- package/dist/{create-project-BOBQ32C5.js → create-project-ZQ7TFIO2.js} +5 -5
- package/dist/{create-prompt-FLCFGITE.js → create-prompt-BS6QRCUU.js} +4 -4
- package/dist/{create-prompt-tag-EMPNIKEA.js → create-prompt-tag-FCQ5ELSQ.js} +4 -4
- package/dist/{create-scenario-BSBIHUL6.js → create-scenario-DRPTPXU3.js} +12 -5
- package/dist/create-test-suite-ZIZCCFJ4.js +37 -0
- package/dist/{delete-agent-7MSYNY5E.js → delete-agent-SA4JMUZF.js} +5 -5
- package/dist/{delete-annotation-ZT5IGN24.js → delete-annotation-X3KKK7QY.js} +5 -5
- package/dist/{delete-dashboard-BTIY5S3L.js → delete-dashboard-LYMRPF27.js} +5 -5
- package/dist/{delete-dataset-U45HQHPR.js → delete-dataset-W6EZ44HY.js} +5 -5
- package/dist/{delete-dataset-records-R43IMAWB.js → delete-dataset-records-7CNOTALB.js} +5 -5
- package/dist/{delete-evaluator-U6JMQ6C7.js → delete-evaluator-RKYC4TNB.js} +5 -5
- package/dist/{delete-prompt-tag-4EV4TDHB.js → delete-prompt-tag-QB3OCZUV.js} +4 -4
- package/dist/{delete-workflow-5QYALXEL.js → delete-workflow-NG2VXYYM.js} +5 -5
- package/dist/{discover-evaluator-schema-MRGHVNTT.js → discover-evaluator-schema-ESSHQGPY.js} +58 -321
- package/dist/{discover-scenario-schema-UPBTCWPG.js → discover-scenario-schema-W7KBXA4J.js} +54 -2
- package/dist/{discover-schema-ICGXPHUW.js → discover-schema-L6QDPON3.js} +2 -2
- package/dist/get-agent-7THF7HCU.js +66 -0
- package/dist/{get-analytics-PVERWOA5.js → get-analytics-R6AZD3AW.js} +4 -4
- package/dist/{get-dashboard-ZZPC3WMD.js → get-dashboard-P4WWVHFU.js} +5 -5
- package/dist/{get-dataset-25TB2LQH.js → get-dataset-AKTYSQDI.js} +5 -5
- package/dist/{get-evaluator-3FTRQKEC.js → get-evaluator-RZ44UIN6.js} +5 -5
- package/dist/{get-project-5OAXYC57.js → get-project-QSQEAB33.js} +5 -5
- package/dist/get-prompt-GVKXF5NL.js +100 -0
- package/dist/get-run-plan-NY7J5GRE.js +61 -0
- package/dist/{get-scenario-DSSTR6FI.js → get-scenario-Q3LLQE4T.js} +9 -5
- package/dist/{get-simulation-run-RQPMS2MC.js → get-simulation-run-WINEYDMK.js} +14 -5
- package/dist/get-test-suite-4HKA7OKS.js +49 -0
- package/dist/{get-trace-O5VT3NAC.js → get-trace-RCUV7N4O.js} +4 -4
- package/dist/{get-workflow-3EZDOEIF.js → get-workflow-EEOC7Y45.js} +5 -5
- package/dist/http-security-EU3CHTY7.js +26 -0
- package/dist/{http-server-2ACJ3CXW.js → http-server-WCDUARTH.js} +1038 -693
- package/dist/index.js +6437 -4020
- package/dist/{langwatch-api-annotations-E65MI3WZ.js → langwatch-api-annotations-6DHVZWUK.js} +5 -5
- package/dist/{langwatch-api-dashboards-QCDECRCY.js → langwatch-api-dashboards-2KJL2M42.js} +5 -5
- package/dist/{langwatch-api-monitors-UIO2KTDV.js → langwatch-api-monitors-6ISFJ7WX.js} +4 -4
- package/dist/{langwatch-api-secrets-4T2EKSJ7.js → langwatch-api-secrets-OEBD6GBU.js} +4 -4
- package/dist/{langwatch-api-triggers-MPOEKPUY.js → langwatch-api-triggers-5E6DDGX6.js} +4 -4
- package/dist/langwatch-api-workflows-6UJL5U4Z.js +16 -0
- package/dist/list-agents-AMVDQPQL.js +43 -0
- package/dist/{list-annotations-CGX5BT7E.js → list-annotations-6PR5NLIC.js} +5 -5
- package/dist/{list-api-keys-JLWJPFRX.js → list-api-keys-GVPNBIHQ.js} +5 -5
- package/dist/{list-dashboards-RNIWX3TX.js → list-dashboards-Z3BZKDGF.js} +5 -5
- package/dist/{list-dataset-records-B5MHZR66.js → list-dataset-records-O67HZ35Y.js} +5 -5
- package/dist/{list-datasets-BVCAMQCP.js → list-datasets-WXZL6UMP.js} +5 -5
- package/dist/{list-evaluators-65FBTL2U.js → list-evaluators-PAX72WCN.js} +5 -5
- package/dist/{list-model-providers-GRK3I3VV.js → list-model-providers-VVLZMLCX.js} +5 -5
- package/dist/{list-projects-JLMBFMW3.js → list-projects-AQVI6SUK.js} +5 -5
- package/dist/{list-prompt-tags-UHNWDGQI.js → list-prompt-tags-WZFBJOHG.js} +4 -4
- package/dist/{list-prompts-7254B3U7.js → list-prompts-BQOHRHYT.js} +4 -4
- package/dist/list-run-plans-UBAU2DGH.js +46 -0
- package/dist/{list-scenarios-46YNICKM.js → list-scenarios-MS4IUIVM.js} +10 -7
- package/dist/{list-simulation-runs-LGNEFNPN.js → list-simulation-runs-D6YF4ZOU.js} +13 -7
- package/dist/list-test-suites-L7R2O6DR.js +35 -0
- package/dist/{list-workflows-MLQDM6CP.js → list-workflows-6MV4XJHF.js} +5 -5
- package/dist/{rename-prompt-tag-RHQMNEQL.js → rename-prompt-tag-K5LQ4CY6.js} +4 -4
- package/dist/rename-test-suite-NTYHAO5X.js +21 -0
- package/dist/report-issue-RHZZ33IS.js +885 -0
- package/dist/rerun-run-plan-MUDWJBE4.js +22 -0
- package/dist/{revoke-api-key-3Y35EMBT.js → revoke-api-key-6LAZ3QZZ.js} +5 -5
- package/dist/run-agent-CAGWGTYT.js +72 -0
- package/dist/run-plan-T4ZYOZGA.js +39 -0
- package/dist/run-test-suite-L4MZ7G4D.js +26 -0
- package/dist/{run-workflow-5LVBZYLS.js → run-workflow-DFIS23AP.js} +5 -5
- package/dist/{search-traces-U4HLOCZ2.js → search-traces-4XTJYXIK.js} +4 -4
- package/dist/{set-model-provider-V3M5CDIY.js → set-model-provider-XRNMFOVR.js} +5 -5
- package/dist/{update-agent-47RZIHS7.js → update-agent-DUWNYBNA.js} +5 -5
- package/dist/{update-dataset-G6QJJGS5.js → update-dataset-YSOMDCOB.js} +5 -5
- package/dist/{update-dataset-record-RMWLSWCS.js → update-dataset-record-DEA2ITYR.js} +5 -5
- package/dist/{update-evaluator-SFRRI5Q3.js → update-evaluator-NH7Q5GWV.js} +5 -5
- package/dist/{update-project-PI5DWBVK.js → update-project-Q26FBSYH.js} +5 -5
- package/dist/update-prompt-ZQQ2C3KU.js +143 -0
- package/dist/{update-scenario-G3IU2UIB.js → update-scenario-WUX34JAR.js} +12 -5
- package/package.json +8 -9
- package/pyproject.toml +5 -5
- package/src/__tests__/agent-tools.unit.test.ts +257 -0
- package/src/__tests__/all-tools.integration.test.ts +321 -75
- package/src/__tests__/config.unit.test.ts +33 -0
- package/src/__tests__/docker.integration.test.ts +91 -7
- package/src/__tests__/documentation-fetch-security.integration.test.ts +8 -2
- package/src/__tests__/feature-map-tool-drift.unit.test.ts +93 -0
- package/src/__tests__/http-security-origin.unit.test.ts +97 -0
- package/src/__tests__/http-security-sessions.unit.test.ts +209 -0
- package/src/__tests__/http-security-verifier.unit.test.ts +248 -0
- package/src/__tests__/http-server-allocation.integration.test.ts +279 -0
- package/src/__tests__/http-server-origin.integration.test.ts +169 -0
- package/src/__tests__/http-server-session-auth.integration.test.ts +341 -0
- package/src/__tests__/http-transport.integration.test.ts +85 -87
- package/src/__tests__/integration.integration.test.ts +7 -11
- package/src/__tests__/prompt-response-types.unit.test.ts +51 -0
- package/src/__tests__/prompts.integration.test.ts +736 -0
- package/src/__tests__/run-plan-tools.unit.test.ts +565 -0
- package/src/__tests__/scenario-tools.unit.test.ts +19 -0
- package/src/__tests__/simulation-run-tools.unit.test.ts +116 -0
- package/src/__tests__/support/http-server-harness.ts +138 -0
- package/src/__tests__/test-suite-tools.unit.test.ts +615 -0
- package/src/__tests__/{tools.unit.test.ts → tools.integration.test.ts} +31 -59
- package/src/config.ts +17 -2
- package/src/create-mcp-server.ts +334 -55
- package/src/http-security.ts +492 -0
- package/src/http-server.ts +536 -152
- package/src/index.ts +49 -2
- package/src/langwatch-api-agents.ts +196 -12
- package/src/langwatch-api-run-plans.ts +157 -0
- package/src/langwatch-api-scenarios.ts +9 -0
- package/src/langwatch-api-simulation-runs.ts +5 -0
- package/src/langwatch-api-test-suites.ts +135 -0
- package/src/langwatch-api.ts +62 -12
- package/src/schemas/index.ts +5 -0
- package/src/schemas/run-plan.ts +92 -0
- package/src/schemas/suite-fields.ts +149 -0
- package/src/tools/archive-run-plan.ts +12 -0
- package/src/tools/archive-test-suite.ts +12 -0
- package/src/tools/create-scenario.ts +8 -0
- package/src/tools/create-test-suite.ts +43 -0
- package/src/tools/discover-evaluator-schema.ts +2 -2
- package/src/tools/discover-scenario-schema.ts +54 -0
- package/src/tools/format-run-plan.ts +68 -0
- package/src/tools/format-scenario.ts +14 -0
- package/src/tools/format-suite-details.ts +96 -0
- package/src/tools/get-agent.ts +45 -3
- package/src/tools/get-prompt.ts +108 -24
- package/src/tools/get-run-plan.ts +62 -0
- package/src/tools/get-scenario.ts +3 -0
- package/src/tools/get-simulation-run.ts +9 -0
- package/src/tools/get-test-suite.ts +49 -0
- package/src/tools/list-agents.ts +20 -3
- package/src/tools/list-run-plans.ts +47 -0
- package/src/tools/list-scenarios.ts +12 -2
- package/src/tools/list-simulation-runs.ts +6 -2
- package/src/tools/list-test-suites.ts +35 -0
- package/src/tools/rename-test-suite.ts +18 -0
- package/src/tools/rerun-run-plan.ts +21 -0
- package/src/tools/run-agent.ts +54 -6
- package/src/tools/run-plan.ts +49 -0
- package/src/tools/run-test-suite.ts +30 -0
- package/src/tools/test-agent.ts +18 -0
- package/src/tools/update-prompt.ts +195 -10
- package/src/tools/update-scenario.ts +8 -0
- package/src/tools/update-test-suite.ts +45 -0
- package/tsconfig.json +7 -2
- package/uv.lock +418 -298
- package/dist/archive-suite-C3KX6IIZ.js +0 -16
- package/dist/chunk-WDH5P5RF.js +0 -48
- package/dist/create-suite-TWF6BBR4.js +0 -39
- package/dist/get-agent-7QJVPH72.js +0 -29
- package/dist/get-prompt-I753LZKZ.js +0 -46
- package/dist/get-suite-GTOCY5PL.js +0 -45
- package/dist/langwatch-api-workflows-5HRZ7HR4.js +0 -16
- package/dist/list-agents-TIJTP566.js +0 -33
- package/dist/list-suites-5VK4HXEK.js +0 -43
- package/dist/report-issue-DXJ6TPTC.js +0 -412
- package/dist/run-agent-RBSIFKRT.js +0 -37
- package/dist/run-suite-K3DNQ3SE.js +0 -30
- package/dist/update-prompt-RQFTEO7Y.js +0 -25
- package/dist/update-suite-TRGGLNR7.js +0 -40
- package/pnpm-workspace.yaml +0 -34
- package/src/langwatch-api-suites.ts +0 -118
- package/src/tools/archive-suite.ts +0 -12
- package/src/tools/create-suite.ts +0 -42
- package/src/tools/get-suite.ts +0 -47
- package/src/tools/list-suites.ts +0 -43
- package/src/tools/run-suite.ts +0 -28
- package/src/tools/update-suite.ts +0 -44
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared fixtures for the standalone MCP HTTP server tests.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { expect, vi } from "vitest";
|
|
6
|
+
|
|
7
|
+
import type { ApiKeyVerifier } from "../../http-security.js";
|
|
8
|
+
import { startHttpServer } from "../../http-server.js";
|
|
9
|
+
|
|
10
|
+
/** Standard headers the MCP Streamable HTTP protocol requires on POST. */
|
|
11
|
+
export const MCP_POST_HEADERS = {
|
|
12
|
+
"Content-Type": "application/json",
|
|
13
|
+
Accept: "application/json, text/event-stream",
|
|
14
|
+
};
|
|
15
|
+
|
|
16
|
+
export const VALID_KEY = "sk-lw-valid-key";
|
|
17
|
+
export const OTHER_VALID_KEY = "sk-lw-other-valid-key";
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Stands in for the LangWatch API. The standalone server has no database, so
|
|
21
|
+
* it asks the API whether a key is real; here a fixed list is.
|
|
22
|
+
*/
|
|
23
|
+
export function countingVerifier(validKeys: string[]) {
|
|
24
|
+
const verify = vi.fn(async (apiKey: string) => validKeys.includes(apiKey));
|
|
25
|
+
const verifier: ApiKeyVerifier = {
|
|
26
|
+
verify,
|
|
27
|
+
sweep: () => undefined,
|
|
28
|
+
clear: () => undefined,
|
|
29
|
+
};
|
|
30
|
+
return { verifier, verify };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** A verifier whose accepted keys can change mid-test, to model revocation. */
|
|
34
|
+
export function revocableVerifier(initialKeys: string[]) {
|
|
35
|
+
const live = new Set(initialKeys);
|
|
36
|
+
const verifier: ApiKeyVerifier = {
|
|
37
|
+
verify: async (apiKey: string) => live.has(apiKey),
|
|
38
|
+
sweep: () => undefined,
|
|
39
|
+
clear: () => undefined,
|
|
40
|
+
};
|
|
41
|
+
return { verifier, revoke: (apiKey: string) => live.delete(apiKey) };
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export interface Harness {
|
|
45
|
+
baseUrl: string;
|
|
46
|
+
close: () => Promise<void>;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export async function startHarness(
|
|
50
|
+
options: Parameters<typeof startHttpServer>[0]
|
|
51
|
+
): Promise<Harness> {
|
|
52
|
+
const { server, port, host } = await startHttpServer(options);
|
|
53
|
+
return {
|
|
54
|
+
baseUrl: `http://${host === "0.0.0.0" ? "127.0.0.1" : host}:${port}`,
|
|
55
|
+
close: () =>
|
|
56
|
+
new Promise<void>((resolve) => {
|
|
57
|
+
server.close(() => resolve());
|
|
58
|
+
}),
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export function initializeBody(): string {
|
|
63
|
+
return JSON.stringify({
|
|
64
|
+
jsonrpc: "2.0",
|
|
65
|
+
method: "initialize",
|
|
66
|
+
params: {
|
|
67
|
+
protocolVersion: "2025-03-26",
|
|
68
|
+
capabilities: {},
|
|
69
|
+
clientInfo: { name: "test-client", version: "1.0.0" },
|
|
70
|
+
},
|
|
71
|
+
id: 1,
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export function toolsListBody(): string {
|
|
76
|
+
return JSON.stringify({ jsonrpc: "2.0", method: "tools/list", id: 2 });
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** Opens a Streamable HTTP session and returns its id. */
|
|
80
|
+
export async function openSession({
|
|
81
|
+
baseUrl,
|
|
82
|
+
apiKey = VALID_KEY,
|
|
83
|
+
}: {
|
|
84
|
+
baseUrl: string;
|
|
85
|
+
apiKey?: string;
|
|
86
|
+
}): Promise<string> {
|
|
87
|
+
const response = await fetch(`${baseUrl}/mcp`, {
|
|
88
|
+
method: "POST",
|
|
89
|
+
headers: { ...MCP_POST_HEADERS, Authorization: `Bearer ${apiKey}` },
|
|
90
|
+
body: initializeBody(),
|
|
91
|
+
});
|
|
92
|
+
expect(response.status).toBe(200);
|
|
93
|
+
await response.text();
|
|
94
|
+
return response.headers.get("mcp-session-id")!;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Reads an SSE stream until the endpoint event carrying the session id has
|
|
99
|
+
* arrived. A single read can return a partial frame, so waiting for the
|
|
100
|
+
* pattern rather than the first chunk keeps this from failing intermittently.
|
|
101
|
+
*/
|
|
102
|
+
export async function readSessionId(response: Response): Promise<string> {
|
|
103
|
+
const reader = response.body!.getReader();
|
|
104
|
+
const decoder = new TextDecoder();
|
|
105
|
+
let buffered = "";
|
|
106
|
+
|
|
107
|
+
for (let reads = 0; reads < 20; reads++) {
|
|
108
|
+
const { value, done } = await reader.read();
|
|
109
|
+
if (done) break;
|
|
110
|
+
buffered += decoder.decode(value, { stream: true });
|
|
111
|
+
const match = /sessionId=([\w-]+)/.exec(buffered);
|
|
112
|
+
if (match?.[1]) return match[1];
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
throw new Error(`No session id in SSE stream. Received: ${buffered}`);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Sends `request` repeatedly until it answers 429, and returns that response
|
|
120
|
+
* along with how many attempts it took. Returns `rejected: undefined` when the
|
|
121
|
+
* limit never triggered within `maxAttempts`.
|
|
122
|
+
*/
|
|
123
|
+
export async function requestUntilThrottled({
|
|
124
|
+
request,
|
|
125
|
+
maxAttempts = 30,
|
|
126
|
+
}: {
|
|
127
|
+
request: (attempt: number) => Promise<Response>;
|
|
128
|
+
maxAttempts?: number;
|
|
129
|
+
}): Promise<{ attempts: number; rejected: Response | undefined }> {
|
|
130
|
+
for (let attempt = 0; attempt < maxAttempts; attempt++) {
|
|
131
|
+
const response = await request(attempt);
|
|
132
|
+
if (response.status === 429) {
|
|
133
|
+
return { attempts: attempt + 1, rejected: response };
|
|
134
|
+
}
|
|
135
|
+
await response.text();
|
|
136
|
+
}
|
|
137
|
+
return { attempts: maxAttempts, rejected: undefined };
|
|
138
|
+
}
|