@specific.dev/spectest 0.55.0 → 0.56.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/daemon.js +10 -0
- package/dist/harness/raw-fetch.d.ts +7 -0
- package/dist/harness/raw-fetch.js +33 -0
- package/dist/ids.d.ts +9 -0
- package/dist/ids.js +11 -1
- package/dist/index.d.ts +43 -0
- package/dist/index.js +4 -0
- package/dist/mcp-auth.d.ts +176 -0
- package/dist/mcp-auth.js +455 -0
- package/dist/mcp-transport.d.ts +130 -0
- package/dist/mcp-transport.js +337 -0
- package/dist/mcp.d.ts +246 -0
- package/dist/mcp.js +1060 -0
- package/dist/recorder.d.ts +5 -0
- package/package.json +1 -1
- package/src/daemon.ts +10 -0
- package/src/harness/raw-fetch.ts +36 -0
- package/src/ids.ts +12 -1
- package/src/index.ts +69 -0
- package/src/mcp-auth.ts +626 -0
- package/src/mcp-transport.ts +434 -0
- package/src/mcp.test.ts +94 -0
- package/src/mcp.ts +1435 -0
- package/src/recorder.ts +5 -2
package/dist/mcp.js
ADDED
|
@@ -0,0 +1,1060 @@
|
|
|
1
|
+
// `ctx.mcp(url)` — drive a Model Context Protocol server from a test.
|
|
2
|
+
//
|
|
3
|
+
// An MCP server is the interface an application exposes to an AI client:
|
|
4
|
+
// tools it can call, resources it can read, prompts it can fill in. This
|
|
5
|
+
// module is the client half. A test connects to the server under test,
|
|
6
|
+
// calls its tools like a real client would, and asserts on what comes
|
|
7
|
+
// back — and every call renders on the timeline as its own step, with the
|
|
8
|
+
// arguments and the result, instead of as raw HTTP.
|
|
9
|
+
//
|
|
10
|
+
// ## No session registry, and no names
|
|
11
|
+
//
|
|
12
|
+
// `ctx.mcp(url)` connects and returns a NEW client every time. There is no
|
|
13
|
+
// per-name registry like `ctx.browser("alice")` has. A client that has
|
|
14
|
+
// authenticated is passed to the tests that need it, as the test's return
|
|
15
|
+
// value:
|
|
16
|
+
//
|
|
17
|
+
// export const signedIn = env.test("sign in", async (ctx) => {
|
|
18
|
+
// const mcp = await ctx.mcp(URL);
|
|
19
|
+
// …authorize…
|
|
20
|
+
// return mcp; // ctx.parent, for every child
|
|
21
|
+
// });
|
|
22
|
+
//
|
|
23
|
+
// env.test("call a tool", { dependsOn: signedIn }, async (ctx) => {
|
|
24
|
+
// await ctx.parent.call("create_invoice", { amount: 250 });
|
|
25
|
+
// });
|
|
26
|
+
//
|
|
27
|
+
// That works because a test's return value crosses the fork in daemon
|
|
28
|
+
// memory, live connections included. Two identities are two calls to
|
|
29
|
+
// `ctx.mcp`, with no naming concept at all.
|
|
30
|
+
//
|
|
31
|
+
// ## The client does not police the server
|
|
32
|
+
//
|
|
33
|
+
// `call()` sends the request with whatever credentials it holds, including
|
|
34
|
+
// none. A protected server answers `401` and that is what the test sees —
|
|
35
|
+
// an {@link McpHttpError} carrying the parsed challenge. The client never
|
|
36
|
+
// refuses to try, because "the server rejects an unauthenticated call" is
|
|
37
|
+
// a thing tests must be able to prove.
|
|
38
|
+
//
|
|
39
|
+
// ## Authentication
|
|
40
|
+
//
|
|
41
|
+
// The SDK owns the protocol (discovery, registration, PKCE, the RFC 8707
|
|
42
|
+
// `resource` binding, the token exchange, refresh — all in `mcp-auth.ts`).
|
|
43
|
+
// The test owns the human: it drives the login and the consent screen in
|
|
44
|
+
// its own browser. Nothing here guesses at a button.
|
|
45
|
+
//
|
|
46
|
+
// const auth = await mcp.authorize();
|
|
47
|
+
// await page.goto(auth.url);
|
|
48
|
+
// …sign in, click Allow…
|
|
49
|
+
// await auth.complete();
|
|
50
|
+
//
|
|
51
|
+
// ## Recording
|
|
52
|
+
//
|
|
53
|
+
// Each operation records one `mcp` step in the dashboard's generic
|
|
54
|
+
// presentation vocabulary (`crates/control-plane/src/web/blocks.rs`), so
|
|
55
|
+
// no server change was needed to render any of this. HTTP is done with
|
|
56
|
+
// recording PAUSED — otherwise every JSON-RPC round trip would also land
|
|
57
|
+
// as an `http` row and bury the step it belongs to. Tokens are redacted
|
|
58
|
+
// before an event exists.
|
|
59
|
+
import { pauseRecording, recordStep, reserveEvent, resumeRecording } from "./recorder.js";
|
|
60
|
+
import { wrap } from "./inspect.js";
|
|
61
|
+
import { McpHttpError, McpRpcError, McpTransport, PROTOCOL_VERSION, } from "./mcp-transport.js";
|
|
62
|
+
import { authorize as beginAuthorization, refreshIdentity, } from "./mcp-auth.js";
|
|
63
|
+
export { McpHttpError, McpRpcError } from "./mcp-transport.js";
|
|
64
|
+
export { McpAuthDeniedError, } from "./mcp-auth.js";
|
|
65
|
+
const DEFAULT_CLIENT_INFO = { name: "spectest", version: "1.0.0" };
|
|
66
|
+
/**
|
|
67
|
+
* Connect to an MCP server over streamable HTTP.
|
|
68
|
+
*
|
|
69
|
+
* Does not throw when the server answers the handshake with `401`: that
|
|
70
|
+
* response is the documented start of the OAuth flow, and it is kept on
|
|
71
|
+
* the client as {@link Mcp.challenge}. Any other failure throws.
|
|
72
|
+
*/
|
|
73
|
+
export async function openMcp(url, opts = {}) {
|
|
74
|
+
const client = new McpClient(url, opts);
|
|
75
|
+
await client.connectQuietly();
|
|
76
|
+
return client;
|
|
77
|
+
}
|
|
78
|
+
class McpClient {
|
|
79
|
+
url;
|
|
80
|
+
connected = false;
|
|
81
|
+
// Raw state. Everything a test READS comes back through the getters
|
|
82
|
+
// below, wrapped against the step that produced it — so an assertion on
|
|
83
|
+
// `mcp.serverInfo.name` nests under the handshake, and one on
|
|
84
|
+
// `mcp.challenge` nests under the call that was refused. Internal code
|
|
85
|
+
// uses these fields, never the getters: a wrapped value handed to
|
|
86
|
+
// `fetch` or to a header would be an object, not a string.
|
|
87
|
+
_serverInfo;
|
|
88
|
+
_capabilities;
|
|
89
|
+
_protocolVersion;
|
|
90
|
+
_instructions;
|
|
91
|
+
_challenge;
|
|
92
|
+
_identity;
|
|
93
|
+
/** Seq of the handshake step, the refused step, and the step that
|
|
94
|
+
* completed an authorization. */
|
|
95
|
+
connectSeq;
|
|
96
|
+
challengeSeq;
|
|
97
|
+
identitySeq;
|
|
98
|
+
/** Seq of the step this client recorded last. */
|
|
99
|
+
lastStepSeq;
|
|
100
|
+
/**
|
|
101
|
+
* The credential the server last refused the handshake for.
|
|
102
|
+
*
|
|
103
|
+
* Without this, an unauthenticated `call()` recorded TWO identical
|
|
104
|
+
* failed steps: one for the handshake it retried, one for the call. The
|
|
105
|
+
* handshake cannot succeed with a credential that was just refused, so
|
|
106
|
+
* the call goes straight out and the server refuses the thing the test
|
|
107
|
+
* actually asked for.
|
|
108
|
+
*/
|
|
109
|
+
refusedToken;
|
|
110
|
+
/**
|
|
111
|
+
* How many times THIS client has paused the recorder.
|
|
112
|
+
*
|
|
113
|
+
* Recording is paused for the length of an operation so its HTTP does
|
|
114
|
+
* not also land as `http` rows. Two things record from inside that
|
|
115
|
+
* region — the answer to a server-to-client request, and a token
|
|
116
|
+
* refresh — and they must lift the pause to do it. `resumeRecording`
|
|
117
|
+
* clamps at zero, so lifting a pause that is no longer held would leave
|
|
118
|
+
* the recorder paused for the rest of the test after the matching
|
|
119
|
+
* `pauseRecording`. The answer runs on a detached task, so that
|
|
120
|
+
* ordering is reachable. This counter is how we know.
|
|
121
|
+
*/
|
|
122
|
+
pauseDepth = 0;
|
|
123
|
+
opts;
|
|
124
|
+
transport;
|
|
125
|
+
createdAt = Date.now();
|
|
126
|
+
/** Notifications, each with the seq of the step it was recorded as. */
|
|
127
|
+
received = [];
|
|
128
|
+
/** The step a server-to-client request should nest under: the tool call
|
|
129
|
+
* that provoked it. */
|
|
130
|
+
activeSeq;
|
|
131
|
+
token;
|
|
132
|
+
clientSecret;
|
|
133
|
+
lastExchange;
|
|
134
|
+
constructor(url, opts) {
|
|
135
|
+
this.url = url;
|
|
136
|
+
this.opts = opts;
|
|
137
|
+
this.token = opts.bearer;
|
|
138
|
+
this.transport = new McpTransport({
|
|
139
|
+
url,
|
|
140
|
+
headers: opts.headers,
|
|
141
|
+
token: () => this.token,
|
|
142
|
+
timeoutMs: opts.timeoutMs,
|
|
143
|
+
onNotification: (n) => this.onNotification(n),
|
|
144
|
+
onRequest: (r) => this.onServerRequest(r),
|
|
145
|
+
onExchange: (x) => {
|
|
146
|
+
this.lastExchange = x;
|
|
147
|
+
},
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
get serverInfo() {
|
|
151
|
+
return wrapped(this._serverInfo, this.connectSeq);
|
|
152
|
+
}
|
|
153
|
+
get capabilities() {
|
|
154
|
+
return wrapped(this._capabilities, this.connectSeq);
|
|
155
|
+
}
|
|
156
|
+
get protocolVersion() {
|
|
157
|
+
return wrapped(this._protocolVersion, this.connectSeq);
|
|
158
|
+
}
|
|
159
|
+
get instructions() {
|
|
160
|
+
return wrapped(this._instructions, this.connectSeq);
|
|
161
|
+
}
|
|
162
|
+
get sessionId() {
|
|
163
|
+
return wrapped(this.transport.sessionId, this.connectSeq);
|
|
164
|
+
}
|
|
165
|
+
get challenge() {
|
|
166
|
+
return wrapped(this._challenge, this.challengeSeq);
|
|
167
|
+
}
|
|
168
|
+
get identity() {
|
|
169
|
+
return wrapped(this._identity, this.identitySeq);
|
|
170
|
+
}
|
|
171
|
+
/** Handshake, but treat a `401` as information rather than as failure. */
|
|
172
|
+
async connectQuietly() {
|
|
173
|
+
try {
|
|
174
|
+
const { seq } = await this.runStep("connect", () => this.handshake(), {
|
|
175
|
+
summary: () => this.handshakeBlocks(),
|
|
176
|
+
connection: false,
|
|
177
|
+
});
|
|
178
|
+
this.connectSeq = seq;
|
|
179
|
+
}
|
|
180
|
+
catch (err) {
|
|
181
|
+
// A refused handshake is the documented start of the OAuth flow,
|
|
182
|
+
// not a failure to report. The challenge it carried is kept, and
|
|
183
|
+
// `authorize()` starts from it.
|
|
184
|
+
if (err instanceof McpHttpError && err.status === 401) {
|
|
185
|
+
this._challenge = err.challenge;
|
|
186
|
+
this.refusedToken = this.token ?? null;
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
throw err;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
async handshake() {
|
|
193
|
+
const result = await this.transport.request("initialize", {
|
|
194
|
+
protocolVersion: PROTOCOL_VERSION,
|
|
195
|
+
capabilities: this.clientCapabilities(),
|
|
196
|
+
clientInfo: this.opts.clientInfo ?? DEFAULT_CLIENT_INFO,
|
|
197
|
+
});
|
|
198
|
+
this.refusedToken = undefined;
|
|
199
|
+
this._protocolVersion = result.protocolVersion ?? PROTOCOL_VERSION;
|
|
200
|
+
this.transport.protocolVersion = this._protocolVersion;
|
|
201
|
+
this._capabilities = result.capabilities;
|
|
202
|
+
this._serverInfo = result.serverInfo;
|
|
203
|
+
this._instructions = result.instructions;
|
|
204
|
+
this.connected = true;
|
|
205
|
+
this._challenge = undefined;
|
|
206
|
+
await this.transport.notify("notifications/initialized");
|
|
207
|
+
}
|
|
208
|
+
clientCapabilities() {
|
|
209
|
+
const capabilities = {};
|
|
210
|
+
if (this.opts.sampling)
|
|
211
|
+
capabilities["sampling"] = {};
|
|
212
|
+
if (this.opts.roots)
|
|
213
|
+
capabilities["roots"] = { listChanged: false };
|
|
214
|
+
return capabilities;
|
|
215
|
+
}
|
|
216
|
+
async authorize(opts = {}) {
|
|
217
|
+
const { value: flow, seq: authorizeSeq } = await this.runStep("authorize", () => beginAuthorization(this.url, this._challenge?.resourceMetadataUrl, opts), {
|
|
218
|
+
connection: false,
|
|
219
|
+
summary: (auth) => [
|
|
220
|
+
kv([
|
|
221
|
+
["Issuer", auth.server.issuer],
|
|
222
|
+
["Client", auth.clientId],
|
|
223
|
+
[
|
|
224
|
+
"Registered",
|
|
225
|
+
auth.server.dynamicallyRegistered ? "dynamically, for this flow" : "already known",
|
|
226
|
+
],
|
|
227
|
+
["Redirect", auth.redirectUri],
|
|
228
|
+
["Scopes", auth.scopes.length > 0 ? auth.scopes.join(" ") : "none requested"],
|
|
229
|
+
]),
|
|
230
|
+
{
|
|
231
|
+
type: "text",
|
|
232
|
+
text: "Waiting for a browser to sign in and consent.",
|
|
233
|
+
},
|
|
234
|
+
{
|
|
235
|
+
type: "details",
|
|
236
|
+
summary: "Discovery",
|
|
237
|
+
blocks: [
|
|
238
|
+
kv([
|
|
239
|
+
["Authorization", auth.server.authorizationEndpoint],
|
|
240
|
+
["Token", auth.server.tokenEndpoint],
|
|
241
|
+
["Registration", auth.server.registrationEndpoint],
|
|
242
|
+
["Resource", auth.resource],
|
|
243
|
+
["PKCE", "S256"],
|
|
244
|
+
]),
|
|
245
|
+
{ type: "code", code: auth.url, label: "Authorization URL" },
|
|
246
|
+
],
|
|
247
|
+
},
|
|
248
|
+
],
|
|
249
|
+
});
|
|
250
|
+
this.clientSecret = opts.clientSecret;
|
|
251
|
+
const client = this;
|
|
252
|
+
return {
|
|
253
|
+
url: flow.url,
|
|
254
|
+
redirectUri: flow.redirectUri,
|
|
255
|
+
state: flow.state,
|
|
256
|
+
clientId: flow.clientId,
|
|
257
|
+
server: wrapped(flow.server, authorizeSeq),
|
|
258
|
+
scopes: flow.scopes,
|
|
259
|
+
resource: flow.resource,
|
|
260
|
+
cancel: () => flow.cancel(),
|
|
261
|
+
async complete(completeOpts) {
|
|
262
|
+
const { value: identity, seq } = await client.runStep("complete authorization", async () => {
|
|
263
|
+
const identity = await flow.complete(completeOpts);
|
|
264
|
+
client._identity = identity;
|
|
265
|
+
client.token = identity.accessToken;
|
|
266
|
+
// A session opened before the token was issued was refused,
|
|
267
|
+
// so start a clean one rather than reusing its id.
|
|
268
|
+
client.transport.sessionId = undefined;
|
|
269
|
+
await client.handshake();
|
|
270
|
+
return identity;
|
|
271
|
+
}, {
|
|
272
|
+
connection: false,
|
|
273
|
+
// Every field of the identity is readable — and therefore
|
|
274
|
+
// assertable — so every field is shown. An assertion on
|
|
275
|
+
// `identity.redirectUri` that names a row nobody can see is
|
|
276
|
+
// worse than no assertion at all.
|
|
277
|
+
summary: (identity) => [
|
|
278
|
+
kv([
|
|
279
|
+
["Granted scopes", identity.scopes.join(" ") || "none"],
|
|
280
|
+
["Token", redactToken(identity.accessToken)],
|
|
281
|
+
["Token type", identity.tokenType],
|
|
282
|
+
["Refresh token", identity.refreshToken ? "issued" : "none"],
|
|
283
|
+
[
|
|
284
|
+
"Expires",
|
|
285
|
+
identity.expiresAt
|
|
286
|
+
? new Date(identity.expiresAt).toISOString().replace("T", " ").slice(0, 19)
|
|
287
|
+
: "not stated",
|
|
288
|
+
],
|
|
289
|
+
["Resource", identity.resource],
|
|
290
|
+
["Client", identity.clientId],
|
|
291
|
+
["Issuer", identity.issuer],
|
|
292
|
+
["Redirect URI", identity.redirectUri],
|
|
293
|
+
["Token endpoint", identity.tokenEndpoint],
|
|
294
|
+
]),
|
|
295
|
+
// This step ran the handshake, so the server's own identity
|
|
296
|
+
// was read here too — and `mcp.serverInfo` assertions point
|
|
297
|
+
// at this step.
|
|
298
|
+
...client.handshakeBlocks("Server"),
|
|
299
|
+
],
|
|
300
|
+
});
|
|
301
|
+
// Both the grant and the handshake it ran belong to this step, so
|
|
302
|
+
// `mcp.identity` and `mcp.serverInfo` assert against it.
|
|
303
|
+
client.identitySeq = seq;
|
|
304
|
+
client.connectSeq = seq;
|
|
305
|
+
return wrapped(identity, seq);
|
|
306
|
+
},
|
|
307
|
+
};
|
|
308
|
+
}
|
|
309
|
+
async tools() {
|
|
310
|
+
return this.stepWrapped("list tools", async () => {
|
|
311
|
+
const result = await this.send("tools/list", {});
|
|
312
|
+
return result.tools ?? [];
|
|
313
|
+
}, {
|
|
314
|
+
summary: (tools) => [
|
|
315
|
+
{
|
|
316
|
+
type: "table",
|
|
317
|
+
columns: ["Tool", "Title", "Description"],
|
|
318
|
+
rows: tools.map((t) => [t.name, t.title ?? "", t.description ?? ""]),
|
|
319
|
+
},
|
|
320
|
+
// The table carries three columns; a tool also has its input and
|
|
321
|
+
// output schemas, and a test may assert on either.
|
|
322
|
+
...rawBlock("Full catalogue", tools),
|
|
323
|
+
],
|
|
324
|
+
});
|
|
325
|
+
}
|
|
326
|
+
async call(name, args = {}, opts) {
|
|
327
|
+
const resv = reserveEvent();
|
|
328
|
+
const started = Date.now();
|
|
329
|
+
this.pause();
|
|
330
|
+
let raw;
|
|
331
|
+
let failure;
|
|
332
|
+
try {
|
|
333
|
+
this.activeSeq = resv?.seq;
|
|
334
|
+
raw = await this.send("tools/call", { name, arguments: args }, opts);
|
|
335
|
+
}
|
|
336
|
+
catch (err) {
|
|
337
|
+
failure = err;
|
|
338
|
+
}
|
|
339
|
+
finally {
|
|
340
|
+
this.activeSeq = undefined;
|
|
341
|
+
this.resume();
|
|
342
|
+
}
|
|
343
|
+
const durationMs = Date.now() - started;
|
|
344
|
+
if (failure || !raw) {
|
|
345
|
+
const seq = recordStep({
|
|
346
|
+
kind: "mcp",
|
|
347
|
+
title: this.title(name),
|
|
348
|
+
status: "failed",
|
|
349
|
+
durationMs,
|
|
350
|
+
error: errorMessage(failure),
|
|
351
|
+
blocks: [
|
|
352
|
+
...argumentBlocks(args),
|
|
353
|
+
...this.failureBlocks(failure),
|
|
354
|
+
...this.connectionBlock(),
|
|
355
|
+
],
|
|
356
|
+
}, resv);
|
|
357
|
+
this.noteChallenge(failure, seq);
|
|
358
|
+
throw failure;
|
|
359
|
+
}
|
|
360
|
+
const content = raw.content ?? [];
|
|
361
|
+
const text = textOf(content);
|
|
362
|
+
const isError = raw.isError === true;
|
|
363
|
+
const seq = recordStep({
|
|
364
|
+
kind: "mcp",
|
|
365
|
+
title: this.title(name),
|
|
366
|
+
status: isError ? "failed" : "passed",
|
|
367
|
+
durationMs,
|
|
368
|
+
error: isError ? text || "the tool reported an error" : undefined,
|
|
369
|
+
blocks: [
|
|
370
|
+
...argumentBlocks(args),
|
|
371
|
+
// `res.isError` is assertable, so it is shown rather than left
|
|
372
|
+
// to be inferred from the step's pass/fail chip. It needs its
|
|
373
|
+
// own label: two kv blocks in a row read as one list, and
|
|
374
|
+
// without a heading this looked like a third argument.
|
|
375
|
+
{
|
|
376
|
+
type: "kv",
|
|
377
|
+
label: "Outcome",
|
|
378
|
+
rows: [{ label: "isError", value: String(isError), error: isError }],
|
|
379
|
+
},
|
|
380
|
+
// A failed tool's message IS its text content, and the step
|
|
381
|
+
// already leads with it as the error. Printing it twice reads
|
|
382
|
+
// as two different things having gone wrong.
|
|
383
|
+
...resultBlocks(content, raw.structuredContent, isError ? text : undefined),
|
|
384
|
+
...this.connectionBlock(),
|
|
385
|
+
],
|
|
386
|
+
}, resv);
|
|
387
|
+
return makeToolResult(raw, content, text, isError, seq);
|
|
388
|
+
}
|
|
389
|
+
async resources() {
|
|
390
|
+
return this.stepWrapped("list resources", async () => {
|
|
391
|
+
const result = await this.send("resources/list", {});
|
|
392
|
+
return result.resources ?? [];
|
|
393
|
+
}, {
|
|
394
|
+
summary: (resources) => [
|
|
395
|
+
{
|
|
396
|
+
type: "table",
|
|
397
|
+
columns: ["URI", "Name", "Type"],
|
|
398
|
+
rows: resources.map((r) => [r.uri, r.name ?? r.title ?? "", r.mimeType ?? ""]),
|
|
399
|
+
},
|
|
400
|
+
...rawBlock("Full listing", resources),
|
|
401
|
+
],
|
|
402
|
+
});
|
|
403
|
+
}
|
|
404
|
+
async read(uri) {
|
|
405
|
+
return this.stepWrapped(`read ${uri}`, async () => {
|
|
406
|
+
const result = await this.send("resources/read", { uri });
|
|
407
|
+
return result.contents ?? [];
|
|
408
|
+
}, {
|
|
409
|
+
summary: (contents) => contents.flatMap((c) => resourceBlocks(c)),
|
|
410
|
+
});
|
|
411
|
+
}
|
|
412
|
+
async prompts() {
|
|
413
|
+
return this.stepWrapped("list prompts", async () => {
|
|
414
|
+
const result = await this.send("prompts/list", {});
|
|
415
|
+
return result.prompts ?? [];
|
|
416
|
+
}, {
|
|
417
|
+
summary: (prompts) => [
|
|
418
|
+
{
|
|
419
|
+
type: "table",
|
|
420
|
+
columns: ["Prompt", "Description"],
|
|
421
|
+
rows: prompts.map((p) => [p.name, p.description ?? ""]),
|
|
422
|
+
},
|
|
423
|
+
...rawBlock("Full listing", prompts),
|
|
424
|
+
],
|
|
425
|
+
});
|
|
426
|
+
}
|
|
427
|
+
async prompt(name, args) {
|
|
428
|
+
return this.stepWrapped(`prompt ${name}`, () => this.send("prompts/get", { name, arguments: args ?? {} }), {
|
|
429
|
+
summary: (result) => [
|
|
430
|
+
...(result.description ? [{ type: "text", text: result.description }] : []),
|
|
431
|
+
{
|
|
432
|
+
type: "chat",
|
|
433
|
+
messages: (result.messages ?? []).map((m) => ({
|
|
434
|
+
side: m.role === "assistant" ? "other" : "self",
|
|
435
|
+
text: contentText(m.content),
|
|
436
|
+
})),
|
|
437
|
+
},
|
|
438
|
+
// Bubbles carry the text of each turn. The role names and any
|
|
439
|
+
// non-text content are only in the message objects themselves.
|
|
440
|
+
...rawBlock("Messages", result.messages ?? []),
|
|
441
|
+
],
|
|
442
|
+
});
|
|
443
|
+
}
|
|
444
|
+
notifications(filter) {
|
|
445
|
+
const matches = filter === undefined
|
|
446
|
+
? this.received
|
|
447
|
+
: this.received.filter((n) => typeof filter === "string" ? n.method === filter : filter.test(n.method));
|
|
448
|
+
// Wrapped per element, not per array: each notification came from a
|
|
449
|
+
// different step, and a whole-array tag would point every assertion at
|
|
450
|
+
// whichever one happened to be last.
|
|
451
|
+
return matches.map(({ seq, ...n }) => wrapped(n, seq));
|
|
452
|
+
}
|
|
453
|
+
async close() {
|
|
454
|
+
await this.step("close", async () => {
|
|
455
|
+
await this.transport.close();
|
|
456
|
+
this.connected = false;
|
|
457
|
+
});
|
|
458
|
+
}
|
|
459
|
+
/**
|
|
460
|
+
* Send one request, repairing the two failures a test environment
|
|
461
|
+
* produces by itself.
|
|
462
|
+
*
|
|
463
|
+
* A client is normally handed down from the test that authenticated it,
|
|
464
|
+
* so by the time it is used it lives in a RESTORED fork: the connection
|
|
465
|
+
* it holds was opened before the snapshot and the peer may have reset
|
|
466
|
+
* it. And an access token expires. Both are repaired once, silently,
|
|
467
|
+
* because neither is something a test should have to write.
|
|
468
|
+
*/
|
|
469
|
+
async send(method, params, opts) {
|
|
470
|
+
if (!this.connected && this.refusedToken !== (this.token ?? null)) {
|
|
471
|
+
// No session yet — the handshake was refused, or the server dropped
|
|
472
|
+
// it. Open one now and let a `401` propagate: a rejection is the
|
|
473
|
+
// server's answer to this call, not something to hide behind a
|
|
474
|
+
// client-side gate.
|
|
475
|
+
try {
|
|
476
|
+
await this.handshake();
|
|
477
|
+
}
|
|
478
|
+
catch (err) {
|
|
479
|
+
if (err instanceof McpHttpError && err.status === 401) {
|
|
480
|
+
this._challenge = err.challenge;
|
|
481
|
+
this.refusedToken = this.token ?? null;
|
|
482
|
+
}
|
|
483
|
+
throw err;
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
try {
|
|
487
|
+
return await this.transport.request(method, params, opts);
|
|
488
|
+
}
|
|
489
|
+
catch (err) {
|
|
490
|
+
if (await this.repair(err))
|
|
491
|
+
return this.transport.request(method, params, opts);
|
|
492
|
+
throw err;
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
/** Try to make one failure survivable. Returns true when the caller
|
|
496
|
+
* should retry exactly once. */
|
|
497
|
+
async repair(err) {
|
|
498
|
+
if (err instanceof McpHttpError) {
|
|
499
|
+
if (err.status === 401) {
|
|
500
|
+
this._challenge = err.challenge;
|
|
501
|
+
// An expired token, and a refresh token to spend on it.
|
|
502
|
+
if (this._identity?.refreshToken) {
|
|
503
|
+
const refreshed = await this.refresh();
|
|
504
|
+
if (refreshed)
|
|
505
|
+
return true;
|
|
506
|
+
}
|
|
507
|
+
return false;
|
|
508
|
+
}
|
|
509
|
+
// The server forgot this session — it restarted, or the session id
|
|
510
|
+
// came from before the fork. A fresh handshake is the whole repair.
|
|
511
|
+
if (err.status === 404 || err.status === 400) {
|
|
512
|
+
return this.reconnect();
|
|
513
|
+
}
|
|
514
|
+
return false;
|
|
515
|
+
}
|
|
516
|
+
// A connection the peer reset. Common in a restored fork: the flow was
|
|
517
|
+
// established before the snapshot.
|
|
518
|
+
if (isConnectionError(err))
|
|
519
|
+
return this.reconnect();
|
|
520
|
+
return false;
|
|
521
|
+
}
|
|
522
|
+
async reconnect() {
|
|
523
|
+
this.transport.sessionId = undefined;
|
|
524
|
+
this.connected = false;
|
|
525
|
+
try {
|
|
526
|
+
await this.handshake();
|
|
527
|
+
return true;
|
|
528
|
+
}
|
|
529
|
+
catch {
|
|
530
|
+
return false;
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
async refresh() {
|
|
534
|
+
if (!this._identity)
|
|
535
|
+
return false;
|
|
536
|
+
if (!this._identity.tokenEndpoint)
|
|
537
|
+
return false;
|
|
538
|
+
const started = Date.now();
|
|
539
|
+
let refreshed;
|
|
540
|
+
let failure;
|
|
541
|
+
try {
|
|
542
|
+
refreshed = await refreshIdentity(this._identity, this.clientSecret);
|
|
543
|
+
}
|
|
544
|
+
catch (err) {
|
|
545
|
+
failure = err;
|
|
546
|
+
}
|
|
547
|
+
if (refreshed) {
|
|
548
|
+
this._identity = refreshed;
|
|
549
|
+
this.token = refreshed.accessToken;
|
|
550
|
+
}
|
|
551
|
+
// The caller paused the recorder for the length of its own step; lift
|
|
552
|
+
// that just long enough to put this repair on the timeline.
|
|
553
|
+
this.recording(() => recordStep({
|
|
554
|
+
kind: "mcp",
|
|
555
|
+
title: this.title("refresh token"),
|
|
556
|
+
status: refreshed ? "passed" : "failed",
|
|
557
|
+
durationMs: Date.now() - started,
|
|
558
|
+
error: refreshed
|
|
559
|
+
? undefined
|
|
560
|
+
: (errorMessage(failure) ?? "the server issued no refresh token"),
|
|
561
|
+
blocks: refreshed
|
|
562
|
+
? [
|
|
563
|
+
kv([
|
|
564
|
+
["Token", redactToken(refreshed.accessToken)],
|
|
565
|
+
[
|
|
566
|
+
"Expires",
|
|
567
|
+
refreshed.expiresAt ? new Date(refreshed.expiresAt).toISOString() : "not stated",
|
|
568
|
+
],
|
|
569
|
+
]),
|
|
570
|
+
]
|
|
571
|
+
: [],
|
|
572
|
+
}));
|
|
573
|
+
return refreshed !== undefined;
|
|
574
|
+
}
|
|
575
|
+
onNotification(n) {
|
|
576
|
+
// A notification the server pushed is something that HAPPENED, and
|
|
577
|
+
// `mcp.notifications()` can be asserted on, so it gets a step of its
|
|
578
|
+
// own — nested under the call it arrived during, when there is one.
|
|
579
|
+
const seq = this.recording(() => recordStep({
|
|
580
|
+
kind: "mcp",
|
|
581
|
+
title: this.title(`notification ${n.method}`),
|
|
582
|
+
status: "passed",
|
|
583
|
+
parentSeq: this.activeSeq,
|
|
584
|
+
blocks: n.params === undefined ? [] : [{ type: "json", value: n.params, label: "Params" }],
|
|
585
|
+
}));
|
|
586
|
+
this.received.push({
|
|
587
|
+
method: n.method,
|
|
588
|
+
params: n.params,
|
|
589
|
+
atMs: Date.now() - this.createdAt,
|
|
590
|
+
seq,
|
|
591
|
+
});
|
|
592
|
+
}
|
|
593
|
+
/**
|
|
594
|
+
* Answer a server-to-client request.
|
|
595
|
+
*
|
|
596
|
+
* Recorded as a child of the call that provoked it (`parentSeq`), so a
|
|
597
|
+
* tool that asks the user something renders inside that tool's step
|
|
598
|
+
* rather than as a stray event somewhere below it.
|
|
599
|
+
*/
|
|
600
|
+
async onServerRequest(request) {
|
|
601
|
+
const parentSeq = this.activeSeq;
|
|
602
|
+
const started = Date.now();
|
|
603
|
+
const params = (request.params ?? {});
|
|
604
|
+
const finish = (title, blocks, error) => {
|
|
605
|
+
this.recording(() => recordStep({
|
|
606
|
+
kind: "mcp",
|
|
607
|
+
title: this.title(title),
|
|
608
|
+
status: error ? "failed" : "passed",
|
|
609
|
+
durationMs: Date.now() - started,
|
|
610
|
+
error,
|
|
611
|
+
parentSeq,
|
|
612
|
+
blocks,
|
|
613
|
+
}));
|
|
614
|
+
};
|
|
615
|
+
if (request.method === "sampling/createMessage" && this.opts.sampling) {
|
|
616
|
+
const req = params;
|
|
617
|
+
try {
|
|
618
|
+
const answer = await this.opts.sampling(req);
|
|
619
|
+
const result = samplingResult(answer);
|
|
620
|
+
finish("sampling", [
|
|
621
|
+
{
|
|
622
|
+
type: "chat",
|
|
623
|
+
messages: [
|
|
624
|
+
...(req.messages ?? []).map((m) => ({
|
|
625
|
+
side: (m.role === "assistant" ? "other" : "self"),
|
|
626
|
+
text: contentText(m.content),
|
|
627
|
+
})),
|
|
628
|
+
{ side: "other", text: contentText(result.content), new: true },
|
|
629
|
+
],
|
|
630
|
+
},
|
|
631
|
+
]);
|
|
632
|
+
return result;
|
|
633
|
+
}
|
|
634
|
+
catch (err) {
|
|
635
|
+
finish("sampling", [], errorMessage(err));
|
|
636
|
+
throw err;
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
if (request.method === "roots/list") {
|
|
640
|
+
const roots = (this.opts.roots ?? []).map((r) => typeof r === "string" ? { uri: r } : { uri: r.uri, name: r.name });
|
|
641
|
+
finish("list roots", [{ type: "json", value: roots, label: "Roots" }]);
|
|
642
|
+
return { roots };
|
|
643
|
+
}
|
|
644
|
+
if (request.method === "ping")
|
|
645
|
+
return {};
|
|
646
|
+
throw new Error(`the server asked for "${request.method}", which this client did not advertise. ` +
|
|
647
|
+
"Declare it on ctx.mcp(url, { … }) to answer it.");
|
|
648
|
+
}
|
|
649
|
+
/** Run one operation as a recorded step, with HTTP paused inside it. */
|
|
650
|
+
async step(title, run, opts) {
|
|
651
|
+
return (await this.runStep(title, run, opts)).value;
|
|
652
|
+
}
|
|
653
|
+
/** The same, with the result provenance-wrapped so an assertion on it
|
|
654
|
+
* nests under this step. */
|
|
655
|
+
async stepWrapped(title, run, opts) {
|
|
656
|
+
const { value, seq } = await this.runStep(title, run, opts);
|
|
657
|
+
return wrap(value, seq);
|
|
658
|
+
}
|
|
659
|
+
async runStep(title, run, opts) {
|
|
660
|
+
const resv = reserveEvent();
|
|
661
|
+
const started = Date.now();
|
|
662
|
+
this.pause();
|
|
663
|
+
let value;
|
|
664
|
+
try {
|
|
665
|
+
value = await run();
|
|
666
|
+
}
|
|
667
|
+
catch (err) {
|
|
668
|
+
this.resume();
|
|
669
|
+
const seq = recordStep({
|
|
670
|
+
kind: "mcp",
|
|
671
|
+
title: this.title(title),
|
|
672
|
+
status: "failed",
|
|
673
|
+
durationMs: Date.now() - started,
|
|
674
|
+
error: errorMessage(err),
|
|
675
|
+
blocks: [...this.failureBlocks(err), ...this.connectionBlock()],
|
|
676
|
+
}, resv);
|
|
677
|
+
this.noteChallenge(err, seq);
|
|
678
|
+
throw err;
|
|
679
|
+
}
|
|
680
|
+
this.resume();
|
|
681
|
+
const seq = recordStep({
|
|
682
|
+
kind: "mcp",
|
|
683
|
+
title: this.title(title),
|
|
684
|
+
status: "passed",
|
|
685
|
+
durationMs: Date.now() - started,
|
|
686
|
+
blocks: [
|
|
687
|
+
...(opts?.summary?.(value) ?? []),
|
|
688
|
+
...(opts?.connection === false ? [] : this.connectionBlock()),
|
|
689
|
+
],
|
|
690
|
+
}, resv);
|
|
691
|
+
return { value, seq };
|
|
692
|
+
}
|
|
693
|
+
/**
|
|
694
|
+
* Keep the `401` challenge a refused step carried, tagged with that
|
|
695
|
+
* step.
|
|
696
|
+
*
|
|
697
|
+
* This is what lets a test assert that a server really is protected —
|
|
698
|
+
* `expect(mcp.challenge?.status).toBe(401)` — with the assertion nested
|
|
699
|
+
* under the call that was refused. The thrown error carries the same
|
|
700
|
+
* information, but a caught error has no provenance to assert through.
|
|
701
|
+
*/
|
|
702
|
+
noteChallenge(err, seq) {
|
|
703
|
+
if (!(err instanceof McpHttpError) || err.status !== 401 || !err.challenge)
|
|
704
|
+
return;
|
|
705
|
+
this._challenge = err.challenge;
|
|
706
|
+
this.challengeSeq = seq;
|
|
707
|
+
}
|
|
708
|
+
/** Pause the recorder for the length of one operation's HTTP. */
|
|
709
|
+
pause() {
|
|
710
|
+
this.pauseDepth += 1;
|
|
711
|
+
pauseRecording();
|
|
712
|
+
}
|
|
713
|
+
resume() {
|
|
714
|
+
if (this.pauseDepth === 0)
|
|
715
|
+
return;
|
|
716
|
+
this.pauseDepth -= 1;
|
|
717
|
+
resumeRecording();
|
|
718
|
+
}
|
|
719
|
+
/** Run `fn` with this client's own pause lifted, then put it back. A
|
|
720
|
+
* no-op wrapper when we hold no pause. */
|
|
721
|
+
recording(fn) {
|
|
722
|
+
if (this.pauseDepth === 0)
|
|
723
|
+
return fn();
|
|
724
|
+
this.resume();
|
|
725
|
+
try {
|
|
726
|
+
return fn();
|
|
727
|
+
}
|
|
728
|
+
finally {
|
|
729
|
+
this.pause();
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
title(op) {
|
|
733
|
+
return this.opts.label ? `${this.opts.label}: ${op}` : op;
|
|
734
|
+
}
|
|
735
|
+
/**
|
|
736
|
+
* Which connection this step ran on — folded away.
|
|
737
|
+
*
|
|
738
|
+
* It is the same six rows on every step of a session, and none of them
|
|
739
|
+
* is why anyone opened the step. Above the content they buried it (the
|
|
740
|
+
* arguments of a tool call started below the fold); in a disclosure at
|
|
741
|
+
* the bottom they are one click away when a session id or a token
|
|
742
|
+
* actually is the question.
|
|
743
|
+
*/
|
|
744
|
+
connectionBlock() {
|
|
745
|
+
const rows = [["Server", this.url]];
|
|
746
|
+
if (this._serverInfo) {
|
|
747
|
+
rows.push(["Implementation", `${this._serverInfo.name} ${this._serverInfo.version}`]);
|
|
748
|
+
}
|
|
749
|
+
if (this._protocolVersion)
|
|
750
|
+
rows.push(["Protocol", this._protocolVersion]);
|
|
751
|
+
if (this.transport.sessionId)
|
|
752
|
+
rows.push(["Session", this.transport.sessionId]);
|
|
753
|
+
if (this._identity) {
|
|
754
|
+
rows.push(["Token", redactToken(this._identity.accessToken)]);
|
|
755
|
+
if (this._identity.scopes.length > 0)
|
|
756
|
+
rows.push(["Scopes", this._identity.scopes.join(" ")]);
|
|
757
|
+
}
|
|
758
|
+
return [{ type: "details", summary: "Connection", blocks: [kv(rows)] }];
|
|
759
|
+
}
|
|
760
|
+
/**
|
|
761
|
+
* What the handshake established.
|
|
762
|
+
*
|
|
763
|
+
* Shown on the two steps that perform one — the plain `connect`, and
|
|
764
|
+
* the `complete authorization` that re-runs it with the token — because
|
|
765
|
+
* `mcp.serverInfo` / `capabilities` / `protocolVersion` / `sessionId`
|
|
766
|
+
* are all wrapped against whichever of those ran, and a value a test can
|
|
767
|
+
* assert on has to be a value the reader can see.
|
|
768
|
+
*
|
|
769
|
+
* `fold` puts it in a disclosure, for the step whose own subject is
|
|
770
|
+
* something else.
|
|
771
|
+
*/
|
|
772
|
+
handshakeBlocks(fold) {
|
|
773
|
+
const blocks = [
|
|
774
|
+
kv([
|
|
775
|
+
["Server", this.url],
|
|
776
|
+
["Name", this._serverInfo?.name],
|
|
777
|
+
["Version", this._serverInfo?.version],
|
|
778
|
+
["Title", this._serverInfo?.title],
|
|
779
|
+
["Protocol", this._protocolVersion],
|
|
780
|
+
["Session", this.transport.sessionId],
|
|
781
|
+
["Capabilities", Object.keys(this._capabilities ?? {}).join(", ") || "none"],
|
|
782
|
+
]),
|
|
783
|
+
...(this._instructions
|
|
784
|
+
? [{ type: "text", text: this._instructions }]
|
|
785
|
+
: []),
|
|
786
|
+
// The capability map is nested, so a row cannot carry it — and a
|
|
787
|
+
// test may assert on any leaf of it.
|
|
788
|
+
...(this._capabilities && Object.keys(this._capabilities).length > 0
|
|
789
|
+
? [{ type: "json", value: this._capabilities, label: "Capabilities" }]
|
|
790
|
+
: []),
|
|
791
|
+
];
|
|
792
|
+
return fold ? [{ type: "details", summary: fold, blocks }] : blocks;
|
|
793
|
+
}
|
|
794
|
+
failureBlocks(err) {
|
|
795
|
+
const blocks = [];
|
|
796
|
+
if (err instanceof McpHttpError) {
|
|
797
|
+
blocks.push(kv([
|
|
798
|
+
["HTTP status", String(err.status)],
|
|
799
|
+
["Scheme", err.challenge?.scheme],
|
|
800
|
+
["Error", err.challenge?.error],
|
|
801
|
+
["Scope", err.challenge?.scope],
|
|
802
|
+
["Resource metadata", err.challenge?.resourceMetadataUrl],
|
|
803
|
+
["WWW-Authenticate", err.challenge?.raw],
|
|
804
|
+
]));
|
|
805
|
+
if (err.body) {
|
|
806
|
+
const body = err.body.slice(0, 4000);
|
|
807
|
+
const json = tryParse(body);
|
|
808
|
+
blocks.push(json === undefined
|
|
809
|
+
? { type: "code", code: body, label: "Response" }
|
|
810
|
+
: { type: "json", value: json, label: "Response" });
|
|
811
|
+
}
|
|
812
|
+
}
|
|
813
|
+
else if (err instanceof McpRpcError) {
|
|
814
|
+
blocks.push(kv([["JSON-RPC error", String(err.code)]]));
|
|
815
|
+
if (err.data !== undefined)
|
|
816
|
+
blocks.push({ type: "json", value: err.data, label: "Error data" });
|
|
817
|
+
}
|
|
818
|
+
else if (this.lastExchange) {
|
|
819
|
+
blocks.push(kv([["Last HTTP status", String(this.lastExchange.status)]]));
|
|
820
|
+
}
|
|
821
|
+
return blocks;
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
/**
|
|
825
|
+
* A tool call's arguments.
|
|
826
|
+
*
|
|
827
|
+
* A flat object — which is what most tool calls take — reads far better as
|
|
828
|
+
* a definition list than as a JSON blob, and it lines up with the rest of
|
|
829
|
+
* the panel. Anything nested keeps its JSON, where the shape matters.
|
|
830
|
+
*/
|
|
831
|
+
function argumentBlocks(args) {
|
|
832
|
+
const entries = Object.entries(args);
|
|
833
|
+
if (entries.length === 0)
|
|
834
|
+
return [];
|
|
835
|
+
const flat = entries.every(([, v]) => v === null || typeof v !== "object");
|
|
836
|
+
if (!flat)
|
|
837
|
+
return [{ type: "json", value: args, label: "Arguments" }];
|
|
838
|
+
return [
|
|
839
|
+
{
|
|
840
|
+
type: "kv",
|
|
841
|
+
label: "Arguments",
|
|
842
|
+
rows: entries.map(([label, value]) => ({ label, value: String(value) })),
|
|
843
|
+
},
|
|
844
|
+
];
|
|
845
|
+
}
|
|
846
|
+
function makeToolResult(raw, content, text, isError, seq) {
|
|
847
|
+
return {
|
|
848
|
+
isError: wrap(isError, seq, ["isError"]),
|
|
849
|
+
content: wrap(content, seq, ["content"]),
|
|
850
|
+
text: wrap(text, seq, ["text"]),
|
|
851
|
+
structured: raw.structuredContent === undefined
|
|
852
|
+
? undefined
|
|
853
|
+
: wrap(raw.structuredContent, seq, ["structured"]),
|
|
854
|
+
json() {
|
|
855
|
+
if (raw.structuredContent !== undefined) {
|
|
856
|
+
return wrap(raw.structuredContent, seq, ["json()"]);
|
|
857
|
+
}
|
|
858
|
+
let parsed;
|
|
859
|
+
try {
|
|
860
|
+
parsed = JSON.parse(text);
|
|
861
|
+
}
|
|
862
|
+
catch {
|
|
863
|
+
throw new Error("the tool returned no structuredContent and its text is not JSON. " +
|
|
864
|
+
"Assert on res.text, or read res.content.");
|
|
865
|
+
}
|
|
866
|
+
return wrap(parsed, seq, ["json()"]);
|
|
867
|
+
},
|
|
868
|
+
unwrap: () => ({ isError, content, structuredContent: raw.structuredContent }),
|
|
869
|
+
};
|
|
870
|
+
}
|
|
871
|
+
/** Join the text parts. The usual thing a test asserts on. */
|
|
872
|
+
function textOf(content) {
|
|
873
|
+
return content
|
|
874
|
+
.filter((c) => c.type === "text")
|
|
875
|
+
.map((c) => c.text ?? "")
|
|
876
|
+
.join("\n");
|
|
877
|
+
}
|
|
878
|
+
function contentText(content) {
|
|
879
|
+
if (!content)
|
|
880
|
+
return "";
|
|
881
|
+
if (content.type === "text")
|
|
882
|
+
return content.text ?? "";
|
|
883
|
+
if (content.type === "image" || content.type === "audio") {
|
|
884
|
+
return `[${content.type} ${content.mimeType ?? ""}]`;
|
|
885
|
+
}
|
|
886
|
+
return JSON.stringify(content);
|
|
887
|
+
}
|
|
888
|
+
/**
|
|
889
|
+
* Render a tool result.
|
|
890
|
+
*
|
|
891
|
+
* A tool that declares an output schema usually returns the SAME value
|
|
892
|
+
* twice — once as `structuredContent` and once as a JSON text part, since
|
|
893
|
+
* a client that does not read structured output still has to see
|
|
894
|
+
* something. Rendering both is noise, so a text part that parses to the
|
|
895
|
+
* structured value is dropped.
|
|
896
|
+
*
|
|
897
|
+
* An image renders as a line of metadata rather than a picture:
|
|
898
|
+
* `blocks.rs` has no image block yet. When one is added, this is the only
|
|
899
|
+
* place that changes.
|
|
900
|
+
*/
|
|
901
|
+
function resultBlocks(content, structured, shownAsError) {
|
|
902
|
+
const blocks = [];
|
|
903
|
+
// Text parts a reader does not need when `structuredContent` already
|
|
904
|
+
// answers the question. Folded, not dropped: what the model would have
|
|
905
|
+
// read is still one click away.
|
|
906
|
+
const folded = [];
|
|
907
|
+
if (structured !== undefined)
|
|
908
|
+
blocks.push({ type: "json", value: structured, label: "Result" });
|
|
909
|
+
const textCount = content.filter((c) => c.type === "text").length;
|
|
910
|
+
const label = (index) => (textCount > 1 ? `Text ${index + 1}` : "Result");
|
|
911
|
+
let textIndex = 0;
|
|
912
|
+
for (const part of content) {
|
|
913
|
+
if (part.type === "text") {
|
|
914
|
+
const text = part.text ?? "";
|
|
915
|
+
const json = tryParse(text);
|
|
916
|
+
const index = textIndex++;
|
|
917
|
+
// The same value twice — a tool with an output schema usually
|
|
918
|
+
// returns both forms. Once is enough.
|
|
919
|
+
if (json !== undefined && structured !== undefined && sameJson(json, structured))
|
|
920
|
+
continue;
|
|
921
|
+
// Already the step's error line.
|
|
922
|
+
if (shownAsError !== undefined && text === shownAsError)
|
|
923
|
+
continue;
|
|
924
|
+
const block = json === undefined
|
|
925
|
+
? { type: "code", code: text, label: label(index) }
|
|
926
|
+
: { type: "json", value: json, label: label(index) };
|
|
927
|
+
(structured === undefined ? blocks : folded).push(block);
|
|
928
|
+
continue;
|
|
929
|
+
}
|
|
930
|
+
if (part.type === "image" || part.type === "audio") {
|
|
931
|
+
const p = part;
|
|
932
|
+
blocks.push({
|
|
933
|
+
type: "kv",
|
|
934
|
+
rows: [{ label: part.type, value: `${p.mimeType ?? "unknown type"}, ${byteSize(p.data)}` }],
|
|
935
|
+
});
|
|
936
|
+
continue;
|
|
937
|
+
}
|
|
938
|
+
if (part.type === "resource") {
|
|
939
|
+
blocks.push(...resourceBlocks(part.resource));
|
|
940
|
+
continue;
|
|
941
|
+
}
|
|
942
|
+
blocks.push({ type: "json", value: part, label: part.type });
|
|
943
|
+
}
|
|
944
|
+
if (folded.length > 0) {
|
|
945
|
+
blocks.push({ type: "details", summary: "Text content", blocks: folded });
|
|
946
|
+
}
|
|
947
|
+
return blocks;
|
|
948
|
+
}
|
|
949
|
+
/**
|
|
950
|
+
* A folded block carrying a value in full.
|
|
951
|
+
*
|
|
952
|
+
* The rule this serves: a test can assert on any field of what a call
|
|
953
|
+
* returned, so every field has to be somewhere a reader can reach. A
|
|
954
|
+
* table or a set of bubbles shows what matters; this keeps the rest one
|
|
955
|
+
* click away instead of nowhere.
|
|
956
|
+
*/
|
|
957
|
+
function rawBlock(summary, value) {
|
|
958
|
+
if (Array.isArray(value) && value.length === 0)
|
|
959
|
+
return [];
|
|
960
|
+
return [{ type: "details", summary, blocks: [{ type: "json", value }] }];
|
|
961
|
+
}
|
|
962
|
+
/** Structural equality, for the duplicate-result check above. */
|
|
963
|
+
function sameJson(a, b) {
|
|
964
|
+
try {
|
|
965
|
+
return JSON.stringify(a) === JSON.stringify(b);
|
|
966
|
+
}
|
|
967
|
+
catch {
|
|
968
|
+
return false;
|
|
969
|
+
}
|
|
970
|
+
}
|
|
971
|
+
function resourceBlocks(contents) {
|
|
972
|
+
const blocks = [
|
|
973
|
+
kv([
|
|
974
|
+
["URI", contents.uri],
|
|
975
|
+
["Type", contents.mimeType],
|
|
976
|
+
]),
|
|
977
|
+
];
|
|
978
|
+
if (contents.text !== undefined) {
|
|
979
|
+
const json = tryParse(contents.text);
|
|
980
|
+
blocks.push(json === undefined
|
|
981
|
+
? { type: "code", code: contents.text, lang: langOf(contents.mimeType), label: "Contents" }
|
|
982
|
+
: { type: "json", value: json, label: "Contents" });
|
|
983
|
+
}
|
|
984
|
+
else if (contents.blob !== undefined) {
|
|
985
|
+
blocks.push({ type: "text", text: `binary contents, ${byteSize(contents.blob)}` });
|
|
986
|
+
}
|
|
987
|
+
return blocks;
|
|
988
|
+
}
|
|
989
|
+
function samplingResult(answer) {
|
|
990
|
+
if (typeof answer === "string") {
|
|
991
|
+
return { role: "assistant", content: { type: "text", text: answer }, model: "spectest" };
|
|
992
|
+
}
|
|
993
|
+
const content = answer.content ?? { type: "text", text: answer.text ?? "" };
|
|
994
|
+
return {
|
|
995
|
+
role: answer.role ?? "assistant",
|
|
996
|
+
content,
|
|
997
|
+
model: answer.model ?? "spectest",
|
|
998
|
+
stopReason: answer.stopReason,
|
|
999
|
+
};
|
|
1000
|
+
}
|
|
1001
|
+
function isConnectionError(err) {
|
|
1002
|
+
const message = err?.message ?? String(err);
|
|
1003
|
+
return /ECONNRESET|ECONNREFUSED|EPIPE|socket|fetch failed|Unable to connect|closed/i.test(message);
|
|
1004
|
+
}
|
|
1005
|
+
function errorMessage(err) {
|
|
1006
|
+
if (err === undefined || err === null)
|
|
1007
|
+
return undefined;
|
|
1008
|
+
return err?.message ?? String(err);
|
|
1009
|
+
}
|
|
1010
|
+
/** Show that a token exists and let two tokens be told apart, without
|
|
1011
|
+
* putting a credential in a run that is stored forever. */
|
|
1012
|
+
function redactToken(token) {
|
|
1013
|
+
return token.length <= 8 ? "••••" : `••••${token.slice(-4)}`;
|
|
1014
|
+
}
|
|
1015
|
+
/**
|
|
1016
|
+
* A wrapped view of a value this client read, tagged with the step that
|
|
1017
|
+
* produced it. `wrap` is typed as the identity function (it returns a
|
|
1018
|
+
* proxy that behaves like the value), so the wrapped TYPE is asserted
|
|
1019
|
+
* here — in one place, rather than at every getter.
|
|
1020
|
+
*/
|
|
1021
|
+
function wrapped(value, seq) {
|
|
1022
|
+
return value === undefined ? undefined : wrap(value, seq);
|
|
1023
|
+
}
|
|
1024
|
+
function kv(rows, label) {
|
|
1025
|
+
return {
|
|
1026
|
+
type: "kv",
|
|
1027
|
+
label,
|
|
1028
|
+
rows: rows.filter(([, value]) => value !== undefined).map(([label, value]) => ({ label, value })),
|
|
1029
|
+
};
|
|
1030
|
+
}
|
|
1031
|
+
function tryParse(text) {
|
|
1032
|
+
const trimmed = text.trim();
|
|
1033
|
+
if (!trimmed.startsWith("{") && !trimmed.startsWith("["))
|
|
1034
|
+
return undefined;
|
|
1035
|
+
try {
|
|
1036
|
+
return JSON.parse(trimmed);
|
|
1037
|
+
}
|
|
1038
|
+
catch {
|
|
1039
|
+
return undefined;
|
|
1040
|
+
}
|
|
1041
|
+
}
|
|
1042
|
+
function langOf(mimeType) {
|
|
1043
|
+
if (!mimeType)
|
|
1044
|
+
return undefined;
|
|
1045
|
+
if (mimeType.includes("json"))
|
|
1046
|
+
return "json";
|
|
1047
|
+
if (mimeType.includes("html"))
|
|
1048
|
+
return "html";
|
|
1049
|
+
if (mimeType.includes("markdown"))
|
|
1050
|
+
return "markdown";
|
|
1051
|
+
if (mimeType.includes("yaml"))
|
|
1052
|
+
return "yaml";
|
|
1053
|
+
return undefined;
|
|
1054
|
+
}
|
|
1055
|
+
function byteSize(base64) {
|
|
1056
|
+
if (!base64)
|
|
1057
|
+
return "0 bytes";
|
|
1058
|
+
const bytes = Math.floor((base64.length * 3) / 4);
|
|
1059
|
+
return bytes < 1024 ? `${bytes} bytes` : `${(bytes / 1024).toFixed(1)} kB`;
|
|
1060
|
+
}
|