@skybridge/test 0.0.0-dev.f7943e6 → 0.0.0-next.0f12579
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -12
- package/dist/chat.d.ts +14 -3
- package/dist/chat.d.ts.map +1 -1
- package/dist/chat.js +9 -6
- package/dist/chat.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/session-registry.d.ts +44 -9
- package/dist/session-registry.d.ts.map +1 -1
- package/dist/session-registry.js +28 -14
- package/dist/session-registry.js.map +1 -1
- package/package.json +8 -5
package/README.md
CHANGED
|
@@ -4,17 +4,20 @@ Evals for [Skybridge](https://skybridge.tech) MCP apps: they answer whether a
|
|
|
4
4
|
prompt reaches the right tool with the right arguments, which no unit test can
|
|
5
5
|
check.
|
|
6
6
|
|
|
7
|
-
Add the `evals` option to the Skybridge Vite plugin
|
|
8
|
-
vitest's own `it` and `expect`.
|
|
7
|
+
Add the `evals` option to the Skybridge Vite plugin so `expect.chat` exists,
|
|
8
|
+
then write scenarios with vitest's own `it` and `expect`. Pass the app itself
|
|
9
|
+
to `start`: it is served in-process, dialing an in-process handler built on `app.createServerInstance()`, and
|
|
10
|
+
the tool names and argument shapes come from the app value, so no type
|
|
11
|
+
parameter is needed.
|
|
9
12
|
|
|
10
13
|
```ts
|
|
11
14
|
import { anthropic } from "@ai-sdk/anthropic";
|
|
12
15
|
import { expect, it } from "vitest";
|
|
13
16
|
import { start } from "@skybridge/test";
|
|
14
|
-
import
|
|
17
|
+
import { app } from "../src/server.js";
|
|
15
18
|
|
|
16
19
|
it("finds products by category", async () => {
|
|
17
|
-
const chat = await start
|
|
20
|
+
const chat = await start({ app, model: anthropic("claude-sonnet-4-5") });
|
|
18
21
|
await chat.send("I am looking for ski goggles");
|
|
19
22
|
|
|
20
23
|
expect.chat(chat).toHaveCalledToolWith("search-products", {
|
|
@@ -23,6 +26,30 @@ it("finds products by category", async () => {
|
|
|
23
26
|
});
|
|
24
27
|
```
|
|
25
28
|
|
|
29
|
+
There is no child process and no server to boot, so the evals run anywhere
|
|
30
|
+
vitest does. The caveat is that they bypass the Express layer, so nothing
|
|
31
|
+
verifies a bearer token; pass `authInfo` to claim an identity.
|
|
32
|
+
|
|
33
|
+
## Authenticated evals
|
|
34
|
+
|
|
35
|
+
```ts
|
|
36
|
+
const chat = await start({
|
|
37
|
+
app,
|
|
38
|
+
model: anthropic("claude-sonnet-4-5"),
|
|
39
|
+
authInfo: {
|
|
40
|
+
token: "eval-token",
|
|
41
|
+
clientId: "eval-client",
|
|
42
|
+
scopes: ["checkout"],
|
|
43
|
+
extra: { sub: "user_123", email: "ada@example.com" },
|
|
44
|
+
},
|
|
45
|
+
});
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
`authInfo` injects a claimed identity: the app's own scheme and scope
|
|
49
|
+
enforcement runs for real, in-band challenges included, and only token
|
|
50
|
+
signature verification is skipped. Omit it to eval the anonymous path, which is
|
|
51
|
+
how you assert that a protected tool challenges instead of running.
|
|
52
|
+
|
|
26
53
|
## Choosing a model
|
|
27
54
|
|
|
28
55
|
`start` takes any AI SDK model instance, so the provider and its credentials
|
|
@@ -32,7 +59,7 @@ are the project's own. Install the provider package you want alongside `ai`.
|
|
|
32
59
|
import { anthropic } from "@ai-sdk/anthropic";
|
|
33
60
|
import { start } from "@skybridge/test";
|
|
34
61
|
|
|
35
|
-
const chat = await start
|
|
62
|
+
const chat = await start({ app, model: anthropic("claude-sonnet-4-5") });
|
|
36
63
|
```
|
|
37
64
|
|
|
38
65
|
A local runtime or a company gateway works the same way, through the AI SDK's
|
|
@@ -45,19 +72,17 @@ const ollama = createOpenAICompatible({
|
|
|
45
72
|
name: "ollama",
|
|
46
73
|
baseURL: "http://localhost:11434/v1",
|
|
47
74
|
});
|
|
48
|
-
const chat = await start
|
|
75
|
+
const chat = await start({ app, model: ollama("llama3.1") });
|
|
49
76
|
```
|
|
50
77
|
|
|
51
|
-
|
|
78
|
+
The plugin's `evals` option also carries the defaults every conversation starts
|
|
79
|
+
from, so a project can set them once instead of per scenario.
|
|
52
80
|
|
|
53
81
|
```ts
|
|
54
|
-
skybridge({
|
|
55
|
-
evals: { project: { cwd: ".", command: ["node", "dist/server.js"] } },
|
|
56
|
-
});
|
|
82
|
+
skybridge({ evals: { temperature: 0, maxSteps: 6 } });
|
|
57
83
|
```
|
|
58
84
|
|
|
59
|
-
|
|
60
|
-
`pnpm run` do not forward the channel the runner uses to learn the port.
|
|
85
|
+
## Matchers
|
|
61
86
|
|
|
62
87
|
Assertions on what the assistant said work the same way, though model prose
|
|
63
88
|
varies between runs, so keep them loose.
|
package/dist/chat.d.ts
CHANGED
|
@@ -6,7 +6,18 @@ interface HostConfig {
|
|
|
6
6
|
systemPrompt?: string;
|
|
7
7
|
maxSteps?: number;
|
|
8
8
|
}
|
|
9
|
+
/**
|
|
10
|
+
* A `fetch` replacement the transport dials instead of the network. Mirrors the
|
|
11
|
+
* MCP client's own `fetch` option so an in-process app can serve the session.
|
|
12
|
+
*/
|
|
13
|
+
export type TransportFetch = (url: string | URL, init?: RequestInit) => Promise<Response>;
|
|
14
|
+
/**
|
|
15
|
+
* One conversation against one MCP session. A scenario asserts on
|
|
16
|
+
* {@link Chat.toolCalls}, the sequence of calls the model made.
|
|
17
|
+
*/
|
|
9
18
|
export declare class Chat<App = unknown> {
|
|
19
|
+
/** Type-only anchor so `expect.chat` infers `App` from the conversation. */
|
|
20
|
+
readonly $app: App;
|
|
10
21
|
/** Every call the model made, in order, as it crossed the wire. */
|
|
11
22
|
readonly toolCalls: ToolCall<App>[];
|
|
12
23
|
/** What the assistant said, one entry per turn, intermediate ones included. */
|
|
@@ -18,9 +29,9 @@ export declare class Chat<App = unknown> {
|
|
|
18
29
|
private readonly host;
|
|
19
30
|
private tools;
|
|
20
31
|
private constructor();
|
|
21
|
-
static open<App>(
|
|
22
|
-
/** Takes a turn
|
|
23
|
-
send(prompt: string): Promise<
|
|
32
|
+
static open<App>(host: HostConfig, fetchImpl: TransportFetch): Promise<Chat<App>>;
|
|
33
|
+
/** Takes a turn. Assert on the result with `expect.chat`. */
|
|
34
|
+
send(prompt: string): Promise<void>;
|
|
24
35
|
close(): Promise<void>;
|
|
25
36
|
private failureFor;
|
|
26
37
|
private toolSet;
|
package/dist/chat.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAIL,KAAK,aAAa,EAGnB,MAAM,IAAI,CAAC;AAEZ,UAAU,UAAU;IAClB,KAAK,EAAE,aAAa,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAMD;;;GAGG;AACH,MAAM,MAAM,cAAc,GAAG,CAC3B,GAAG,EAAE,MAAM,GAAG,GAAG,EACjB,IAAI,CAAC,EAAE,WAAW,KACf,OAAO,CAAC,QAAQ,CAAC,CAAC;AAEvB;;;GAGG;AACH,qBAAa,IAAI,CAAC,GAAG,GAAG,OAAO;IAC7B,4EAA4E;IAC5E,SAAiB,IAAI,EAAE,GAAG,CAAC;IAE3B,mEAAmE;IACnE,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAM;IAEzC,+EAA+E;IAC/E,QAAQ,CAAC,cAAc,EAAE,MAAM,EAAE,CAAM;IAEvC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA6B;IACtD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAa;IAClC,OAAO,CAAC,KAAK,CAAc;IAE3B,OAAO;WAUM,IAAI,CAAC,GAAG,EACnB,IAAI,EAAE,UAAU,EAChB,SAAS,EAAE,cAAc,GACxB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAkBrB,6DAA6D;IACvD,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAiCnC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,OAAO,CAAC,UAAU;IAalB,OAAO,CAAC,OAAO;CAqChB"}
|
package/dist/chat.js
CHANGED
|
@@ -1,7 +1,11 @@
|
|
|
1
|
-
import { Client } from "@modelcontextprotocol/
|
|
2
|
-
import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
|
|
1
|
+
import { Client, StreamableHTTPClientTransport, } from "@modelcontextprotocol/client";
|
|
3
2
|
import { dynamicTool, generateText, jsonSchema, stepCountIs, } from "ai";
|
|
4
3
|
const DEFAULT_MAX_STEPS = 8;
|
|
4
|
+
const IN_PROCESS_URL = "http://in-process.skybridge.test/mcp";
|
|
5
|
+
/**
|
|
6
|
+
* One conversation against one MCP session. A scenario asserts on
|
|
7
|
+
* {@link Chat.toolCalls}, the sequence of calls the model made.
|
|
8
|
+
*/
|
|
5
9
|
export class Chat {
|
|
6
10
|
/** Every call the model made, in order, as it crossed the wire. */
|
|
7
11
|
toolCalls = [];
|
|
@@ -18,9 +22,9 @@ export class Chat {
|
|
|
18
22
|
this.transport = transport;
|
|
19
23
|
this.host = host;
|
|
20
24
|
}
|
|
21
|
-
static async open(
|
|
25
|
+
static async open(host, fetchImpl) {
|
|
22
26
|
const client = new Client({ name: "skybridge-eval", version: "0" });
|
|
23
|
-
const transport = new StreamableHTTPClientTransport(new URL(
|
|
27
|
+
const transport = new StreamableHTTPClientTransport(new URL(IN_PROCESS_URL), { fetch: fetchImpl });
|
|
24
28
|
await client.connect(transport);
|
|
25
29
|
const chat = new Chat(client, transport, host);
|
|
26
30
|
try {
|
|
@@ -33,7 +37,7 @@ export class Chat {
|
|
|
33
37
|
}
|
|
34
38
|
return chat;
|
|
35
39
|
}
|
|
36
|
-
/** Takes a turn
|
|
40
|
+
/** Takes a turn. Assert on the result with `expect.chat`. */
|
|
37
41
|
async send(prompt) {
|
|
38
42
|
this.messages.push({ role: "user", content: prompt });
|
|
39
43
|
const result = await generateText({
|
|
@@ -61,7 +65,6 @@ export class Chat {
|
|
|
61
65
|
}
|
|
62
66
|
}
|
|
63
67
|
this.messages.push(...result.response.messages);
|
|
64
|
-
return result.text;
|
|
65
68
|
}
|
|
66
69
|
async close() {
|
|
67
70
|
await this.transport.close();
|
package/dist/chat.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,MAAM,EACN,6BAA6B,GAE9B,MAAM,8BAA8B,CAAC;AAEtC,OAAO,EACL,WAAW,EACX,YAAY,EACZ,UAAU,EAGV,WAAW,GACZ,MAAM,IAAI,CAAC;AASZ,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B,MAAM,cAAc,GAAG,sCAAsC,CAAC;AAW9D;;;GAGG;AACH,MAAM,OAAO,IAAI;IAIf,mEAAmE;IAC1D,SAAS,GAAoB,EAAE,CAAC;IAEzC,+EAA+E;IACtE,cAAc,GAAa,EAAE,CAAC;IAEtB,QAAQ,GAAmB,EAAE,CAAC;IAC9B,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrC,MAAM,CAAS;IACf,SAAS,CAAgC;IACzC,IAAI,CAAa;IAC1B,KAAK,GAAW,EAAE,CAAC;IAE3B,YACE,MAAc,EACd,SAAwC,EACxC,IAAgB;QAEhB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,IAAI,CACf,IAAgB,EAChB,SAAyB;QAEzB,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,IAAI,6BAA6B,CACjD,IAAI,GAAG,CAAC,cAAc,CAAC,EACvB,EAAE,KAAK,EAAE,SAAS,EAAE,CACrB,CAAC;QACF,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAM,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QACpD,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACrB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YACxB,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,6DAA6D;IAC7D,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAEtD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;YACtB,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC;YACvC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,KAAK,SAAS;gBACtC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE;YACrB,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;SAC/D,CAAC,CAAC;QAEH,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAChC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAClC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;oBAClB,IAAI,EAAE,IAAI,CAAC,QAAQ;oBACnB,SAAS,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAA4B;oBACxD,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;iBACR,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,IAAI,CAAC,IAAI,KAAK,EAAE,EAAE,CAAC;gBACrB,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;IAEO,UAAU,CAAC,IAA+C;QAGhE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QAC9B,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC1B,OAAO,EAAE,MAAM,EAAE,2CAA2C,EAAE,CAAC;QACjE,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAEO,OAAO;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;YAC5C,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,UAAU,EAAE,GAAG,UAAU,CAAC,WAAW,CAAC;YACpE,OAAO;gBACL,UAAU,CAAC,IAAI;gBACf,WAAW,CAAC;oBACV,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,EAAE;oBACzC,WAAW,EAAE,UAAU,CACrB,UAA8C,CAC/C;oBACD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE;wBACvC,IAAI,MAA+C,CAAC;wBACpD,IAAI,CAAC;4BACH,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;gCAClC,IAAI,EAAE,UAAU,CAAC,IAAI;gCACrB,SAAS,EAAE,CAAC,KAAK,IAAI,EAAE,CAA4B;6BACpD,CAAC,CAAC;wBACL,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BACf,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,sCAAsC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC/F,CAAC;4BACF,MAAM,KAAK,CAAC;wBACd,CAAC;wBACD,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;4BAC5B,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,+BAA+B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAChE,CAAC;wBACJ,CAAC;wBACD,OAAO,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC;oBAClC,CAAC;iBACF,CAAC;aACM,CAAC;QACb,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;CACF"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
export type { ChatAssertion, ChatMatchers, } from "@skybridge/vite-plugin/evals";
|
|
2
2
|
export type { Chat } from "./chat.js";
|
|
3
|
+
export type { EvalApp, EvalIdentity } from "./session-registry.js";
|
|
3
4
|
export { start } from "./session-registry.js";
|
|
4
5
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,aAAa,EACb,YAAY,GACb,MAAM,8BAA8B,CAAC;AACtC,YAAY,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,aAAa,EACb,YAAY,GACb,MAAM,8BAA8B,CAAC;AACtC,YAAY,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACtC,YAAY,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACnE,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
|
|
@@ -1,20 +1,55 @@
|
|
|
1
|
+
import { type Server } from "@modelcontextprotocol/server";
|
|
1
2
|
import type { LanguageModel } from "ai";
|
|
2
3
|
import { Chat } from "./chat.js";
|
|
3
4
|
/**
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
5
|
+
* The minimum a Skybridge app has to expose to be served in-process: the
|
|
6
|
+
* `$types` marker the assertions infer tool names from, and the per-request
|
|
7
|
+
* server builder the session's handler is built on. Structural on purpose, so
|
|
8
|
+
* this package needs no runtime dependency on `skybridge`.
|
|
9
|
+
*/
|
|
10
|
+
export interface EvalApp {
|
|
11
|
+
readonly $types: {
|
|
12
|
+
readonly tools: object;
|
|
13
|
+
};
|
|
14
|
+
createServerInstance(): Promise<Server>;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* The identity an in-process session claims, handed to the app as the
|
|
18
|
+
* request's `authInfo`. Mirrors the SDK's `AuthInfo` structurally; `extra`
|
|
19
|
+
* carries whatever claims the app's verifier would have produced.
|
|
12
20
|
*/
|
|
13
|
-
export
|
|
21
|
+
export interface EvalIdentity {
|
|
22
|
+
token: string;
|
|
23
|
+
clientId: string;
|
|
24
|
+
scopes: string[];
|
|
25
|
+
expiresAt?: number;
|
|
26
|
+
resource?: URL;
|
|
27
|
+
extra?: Record<string, unknown>;
|
|
28
|
+
}
|
|
29
|
+
interface StartOptions {
|
|
14
30
|
/** Any AI SDK model instance, built by the project. */
|
|
15
31
|
model: LanguageModel;
|
|
16
32
|
systemPrompt?: string;
|
|
17
33
|
temperature?: number;
|
|
18
34
|
maxSteps?: number;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Opens a fresh MCP session and conversation against the app under test. The
|
|
38
|
+
* app is served in-process by a handler built on `app.createServerInstance`
|
|
39
|
+
* for this conversation alone, and both are closed when the current test
|
|
40
|
+
* finishes, so scenarios never leak sessions and never write teardown, and
|
|
41
|
+
* tests that run concurrently cannot close each other's sessions.
|
|
42
|
+
*
|
|
43
|
+
* The assertions are inferred from the app value itself, so `expect.chat` gets
|
|
44
|
+
* the project's tool names and argument shapes with no type parameter.
|
|
45
|
+
*
|
|
46
|
+
* `authInfo` claims an identity for the session: the app's per-tool scheme and
|
|
47
|
+
* scope enforcement runs against it for real, only token verification is
|
|
48
|
+
* skipped. Omit it to exercise the anonymous path, challenges included.
|
|
49
|
+
*/
|
|
50
|
+
export declare function start<App extends EvalApp>(options: StartOptions & {
|
|
51
|
+
app: App;
|
|
52
|
+
authInfo?: EvalIdentity;
|
|
19
53
|
}): Promise<Chat<App>>;
|
|
54
|
+
export {};
|
|
20
55
|
//# sourceMappingURL=session-registry.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAExC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC
|
|
1
|
+
{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAoB,KAAK,MAAM,EAAE,MAAM,8BAA8B,CAAC;AAC7E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAExC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC;;;;;GAKG;AACH,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5C,oBAAoB,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;CACzC;AAED;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,GAAG,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACjC;AAED,UAAU,YAAY;IACpB,uDAAuD;IACvD,KAAK,EAAE,aAAa,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAUD;;;;;;;;;;;;;GAaG;AACH,wBAAsB,KAAK,CAAC,GAAG,SAAS,OAAO,EAC7C,OAAO,EAAE,YAAY,GAAG;IAAE,GAAG,EAAE,GAAG,CAAC;IAAC,QAAQ,CAAC,EAAE,YAAY,CAAA;CAAE,GAC5D,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAwBpB"}
|
package/dist/session-registry.js
CHANGED
|
@@ -1,28 +1,42 @@
|
|
|
1
|
+
import { createMcpHandler } from "@modelcontextprotocol/server";
|
|
1
2
|
import { inject, onTestFinished } from "vitest";
|
|
2
3
|
import { Chat } from "./chat.js";
|
|
4
|
+
function sharedDefaults() {
|
|
5
|
+
try {
|
|
6
|
+
return inject("skybridgeEvals");
|
|
7
|
+
}
|
|
8
|
+
catch {
|
|
9
|
+
return undefined;
|
|
10
|
+
}
|
|
11
|
+
}
|
|
3
12
|
/**
|
|
4
|
-
* Opens a fresh MCP session and conversation against the
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
13
|
+
* Opens a fresh MCP session and conversation against the app under test. The
|
|
14
|
+
* app is served in-process by a handler built on `app.createServerInstance`
|
|
15
|
+
* for this conversation alone, and both are closed when the current test
|
|
16
|
+
* finishes, so scenarios never leak sessions and never write teardown, and
|
|
17
|
+
* tests that run concurrently cannot close each other's sessions.
|
|
8
18
|
*
|
|
9
|
-
* The
|
|
10
|
-
*
|
|
11
|
-
*
|
|
19
|
+
* The assertions are inferred from the app value itself, so `expect.chat` gets
|
|
20
|
+
* the project's tool names and argument shapes with no type parameter.
|
|
21
|
+
*
|
|
22
|
+
* `authInfo` claims an identity for the session: the app's per-tool scheme and
|
|
23
|
+
* scope enforcement runs against it for real, only token verification is
|
|
24
|
+
* skipped. Omit it to exercise the anonymous path, challenges included.
|
|
12
25
|
*/
|
|
13
26
|
export async function start(options) {
|
|
14
|
-
const
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
const config = inject("skybridgeEvals");
|
|
19
|
-
const chat = await Chat.open(url, {
|
|
27
|
+
const config = sharedDefaults();
|
|
28
|
+
const { app, authInfo } = options;
|
|
29
|
+
const handler = createMcpHandler(() => app.createServerInstance());
|
|
30
|
+
const chat = await Chat.open({
|
|
20
31
|
model: options.model,
|
|
21
32
|
temperature: options.temperature ?? config?.temperature,
|
|
22
33
|
systemPrompt: options.systemPrompt ?? config?.systemPrompt,
|
|
23
34
|
maxSteps: options.maxSteps ?? config?.maxSteps,
|
|
35
|
+
}, (url, init) => handler.fetch(new Request(url, init), authInfo === undefined ? undefined : { authInfo }));
|
|
36
|
+
onTestFinished(async () => {
|
|
37
|
+
await chat.close();
|
|
38
|
+
await handler.close();
|
|
24
39
|
});
|
|
25
|
-
onTestFinished(() => chat.close());
|
|
26
40
|
return chat;
|
|
27
41
|
}
|
|
28
42
|
//# sourceMappingURL=session-registry.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-registry.js","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"session-registry.js","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAe,MAAM,8BAA8B,CAAC;AAE7E,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAmCjC,SAAS,cAAc;IACrB,IAAI,CAAC;QACH,OAAO,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CACzB,OAA6D;IAE7D,MAAM,MAAM,GAAG,cAAc,EAAE,CAAC;IAChC,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAClC,MAAM,OAAO,GAAG,gBAAgB,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,oBAAoB,EAAE,CAAC,CAAC;IAEnE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAC1B;QACE,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,MAAM,EAAE,WAAW;QACvD,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,MAAM,EAAE,YAAY;QAC1D,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,MAAM,EAAE,QAAQ;KAC/C,EACD,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CACZ,OAAO,CAAC,KAAK,CACX,IAAI,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,EACtB,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAClD,CACJ,CAAC;IAEF,cAAc,CAAC,KAAK,IAAI,EAAE;QACxB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC,CAAC,CAAC;IACH,OAAO,IAAI,CAAC;AACd,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@skybridge/test",
|
|
3
|
-
"version": "0.0.0-
|
|
3
|
+
"version": "0.0.0-next.0f12579",
|
|
4
4
|
"description": "Eval runner for Skybridge MCP apps: does a prompt reach the right tool with the right arguments",
|
|
5
5
|
"license": "ISC",
|
|
6
6
|
"type": "module",
|
|
@@ -18,12 +18,13 @@
|
|
|
18
18
|
"dist"
|
|
19
19
|
],
|
|
20
20
|
"dependencies": {
|
|
21
|
-
"@modelcontextprotocol/
|
|
22
|
-
"@
|
|
21
|
+
"@modelcontextprotocol/client": "^2.0.0",
|
|
22
|
+
"@modelcontextprotocol/server": "^2.0.0"
|
|
23
23
|
},
|
|
24
24
|
"peerDependencies": {
|
|
25
|
+
"@skybridge/vite-plugin": "^2.0.0-beta.0",
|
|
25
26
|
"ai": "^5.0.0",
|
|
26
|
-
"skybridge": "^2.0.0",
|
|
27
|
+
"skybridge": "^2.0.0-beta.0",
|
|
27
28
|
"vitest": "^4.1.0"
|
|
28
29
|
},
|
|
29
30
|
"devDependencies": {
|
|
@@ -31,7 +32,9 @@
|
|
|
31
32
|
"ai": "^5.0.0",
|
|
32
33
|
"typescript": "^5.9.3",
|
|
33
34
|
"vitest": "^4.1.10",
|
|
34
|
-
"
|
|
35
|
+
"zod": "^4.4.3",
|
|
36
|
+
"@skybridge/vite-plugin": "0.0.0-next.0f12579",
|
|
37
|
+
"skybridge": "0.0.0-next.0f12579"
|
|
35
38
|
},
|
|
36
39
|
"engines": {
|
|
37
40
|
"node": ">=22.23.1"
|