@skybridge/test 0.0.0-dev.b9466d5 → 0.0.0-dev.dbab7df
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -3
- package/dist/chat.d.ts +13 -6
- package/dist/chat.d.ts.map +1 -1
- package/dist/chat.js +11 -10
- package/dist/chat.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/session-registry.d.ts +59 -9
- package/dist/session-registry.d.ts.map +1 -1
- package/dist/session-registry.js +25 -17
- package/dist/session-registry.js.map +1 -1
- package/package.json +5 -4
package/README.md
CHANGED
|
@@ -4,8 +4,6 @@ Evals for [Skybridge](https://skybridge.tech) MCP apps: they answer whether a
|
|
|
4
4
|
prompt reaches the right tool with the right arguments, which no unit test can
|
|
5
5
|
check.
|
|
6
6
|
|
|
7
|
-
**Beta.** The API is still settling and may change between releases.
|
|
8
|
-
|
|
9
7
|
Add the `evals` option to the Skybridge Vite plugin, then write scenarios with
|
|
10
8
|
vitest's own `it` and `expect`.
|
|
11
9
|
|
|
@@ -25,6 +23,53 @@ it("finds products by category", async () => {
|
|
|
25
23
|
});
|
|
26
24
|
```
|
|
27
25
|
|
|
26
|
+
## In-process mode
|
|
27
|
+
|
|
28
|
+
Pass the app itself instead of booting a server. `start` then dials
|
|
29
|
+
`app.fetchHandler` directly, and the tool names and argument shapes come from
|
|
30
|
+
the app value, so no type parameter is needed.
|
|
31
|
+
|
|
32
|
+
```ts
|
|
33
|
+
import { anthropic } from "@ai-sdk/anthropic";
|
|
34
|
+
import { expect, it } from "vitest";
|
|
35
|
+
import { start } from "@skybridge/test";
|
|
36
|
+
import { app } from "../src/server.js";
|
|
37
|
+
|
|
38
|
+
it("finds products by category", async () => {
|
|
39
|
+
const chat = await start({ app, model: anthropic("claude-sonnet-4-5") });
|
|
40
|
+
await chat.send("I am looking for ski goggles");
|
|
41
|
+
|
|
42
|
+
expect.chat(chat).toHaveCalledToolWith("search-products", {
|
|
43
|
+
category: "goggles",
|
|
44
|
+
});
|
|
45
|
+
});
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
Prefer it whenever you can: there is no child process and no plugin config, so
|
|
49
|
+
it runs anywhere vitest does. The caveat is that it bypasses the Express layer,
|
|
50
|
+
so nothing verifies a bearer token; pass `authInfo` to claim an identity, or use
|
|
51
|
+
the process mode above when the evals must go through real tokens.
|
|
52
|
+
|
|
53
|
+
### Authenticated evals
|
|
54
|
+
|
|
55
|
+
```ts
|
|
56
|
+
const chat = await start({
|
|
57
|
+
app,
|
|
58
|
+
model: anthropic("claude-sonnet-4-5"),
|
|
59
|
+
authInfo: {
|
|
60
|
+
token: "eval-token",
|
|
61
|
+
clientId: "eval-client",
|
|
62
|
+
scopes: ["checkout"],
|
|
63
|
+
extra: { sub: "user_123", email: "ada@example.com" },
|
|
64
|
+
},
|
|
65
|
+
});
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
`authInfo` injects a claimed identity: the app's own scheme and scope
|
|
69
|
+
enforcement runs for real, in-band challenges included, and only token
|
|
70
|
+
signature verification is skipped. Omit it to eval the anonymous path, which is
|
|
71
|
+
how you assert that a protected tool challenges instead of running.
|
|
72
|
+
|
|
28
73
|
## Choosing a model
|
|
29
74
|
|
|
30
75
|
`start` takes any AI SDK model instance, so the provider and its credentials
|
|
@@ -54,12 +99,20 @@ const chat = await start<AppType>({ model: ollama("llama3.1") });
|
|
|
54
99
|
|
|
55
100
|
```ts
|
|
56
101
|
skybridge({
|
|
57
|
-
evals: { project: { cwd: ".", command: ["node", "dist/
|
|
102
|
+
evals: { project: { cwd: ".", command: ["node", "dist/index.js"] } },
|
|
58
103
|
});
|
|
59
104
|
```
|
|
60
105
|
|
|
61
106
|
`command` has to start the server process directly, since wrappers such as
|
|
62
107
|
`pnpm run` do not forward the channel the runner uses to learn the port.
|
|
63
108
|
|
|
109
|
+
Assertions on what the assistant said work the same way, though model prose
|
|
110
|
+
varies between runs, so keep them loose.
|
|
111
|
+
|
|
112
|
+
```ts
|
|
113
|
+
expect.chat(chat).toHaveSaid("in stock");
|
|
114
|
+
expect.chat(chat).not.toHaveSaid("refund");
|
|
115
|
+
```
|
|
116
|
+
|
|
64
117
|
See the [documentation](https://docs.skybridge.tech) for setup and the full
|
|
65
118
|
matcher list.
|
package/dist/chat.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ToolCall
|
|
1
|
+
import type { ToolCall } from "@skybridge/vite-plugin/evals";
|
|
2
2
|
import { type LanguageModel } from "ai";
|
|
3
3
|
interface HostConfig {
|
|
4
4
|
model: LanguageModel;
|
|
@@ -6,13 +6,22 @@ interface HostConfig {
|
|
|
6
6
|
systemPrompt?: string;
|
|
7
7
|
maxSteps?: number;
|
|
8
8
|
}
|
|
9
|
+
/**
|
|
10
|
+
* A `fetch` replacement the transport dials instead of the network. Mirrors the
|
|
11
|
+
* MCP client's own `fetch` option so an in-process app can serve the session.
|
|
12
|
+
*/
|
|
13
|
+
export type TransportFetch = (url: string | URL, init?: RequestInit) => Promise<Response>;
|
|
9
14
|
/**
|
|
10
15
|
* One conversation against one MCP session. A scenario asserts on
|
|
11
16
|
* {@link Chat.toolCalls}, the sequence of calls the model made.
|
|
12
17
|
*/
|
|
13
18
|
export declare class Chat<App = unknown> {
|
|
19
|
+
/** Type-only anchor so `expect.chat` infers `App` from the conversation. */
|
|
20
|
+
readonly $app: App;
|
|
14
21
|
/** Every call the model made, in order, as it crossed the wire. */
|
|
15
22
|
readonly toolCalls: ToolCall<App>[];
|
|
23
|
+
/** What the assistant said, one entry per turn, intermediate ones included. */
|
|
24
|
+
readonly assistantTurns: string[];
|
|
16
25
|
private readonly messages;
|
|
17
26
|
private readonly failures;
|
|
18
27
|
private readonly client;
|
|
@@ -20,11 +29,9 @@ export declare class Chat<App = unknown> {
|
|
|
20
29
|
private readonly host;
|
|
21
30
|
private tools;
|
|
22
31
|
private constructor();
|
|
23
|
-
static open<App>(serverUrl: string, host: HostConfig): Promise<Chat<App>>;
|
|
24
|
-
/**
|
|
25
|
-
|
|
26
|
-
/** Takes a turn and returns the model's answer. */
|
|
27
|
-
send(prompt: string): Promise<string>;
|
|
32
|
+
static open<App>(serverUrl: string, host: HostConfig, fetchImpl?: TransportFetch): Promise<Chat<App>>;
|
|
33
|
+
/** Takes a turn. Assert on the result with `expect.chat`. */
|
|
34
|
+
send(prompt: string): Promise<void>;
|
|
28
35
|
close(): Promise<void>;
|
|
29
36
|
private failureFor;
|
|
30
37
|
private toolSet;
|
package/dist/chat.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAIL,KAAK,aAAa,EAGnB,MAAM,IAAI,CAAC;AAEZ,UAAU,UAAU;IAClB,KAAK,EAAE,aAAa,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAID;;;GAGG;AACH,MAAM,MAAM,cAAc,GAAG,CAC3B,GAAG,EAAE,MAAM,GAAG,GAAG,EACjB,IAAI,CAAC,EAAE,WAAW,KACf,OAAO,CAAC,QAAQ,CAAC,CAAC;AAEvB;;;GAGG;AACH,qBAAa,IAAI,CAAC,GAAG,GAAG,OAAO;IAC7B,4EAA4E;IAC5E,SAAiB,IAAI,EAAE,GAAG,CAAC;IAE3B,mEAAmE;IACnE,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAM;IAEzC,+EAA+E;IAC/E,QAAQ,CAAC,cAAc,EAAE,MAAM,EAAE,CAAM;IAEvC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA6B;IACtD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAa;IAClC,OAAO,CAAC,KAAK,CAAc;IAE3B,OAAO;WAUM,IAAI,CAAC,GAAG,EACnB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,UAAU,EAChB,SAAS,CAAC,EAAE,cAAc,GACzB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAkBrB,6DAA6D;IACvD,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAiCnC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,OAAO,CAAC,UAAU;IAalB,OAAO,CAAC,OAAO;CAqChB"}
|
package/dist/chat.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import { Client } from "@modelcontextprotocol/
|
|
2
|
-
import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
|
|
1
|
+
import { Client, StreamableHTTPClientTransport, } from "@modelcontextprotocol/client";
|
|
3
2
|
import { dynamicTool, generateText, jsonSchema, stepCountIs, } from "ai";
|
|
4
3
|
const DEFAULT_MAX_STEPS = 8;
|
|
5
4
|
/**
|
|
@@ -9,6 +8,8 @@ const DEFAULT_MAX_STEPS = 8;
|
|
|
9
8
|
export class Chat {
|
|
10
9
|
/** Every call the model made, in order, as it crossed the wire. */
|
|
11
10
|
toolCalls = [];
|
|
11
|
+
/** What the assistant said, one entry per turn, intermediate ones included. */
|
|
12
|
+
assistantTurns = [];
|
|
12
13
|
messages = [];
|
|
13
14
|
failures = new Map();
|
|
14
15
|
client;
|
|
@@ -20,9 +21,9 @@ export class Chat {
|
|
|
20
21
|
this.transport = transport;
|
|
21
22
|
this.host = host;
|
|
22
23
|
}
|
|
23
|
-
static async open(serverUrl, host) {
|
|
24
|
+
static async open(serverUrl, host, fetchImpl) {
|
|
24
25
|
const client = new Client({ name: "skybridge-eval", version: "0" });
|
|
25
|
-
const transport = new StreamableHTTPClientTransport(new URL(serverUrl));
|
|
26
|
+
const transport = new StreamableHTTPClientTransport(new URL(serverUrl), fetchImpl === undefined ? undefined : { fetch: fetchImpl });
|
|
26
27
|
await client.connect(transport);
|
|
27
28
|
const chat = new Chat(client, transport, host);
|
|
28
29
|
try {
|
|
@@ -35,11 +36,7 @@ export class Chat {
|
|
|
35
36
|
}
|
|
36
37
|
return chat;
|
|
37
38
|
}
|
|
38
|
-
/**
|
|
39
|
-
get toolDefinitions() {
|
|
40
|
-
return this.tools;
|
|
41
|
-
}
|
|
42
|
-
/** Takes a turn and returns the model's answer. */
|
|
39
|
+
/** Takes a turn. Assert on the result with `expect.chat`. */
|
|
43
40
|
async send(prompt) {
|
|
44
41
|
this.messages.push({ role: "user", content: prompt });
|
|
45
42
|
const result = await generateText({
|
|
@@ -61,8 +58,12 @@ export class Chat {
|
|
|
61
58
|
});
|
|
62
59
|
}
|
|
63
60
|
}
|
|
61
|
+
for (const step of result.steps) {
|
|
62
|
+
if (step.text !== "") {
|
|
63
|
+
this.assistantTurns.push(step.text);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
64
66
|
this.messages.push(...result.response.messages);
|
|
65
|
-
return result.text;
|
|
66
67
|
}
|
|
67
68
|
async close() {
|
|
68
69
|
await this.transport.close();
|
package/dist/chat.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,MAAM,EACN,6BAA6B,GAE9B,MAAM,8BAA8B,CAAC;AAEtC,OAAO,EACL,WAAW,EACX,YAAY,EACZ,UAAU,EAGV,WAAW,GACZ,MAAM,IAAI,CAAC;AASZ,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAW5B;;;GAGG;AACH,MAAM,OAAO,IAAI;IAIf,mEAAmE;IAC1D,SAAS,GAAoB,EAAE,CAAC;IAEzC,+EAA+E;IACtE,cAAc,GAAa,EAAE,CAAC;IAEtB,QAAQ,GAAmB,EAAE,CAAC;IAC9B,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrC,MAAM,CAAS;IACf,SAAS,CAAgC;IACzC,IAAI,CAAa;IAC1B,KAAK,GAAW,EAAE,CAAC;IAE3B,YACE,MAAc,EACd,SAAwC,EACxC,IAAgB;QAEhB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,IAAI,CACf,SAAiB,EACjB,IAAgB,EAChB,SAA0B;QAE1B,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,IAAI,6BAA6B,CACjD,IAAI,GAAG,CAAC,SAAS,CAAC,EAClB,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,CAC3D,CAAC;QACF,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAM,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QACpD,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACrB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YACxB,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,6DAA6D;IAC7D,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAEtD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;YACtB,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC;YACvC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,KAAK,SAAS;gBACtC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE;YACrB,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;SAC/D,CAAC,CAAC;QAEH,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAChC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAClC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;oBAClB,IAAI,EAAE,IAAI,CAAC,QAAQ;oBACnB,SAAS,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAA4B;oBACxD,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;iBACR,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,IAAI,CAAC,IAAI,KAAK,EAAE,EAAE,CAAC;gBACrB,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;IAEO,UAAU,CAAC,IAA+C;QAGhE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QAC9B,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC1B,OAAO,EAAE,MAAM,EAAE,2CAA2C,EAAE,CAAC;QACjE,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAEO,OAAO;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;YAC5C,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,UAAU,EAAE,GAAG,UAAU,CAAC,WAAW,CAAC;YACpE,OAAO;gBACL,UAAU,CAAC,IAAI;gBACf,WAAW,CAAC;oBACV,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,EAAE;oBACzC,WAAW,EAAE,UAAU,CACrB,UAA8C,CAC/C;oBACD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE;wBACvC,IAAI,MAA+C,CAAC;wBACpD,IAAI,CAAC;4BACH,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;gCAClC,IAAI,EAAE,UAAU,CAAC,IAAI;gCACrB,SAAS,EAAE,CAAC,KAAK,IAAI,EAAE,CAA4B;6BACpD,CAAC,CAAC;wBACL,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BACf,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,sCAAsC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC/F,CAAC;4BACF,MAAM,KAAK,CAAC;wBACd,CAAC;wBACD,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;4BAC5B,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,+BAA+B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAChE,CAAC;wBACJ,CAAC;wBACD,OAAO,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC;oBAClC,CAAC;iBACF,CAAC;aACM,CAAC;QACb,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;CACF"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
export type { ChatAssertion, ChatMatchers, } from "@skybridge/vite-plugin/evals";
|
|
2
2
|
export type { Chat } from "./chat.js";
|
|
3
|
+
export type { EvalApp, EvalIdentity } from "./session-registry.js";
|
|
3
4
|
export { start } from "./session-registry.js";
|
|
4
5
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,aAAa,EACb,YAAY,GACb,MAAM,8BAA8B,CAAC;AACtC,YAAY,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,aAAa,EACb,YAAY,GACb,MAAM,8BAA8B,CAAC;AACtC,YAAY,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACtC,YAAY,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACnE,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
|
|
@@ -1,20 +1,70 @@
|
|
|
1
1
|
import type { LanguageModel } from "ai";
|
|
2
2
|
import { Chat } from "./chat.js";
|
|
3
3
|
/**
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
* The type parameter pins the assertions to the project's registry:
|
|
10
|
-
* `start<AppType>()` returns a `Chat<AppType>`, and `expect.chat` infers the
|
|
11
|
-
* tool names and argument shapes from it.
|
|
4
|
+
* The minimum a Skybridge app has to expose to be served in-process: the
|
|
5
|
+
* `$types` marker the assertions infer tool names from, and the fetch handler
|
|
6
|
+
* the session is dialed through. Structural on purpose, so this package needs
|
|
7
|
+
* no runtime dependency on `skybridge`.
|
|
12
8
|
*/
|
|
13
|
-
export
|
|
9
|
+
export interface EvalApp {
|
|
10
|
+
readonly $types: {
|
|
11
|
+
readonly tools: object;
|
|
12
|
+
};
|
|
13
|
+
readonly fetchHandler: {
|
|
14
|
+
fetch: (request: Request, options?: {
|
|
15
|
+
authInfo?: EvalIdentity;
|
|
16
|
+
}) => Promise<Response>;
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* The identity an in-process session claims, handed to the app as the
|
|
21
|
+
* request's `authInfo`. Mirrors the SDK's `AuthInfo` structurally; `extra`
|
|
22
|
+
* carries whatever claims the app's verifier would have produced.
|
|
23
|
+
*/
|
|
24
|
+
export interface EvalIdentity {
|
|
25
|
+
token: string;
|
|
26
|
+
clientId: string;
|
|
27
|
+
scopes: string[];
|
|
28
|
+
expiresAt?: number;
|
|
29
|
+
resource?: URL;
|
|
30
|
+
extra?: Record<string, unknown>;
|
|
31
|
+
}
|
|
32
|
+
interface StartOptions {
|
|
14
33
|
/** Any AI SDK model instance, built by the project. */
|
|
15
34
|
model: LanguageModel;
|
|
16
35
|
systemPrompt?: string;
|
|
17
36
|
temperature?: number;
|
|
18
37
|
maxSteps?: number;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Opens a fresh MCP session and conversation against the app under test. Pass
|
|
41
|
+
* `app` to serve it in-process, or omit it to reach the server the Vite plugin
|
|
42
|
+
* started. Either way the session is closed when the current test finishes, so
|
|
43
|
+
* scenarios never leak sessions and never write teardown, and tests that run
|
|
44
|
+
* concurrently cannot close each other's sessions.
|
|
45
|
+
*
|
|
46
|
+
* In-process mode infers the assertions from the app value itself, so
|
|
47
|
+
* `expect.chat` gets the project's tool names and argument shapes with no type
|
|
48
|
+
* parameter. It talks straight to `app.fetchHandler`, which never closes: the
|
|
49
|
+
* handler is memoized on the app and shared by every test.
|
|
50
|
+
*
|
|
51
|
+
* `authInfo` claims an identity for the session: the app's per-tool scheme and
|
|
52
|
+
* scope enforcement runs against it for real, only token verification is
|
|
53
|
+
* skipped. Omit it to exercise the anonymous path, challenges included. It is
|
|
54
|
+
* in-process only, since a server reached over HTTP takes real tokens instead.
|
|
55
|
+
*/
|
|
56
|
+
export declare function start<App extends EvalApp>(options: StartOptions & {
|
|
57
|
+
app: App;
|
|
58
|
+
authInfo?: EvalIdentity;
|
|
59
|
+
}): Promise<Chat<App>>;
|
|
60
|
+
/**
|
|
61
|
+
* The type parameter pins the assertions to the project's registry:
|
|
62
|
+
* `start<AppType>()` returns a `Chat<AppType>`, and `expect.chat` infers the
|
|
63
|
+
* tool names and argument shapes from it.
|
|
64
|
+
*/
|
|
65
|
+
export declare function start<App>(options: StartOptions & {
|
|
66
|
+
app?: never;
|
|
67
|
+
authInfo?: never;
|
|
19
68
|
}): Promise<Chat<App>>;
|
|
69
|
+
export {};
|
|
20
70
|
//# sourceMappingURL=session-registry.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAExC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC
|
|
1
|
+
{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAExC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC;;;;;GAKG;AACH,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5C,QAAQ,CAAC,YAAY,EAAE;QACrB,KAAK,EAAE,CACL,OAAO,EAAE,OAAO,EAChB,OAAO,CAAC,EAAE;YAAE,QAAQ,CAAC,EAAE,YAAY,CAAA;SAAE,KAClC,OAAO,CAAC,QAAQ,CAAC,CAAC;KACxB,CAAC;CACH;AAED;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,GAAG,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACjC;AAED,UAAU,YAAY;IACpB,uDAAuD;IACvD,KAAK,EAAE,aAAa,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAYD;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,KAAK,CAAC,GAAG,SAAS,OAAO,EAC7C,OAAO,EAAE,YAAY,GAAG;IAAE,GAAG,EAAE,GAAG,CAAC;IAAC,QAAQ,CAAC,EAAE,YAAY,CAAA;CAAE,GAC5D,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AACtB;;;;GAIG;AACH,wBAAsB,KAAK,CAAC,GAAG,EAC7B,OAAO,EAAE,YAAY,GAAG;IAAE,GAAG,CAAC,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,EAAE,KAAK,CAAA;CAAE,GACxD,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC"}
|
package/dist/session-registry.js
CHANGED
|
@@ -1,28 +1,36 @@
|
|
|
1
1
|
import { inject, onTestFinished } from "vitest";
|
|
2
2
|
import { Chat } from "./chat.js";
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
* concurrently cannot close each other's sessions.
|
|
8
|
-
*
|
|
9
|
-
* The type parameter pins the assertions to the project's registry:
|
|
10
|
-
* `start<AppType>()` returns a `Chat<AppType>`, and `expect.chat` infers the
|
|
11
|
-
* tool names and argument shapes from it.
|
|
12
|
-
*/
|
|
13
|
-
export async function start(options) {
|
|
14
|
-
const url = inject("skybridgeEvalsUrl");
|
|
15
|
-
if (url === undefined) {
|
|
16
|
-
throw new Error("No eval server is running. Add `skybridge({ evals: {...} })` to the vitest config.");
|
|
3
|
+
const IN_PROCESS_URL = "http://in-process.skybridge.test/mcp";
|
|
4
|
+
function sharedDefaults() {
|
|
5
|
+
try {
|
|
6
|
+
return inject("skybridgeEvals");
|
|
17
7
|
}
|
|
18
|
-
|
|
19
|
-
|
|
8
|
+
catch {
|
|
9
|
+
return undefined;
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
export async function start(options) {
|
|
13
|
+
const config = sharedDefaults();
|
|
14
|
+
const host = {
|
|
20
15
|
model: options.model,
|
|
21
16
|
temperature: options.temperature ?? config?.temperature,
|
|
22
17
|
systemPrompt: options.systemPrompt ?? config?.systemPrompt,
|
|
23
18
|
maxSteps: options.maxSteps ?? config?.maxSteps,
|
|
24
|
-
}
|
|
19
|
+
};
|
|
20
|
+
const chat = await (options.app === undefined
|
|
21
|
+
? Chat.open(urlFromPlugin(), host)
|
|
22
|
+
: inProcessChat(options.app, host, options.authInfo));
|
|
25
23
|
onTestFinished(() => chat.close());
|
|
26
24
|
return chat;
|
|
27
25
|
}
|
|
26
|
+
function urlFromPlugin() {
|
|
27
|
+
const url = inject("skybridgeEvalsUrl");
|
|
28
|
+
if (url === undefined) {
|
|
29
|
+
throw new Error("No eval server is running. Pass `app` to `start` to serve the app in-process, or add `skybridge({ evals: {...} })` to the vitest config.");
|
|
30
|
+
}
|
|
31
|
+
return url;
|
|
32
|
+
}
|
|
33
|
+
function inProcessChat(app, host, authInfo) {
|
|
34
|
+
return Chat.open(IN_PROCESS_URL, host, (url, init) => app.fetchHandler.fetch(new Request(url, init), authInfo === undefined ? undefined : { authInfo }));
|
|
35
|
+
}
|
|
28
36
|
//# sourceMappingURL=session-registry.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-registry.js","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"session-registry.js","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAwCjC,MAAM,cAAc,GAAG,sCAAsC,CAAC;AAE9D,SAAS,cAAc;IACrB,IAAI,CAAC;QACH,OAAO,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AA8BD,MAAM,CAAC,KAAK,UAAU,KAAK,CACzB,OAAkE;IAElE,MAAM,MAAM,GAAG,cAAc,EAAE,CAAC;IAChC,MAAM,IAAI,GAAG;QACX,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,MAAM,EAAE,WAAW;QACvD,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,MAAM,EAAE,YAAY;QAC1D,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,MAAM,EAAE,QAAQ;KAC/C,CAAC;IAEF,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,KAAK,SAAS;QAC3C,CAAC,CAAC,IAAI,CAAC,IAAI,CAAM,aAAa,EAAE,EAAE,IAAI,CAAC;QACvC,CAAC,CAAC,aAAa,CAAM,OAAO,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IAE7D,cAAc,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;IACnC,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,aAAa;IACpB,MAAM,GAAG,GAAG,MAAM,CAAC,mBAAmB,CAAC,CAAC;IACxC,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CACb,0IAA0I,CAC3I,CAAC;IACJ,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,aAAa,CACpB,GAAY,EACZ,IAAqC,EACrC,QAAkC;IAElC,OAAO,IAAI,CAAC,IAAI,CAAM,cAAc,EAAE,IAAI,EAAE,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CACxD,GAAG,CAAC,YAAY,CAAC,KAAK,CACpB,IAAI,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,EACtB,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAClD,CACF,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@skybridge/test",
|
|
3
|
-
"version": "0.0.0-dev.
|
|
3
|
+
"version": "0.0.0-dev.dbab7df",
|
|
4
4
|
"description": "Eval runner for Skybridge MCP apps: does a prompt reach the right tool with the right arguments",
|
|
5
5
|
"license": "ISC",
|
|
6
6
|
"type": "module",
|
|
@@ -18,8 +18,8 @@
|
|
|
18
18
|
"dist"
|
|
19
19
|
],
|
|
20
20
|
"dependencies": {
|
|
21
|
-
"@modelcontextprotocol/
|
|
22
|
-
"@skybridge/vite-plugin": "0.0.0-dev.
|
|
21
|
+
"@modelcontextprotocol/client": "^2.0.0",
|
|
22
|
+
"@skybridge/vite-plugin": "0.0.0-dev.dbab7df"
|
|
23
23
|
},
|
|
24
24
|
"peerDependencies": {
|
|
25
25
|
"ai": "^5.0.0",
|
|
@@ -31,7 +31,8 @@
|
|
|
31
31
|
"ai": "^5.0.0",
|
|
32
32
|
"typescript": "^5.9.3",
|
|
33
33
|
"vitest": "^4.1.10",
|
|
34
|
-
"
|
|
34
|
+
"zod": "^4.4.3",
|
|
35
|
+
"skybridge": "0.0.0-dev.dbab7df"
|
|
35
36
|
},
|
|
36
37
|
"engines": {
|
|
37
38
|
"node": ">=22.23.1"
|