@skybridge/test 0.0.0-dev.dde647a → 0.0.0-dev.ed005a8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -45
- package/dist/chat.d.ts +13 -22
- package/dist/chat.d.ts.map +1 -1
- package/dist/chat.js +12 -10
- package/dist/chat.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +0 -1
- package/dist/index.js.map +1 -1
- package/dist/session-registry.d.ts +48 -9
- package/dist/session-registry.d.ts.map +1 -1
- package/dist/session-registry.js +33 -16
- package/dist/session-registry.js.map +1 -1
- package/package.json +9 -34
- package/dist/config.d.ts +0 -9
- package/dist/config.d.ts.map +0 -1
- package/dist/config.js +0 -2
- package/dist/config.js.map +0 -1
- package/dist/global-setup.d.ts +0 -3
- package/dist/global-setup.d.ts.map +0 -1
- package/dist/global-setup.js +0 -24
- package/dist/global-setup.js.map +0 -1
- package/dist/matchers.d.ts +0 -22
- package/dist/matchers.d.ts.map +0 -1
- package/dist/matchers.js +0 -78
- package/dist/matchers.js.map +0 -1
- package/dist/model.d.ts +0 -15
- package/dist/model.d.ts.map +0 -1
- package/dist/model.js +0 -43
- package/dist/model.js.map +0 -1
- package/dist/server-harness.d.ts +0 -18
- package/dist/server-harness.d.ts.map +0 -1
- package/dist/server-harness.js +0 -79
- package/dist/server-harness.js.map +0 -1
- package/dist/vitest-config.d.ts +0 -9
- package/dist/vitest-config.d.ts.map +0 -1
- package/dist/vitest-config.js +0 -18
- package/dist/vitest-config.js.map +0 -1
package/README.md
CHANGED
|
@@ -4,19 +4,20 @@ Evals for [Skybridge](https://skybridge.tech) MCP apps: they answer whether a
|
|
|
4
4
|
prompt reaches the right tool with the right arguments, which no unit test can
|
|
5
5
|
check.
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
7
|
+
Add the `evals` option to the Skybridge Vite plugin so `expect.chat` exists,
|
|
8
|
+
then write scenarios with vitest's own `it` and `expect`. Pass the app itself
|
|
9
|
+
to `start`: it is served in-process, dialing an in-process handler built on `app.createServerInstance()`, and
|
|
10
|
+
the tool names and argument shapes come from the app value, so no type
|
|
11
|
+
parameter is needed.
|
|
12
12
|
|
|
13
13
|
```ts
|
|
14
|
+
import { anthropic } from "@ai-sdk/anthropic";
|
|
14
15
|
import { expect, it } from "vitest";
|
|
15
16
|
import { start } from "@skybridge/test";
|
|
16
|
-
import
|
|
17
|
+
import { app } from "../src/server.js";
|
|
17
18
|
|
|
18
19
|
it("finds products by category", async () => {
|
|
19
|
-
const chat = await start
|
|
20
|
+
const chat = await start({ app, model: anthropic("claude-sonnet-4-5") });
|
|
20
21
|
await chat.send("I am looking for ski goggles");
|
|
21
22
|
|
|
22
23
|
expect.chat(chat).toHaveCalledToolWith("search-products", {
|
|
@@ -25,65 +26,70 @@ it("finds products by category", async () => {
|
|
|
25
26
|
});
|
|
26
27
|
```
|
|
27
28
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
imported, so install just that one as a dev dependency, and the key is read
|
|
32
|
-
from its own environment variable.
|
|
29
|
+
There is no child process and no server to boot, so the evals run anywhere
|
|
30
|
+
vitest does. The caveat is that they bypass the Express layer, so nothing
|
|
31
|
+
verifies a bearer token; pass `authInfo` to claim an identity.
|
|
33
32
|
|
|
34
|
-
|
|
35
|
-
| --- | --- | --- |
|
|
36
|
-
| `anthropic` | `@ai-sdk/anthropic` | `ANTHROPIC_API_KEY` |
|
|
37
|
-
| `openai` | `@ai-sdk/openai` | `OPENAI_API_KEY` |
|
|
38
|
-
| `mistral` | `@ai-sdk/mistral` | `MISTRAL_API_KEY` |
|
|
39
|
-
| `google` | `@ai-sdk/google` | `GOOGLE_GENERATIVE_AI_API_KEY` |
|
|
33
|
+
## Authenticated evals
|
|
40
34
|
|
|
41
35
|
```ts
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
36
|
+
const chat = await start({
|
|
37
|
+
app,
|
|
38
|
+
model: anthropic("claude-sonnet-4-5"),
|
|
39
|
+
authInfo: {
|
|
40
|
+
token: "eval-token",
|
|
41
|
+
clientId: "eval-client",
|
|
42
|
+
scopes: ["checkout"],
|
|
43
|
+
extra: { sub: "user_123", email: "ada@example.com" },
|
|
46
44
|
},
|
|
47
45
|
});
|
|
48
46
|
```
|
|
49
47
|
|
|
50
|
-
`
|
|
51
|
-
|
|
48
|
+
`authInfo` injects a claimed identity: the app's own scheme and scope
|
|
49
|
+
enforcement runs for real, in-band challenges included, and only token
|
|
50
|
+
signature verification is skipped. Omit it to eval the anonymous path, which is
|
|
51
|
+
how you assert that a protected tool challenges instead of running.
|
|
52
52
|
|
|
53
|
-
|
|
54
|
-
`openai/gpt-5` also resolves through the Vercel AI Gateway if you have
|
|
55
|
-
`AI_GATEWAY_API_KEY` set and no provider package installed.
|
|
53
|
+
## Choosing a model
|
|
56
54
|
|
|
57
|
-
|
|
55
|
+
`start` takes any AI SDK model instance, so the provider and its credentials
|
|
56
|
+
are the project's own. Install the provider package you want alongside `ai`.
|
|
58
57
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
58
|
+
```ts
|
|
59
|
+
import { anthropic } from "@ai-sdk/anthropic";
|
|
60
|
+
import { start } from "@skybridge/test";
|
|
61
|
+
|
|
62
|
+
const chat = await start({ app, model: anthropic("claude-sonnet-4-5") });
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
A local runtime or a company gateway works the same way, through the AI SDK's
|
|
66
|
+
own `createOpenAICompatible` or `customProvider`.
|
|
62
67
|
|
|
63
68
|
```ts
|
|
64
|
-
// eval-model.ts
|
|
65
69
|
import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
|
|
66
|
-
import { defineEvalModel } from "@skybridge/test";
|
|
67
70
|
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
+
const ollama = createOpenAICompatible({
|
|
72
|
+
name: "ollama",
|
|
73
|
+
baseURL: "http://localhost:11434/v1",
|
|
74
|
+
});
|
|
75
|
+
const chat = await start({ app, model: ollama("llama3.1") });
|
|
71
76
|
```
|
|
72
77
|
|
|
78
|
+
The plugin's `evals` option also carries the defaults every conversation starts
|
|
79
|
+
from, so a project can set them once instead of per scenario.
|
|
80
|
+
|
|
73
81
|
```ts
|
|
74
|
-
|
|
75
|
-
export default defineConfig({
|
|
76
|
-
plugins: [skybridge({ evals: { project: { ... } } })],
|
|
77
|
-
test: { setupFiles: ["./eval-model.ts"] },
|
|
78
|
-
});
|
|
82
|
+
skybridge({ evals: { temperature: 0, maxSteps: 6 } });
|
|
79
83
|
```
|
|
80
84
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
85
|
+
## Matchers
|
|
86
|
+
|
|
87
|
+
Assertions on what the assistant said work the same way, though model prose
|
|
88
|
+
varies between runs, so keep them loose.
|
|
84
89
|
|
|
85
90
|
```ts
|
|
86
|
-
expect.chat(chat).
|
|
91
|
+
expect.chat(chat).toHaveSaid("in stock");
|
|
92
|
+
expect.chat(chat).not.toHaveSaid("refund");
|
|
87
93
|
```
|
|
88
94
|
|
|
89
95
|
See the [documentation](https://docs.skybridge.tech) for setup and the full
|
package/dist/chat.d.ts
CHANGED
|
@@ -1,34 +1,27 @@
|
|
|
1
|
+
import type { ToolCall } from "@skybridge/vite-plugin/evals";
|
|
1
2
|
import { type LanguageModel } from "ai";
|
|
2
|
-
import type { ToolInput, ToolNames } from "skybridge/server";
|
|
3
|
-
/**
|
|
4
|
-
* One recorded call, typed against the project's own registry: `name` is a
|
|
5
|
-
* union of the project's tool names and `arguments` is discriminated on it.
|
|
6
|
-
*/
|
|
7
|
-
export type ToolCall<App> = {
|
|
8
|
-
[Name in ToolNames<App>]: {
|
|
9
|
-
name: Name;
|
|
10
|
-
arguments: ToolInput<App, Name>;
|
|
11
|
-
failed?: string;
|
|
12
|
-
};
|
|
13
|
-
}[ToolNames<App>];
|
|
14
3
|
interface HostConfig {
|
|
15
4
|
model: LanguageModel;
|
|
16
5
|
temperature?: number;
|
|
17
6
|
systemPrompt?: string;
|
|
18
7
|
maxSteps?: number;
|
|
19
8
|
}
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
9
|
+
/**
|
|
10
|
+
* A `fetch` replacement the transport dials instead of the network. Mirrors the
|
|
11
|
+
* MCP client's own `fetch` option so an in-process app can serve the session.
|
|
12
|
+
*/
|
|
13
|
+
export type TransportFetch = (url: string | URL, init?: RequestInit) => Promise<Response>;
|
|
25
14
|
/**
|
|
26
15
|
* One conversation against one MCP session. A scenario asserts on
|
|
27
16
|
* {@link Chat.toolCalls}, the sequence of calls the model made.
|
|
28
17
|
*/
|
|
29
18
|
export declare class Chat<App = unknown> {
|
|
19
|
+
/** Type-only anchor so `expect.chat` infers `App` from the conversation. */
|
|
20
|
+
readonly $app: App;
|
|
30
21
|
/** Every call the model made, in order, as it crossed the wire. */
|
|
31
22
|
readonly toolCalls: ToolCall<App>[];
|
|
23
|
+
/** What the assistant said, one entry per turn, intermediate ones included. */
|
|
24
|
+
readonly assistantTurns: string[];
|
|
32
25
|
private readonly messages;
|
|
33
26
|
private readonly failures;
|
|
34
27
|
private readonly client;
|
|
@@ -36,11 +29,9 @@ export declare class Chat<App = unknown> {
|
|
|
36
29
|
private readonly host;
|
|
37
30
|
private tools;
|
|
38
31
|
private constructor();
|
|
39
|
-
static open<App>(
|
|
40
|
-
/**
|
|
41
|
-
|
|
42
|
-
/** Takes a turn and returns the model's answer. */
|
|
43
|
-
send(prompt: string): Promise<string>;
|
|
32
|
+
static open<App>(host: HostConfig, fetchImpl: TransportFetch): Promise<Chat<App>>;
|
|
33
|
+
/** Takes a turn. Assert on the result with `expect.chat`. */
|
|
34
|
+
send(prompt: string): Promise<void>;
|
|
44
35
|
close(): Promise<void>;
|
|
45
36
|
private failureFor;
|
|
46
37
|
private toolSet;
|
package/dist/chat.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAIL,KAAK,aAAa,EAGnB,MAAM,IAAI,CAAC;AAEZ,UAAU,UAAU;IAClB,KAAK,EAAE,aAAa,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAMD;;;GAGG;AACH,MAAM,MAAM,cAAc,GAAG,CAC3B,GAAG,EAAE,MAAM,GAAG,GAAG,EACjB,IAAI,CAAC,EAAE,WAAW,KACf,OAAO,CAAC,QAAQ,CAAC,CAAC;AAEvB;;;GAGG;AACH,qBAAa,IAAI,CAAC,GAAG,GAAG,OAAO;IAC7B,4EAA4E;IAC5E,SAAiB,IAAI,EAAE,GAAG,CAAC;IAE3B,mEAAmE;IACnE,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAM;IAEzC,+EAA+E;IAC/E,QAAQ,CAAC,cAAc,EAAE,MAAM,EAAE,CAAM;IAEvC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA6B;IACtD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAa;IAClC,OAAO,CAAC,KAAK,CAAc;IAE3B,OAAO;WAUM,IAAI,CAAC,GAAG,EACnB,IAAI,EAAE,UAAU,EAChB,SAAS,EAAE,cAAc,GACxB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAkBrB,6DAA6D;IACvD,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAiCnC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,OAAO,CAAC,UAAU;IAalB,OAAO,CAAC,OAAO;CAqChB"}
|
package/dist/chat.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { Client } from "@modelcontextprotocol/
|
|
2
|
-
import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
|
|
1
|
+
import { Client, StreamableHTTPClientTransport, } from "@modelcontextprotocol/client";
|
|
3
2
|
import { dynamicTool, generateText, jsonSchema, stepCountIs, } from "ai";
|
|
4
3
|
const DEFAULT_MAX_STEPS = 8;
|
|
4
|
+
const IN_PROCESS_URL = "http://in-process.skybridge.test/mcp";
|
|
5
5
|
/**
|
|
6
6
|
* One conversation against one MCP session. A scenario asserts on
|
|
7
7
|
* {@link Chat.toolCalls}, the sequence of calls the model made.
|
|
@@ -9,6 +9,8 @@ const DEFAULT_MAX_STEPS = 8;
|
|
|
9
9
|
export class Chat {
|
|
10
10
|
/** Every call the model made, in order, as it crossed the wire. */
|
|
11
11
|
toolCalls = [];
|
|
12
|
+
/** What the assistant said, one entry per turn, intermediate ones included. */
|
|
13
|
+
assistantTurns = [];
|
|
12
14
|
messages = [];
|
|
13
15
|
failures = new Map();
|
|
14
16
|
client;
|
|
@@ -20,9 +22,9 @@ export class Chat {
|
|
|
20
22
|
this.transport = transport;
|
|
21
23
|
this.host = host;
|
|
22
24
|
}
|
|
23
|
-
static async open(
|
|
25
|
+
static async open(host, fetchImpl) {
|
|
24
26
|
const client = new Client({ name: "skybridge-eval", version: "0" });
|
|
25
|
-
const transport = new StreamableHTTPClientTransport(new URL(
|
|
27
|
+
const transport = new StreamableHTTPClientTransport(new URL(IN_PROCESS_URL), { fetch: fetchImpl });
|
|
26
28
|
await client.connect(transport);
|
|
27
29
|
const chat = new Chat(client, transport, host);
|
|
28
30
|
try {
|
|
@@ -35,11 +37,7 @@ export class Chat {
|
|
|
35
37
|
}
|
|
36
38
|
return chat;
|
|
37
39
|
}
|
|
38
|
-
/**
|
|
39
|
-
get toolDefinitions() {
|
|
40
|
-
return this.tools;
|
|
41
|
-
}
|
|
42
|
-
/** Takes a turn and returns the model's answer. */
|
|
40
|
+
/** Takes a turn. Assert on the result with `expect.chat`. */
|
|
43
41
|
async send(prompt) {
|
|
44
42
|
this.messages.push({ role: "user", content: prompt });
|
|
45
43
|
const result = await generateText({
|
|
@@ -61,8 +59,12 @@ export class Chat {
|
|
|
61
59
|
});
|
|
62
60
|
}
|
|
63
61
|
}
|
|
62
|
+
for (const step of result.steps) {
|
|
63
|
+
if (step.text !== "") {
|
|
64
|
+
this.assistantTurns.push(step.text);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
64
67
|
this.messages.push(...result.response.messages);
|
|
65
|
-
return result.text;
|
|
66
68
|
}
|
|
67
69
|
async close() {
|
|
68
70
|
await this.transport.close();
|
package/dist/chat.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,MAAM,EACN,6BAA6B,GAE9B,MAAM,8BAA8B,CAAC;AAEtC,OAAO,EACL,WAAW,EACX,YAAY,EACZ,UAAU,EAGV,WAAW,GACZ,MAAM,IAAI,CAAC;AASZ,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B,MAAM,cAAc,GAAG,sCAAsC,CAAC;AAW9D;;;GAGG;AACH,MAAM,OAAO,IAAI;IAIf,mEAAmE;IAC1D,SAAS,GAAoB,EAAE,CAAC;IAEzC,+EAA+E;IACtE,cAAc,GAAa,EAAE,CAAC;IAEtB,QAAQ,GAAmB,EAAE,CAAC;IAC9B,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrC,MAAM,CAAS;IACf,SAAS,CAAgC;IACzC,IAAI,CAAa;IAC1B,KAAK,GAAW,EAAE,CAAC;IAE3B,YACE,MAAc,EACd,SAAwC,EACxC,IAAgB;QAEhB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,IAAI,CACf,IAAgB,EAChB,SAAyB;QAEzB,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,IAAI,6BAA6B,CACjD,IAAI,GAAG,CAAC,cAAc,CAAC,EACvB,EAAE,KAAK,EAAE,SAAS,EAAE,CACrB,CAAC;QACF,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAM,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QACpD,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACrB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YACxB,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,6DAA6D;IAC7D,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAEtD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;YACtB,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC;YACvC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,KAAK,SAAS;gBACtC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE;YACrB,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;SAC/D,CAAC,CAAC;QAEH,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAChC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAClC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;oBAClB,IAAI,EAAE,IAAI,CAAC,QAAQ;oBACnB,SAAS,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAA4B;oBACxD,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;iBACR,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,IAAI,CAAC,IAAI,KAAK,EAAE,EAAE,CAAC;gBACrB,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;IAEO,UAAU,CAAC,IAA+C;QAGhE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QAC9B,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC1B,OAAO,EAAE,MAAM,EAAE,2CAA2C,EAAE,CAAC;QACjE,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAEO,OAAO;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;YAC5C,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,UAAU,EAAE,GAAG,UAAU,CAAC,WAAW,CAAC;YACpE,OAAO;gBACL,UAAU,CAAC,IAAI;gBACf,WAAW,CAAC;oBACV,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,EAAE;oBACzC,WAAW,EAAE,UAAU,CACrB,UAA8C,CAC/C;oBACD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE;wBACvC,IAAI,MAA+C,CAAC;wBACpD,IAAI,CAAC;4BACH,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;gCAClC,IAAI,EAAE,UAAU,CAAC,IAAI;gCACrB,SAAS,EAAE,CAAC,KAAK,IAAI,EAAE,CAA4B;6BACpD,CAAC,CAAC;wBACL,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BACf,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,sCAAsC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC/F,CAAC;4BACF,MAAM,KAAK,CAAC;wBACd,CAAC;wBACD,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;4BAC5B,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,+BAA+B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAChE,CAAC;wBACJ,CAAC;wBACD,OAAO,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC;oBAClC,CAAC;iBACF,CAAC;aACM,CAAC;QACb,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;CACF"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
+
export type { ChatAssertion, ChatMatchers, } from "@skybridge/vite-plugin/evals";
|
|
1
2
|
export type { Chat } from "./chat.js";
|
|
2
|
-
export type {
|
|
3
|
-
export { defineEvalModel } from "./model.js";
|
|
3
|
+
export type { EvalApp, EvalIdentity } from "./session-registry.js";
|
|
4
4
|
export { start } from "./session-registry.js";
|
|
5
5
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,aAAa,EACb,YAAY,GACb,MAAM,8BAA8B,CAAC;AACtC,YAAY,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACtC,YAAY,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACnE,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
|
package/dist/index.js
CHANGED
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
|
|
@@ -1,16 +1,55 @@
|
|
|
1
|
+
import { type Server } from "@modelcontextprotocol/server";
|
|
2
|
+
import type { LanguageModel } from "ai";
|
|
1
3
|
import { Chat } from "./chat.js";
|
|
2
4
|
/**
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
5
|
+
* The minimum a Skybridge app has to expose to be served in-process: the
|
|
6
|
+
* `$types` marker the assertions infer tool names from, and the per-request
|
|
7
|
+
* server builder the session's handler is built on. Structural on purpose, so
|
|
8
|
+
* this package needs no runtime dependency on `skybridge`.
|
|
9
|
+
*/
|
|
10
|
+
export interface EvalApp {
|
|
11
|
+
readonly $types: {
|
|
12
|
+
readonly tools: object;
|
|
13
|
+
};
|
|
14
|
+
createServerInstance(): Promise<Server>;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* The identity an in-process session claims, handed to the app as the
|
|
18
|
+
* request's `authInfo`. Mirrors the SDK's `AuthInfo` structurally; `extra`
|
|
19
|
+
* carries whatever claims the app's verifier would have produced.
|
|
11
20
|
*/
|
|
12
|
-
export
|
|
21
|
+
export interface EvalIdentity {
|
|
22
|
+
token: string;
|
|
23
|
+
clientId: string;
|
|
24
|
+
scopes: string[];
|
|
25
|
+
expiresAt?: number;
|
|
26
|
+
resource?: URL;
|
|
27
|
+
extra?: Record<string, unknown>;
|
|
28
|
+
}
|
|
29
|
+
interface StartOptions {
|
|
30
|
+
/** Any AI SDK model instance, built by the project. */
|
|
31
|
+
model: LanguageModel;
|
|
13
32
|
systemPrompt?: string;
|
|
14
33
|
temperature?: number;
|
|
34
|
+
maxSteps?: number;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Opens a fresh MCP session and conversation against the app under test. The
|
|
38
|
+
* app is served in-process by a handler built on `app.createServerInstance`
|
|
39
|
+
* for this conversation alone, and both are closed when the current test
|
|
40
|
+
* finishes, so scenarios never leak sessions and never write teardown, and
|
|
41
|
+
* tests that run concurrently cannot close each other's sessions.
|
|
42
|
+
*
|
|
43
|
+
* The assertions are inferred from the app value itself, so `expect.chat` gets
|
|
44
|
+
* the project's tool names and argument shapes with no type parameter.
|
|
45
|
+
*
|
|
46
|
+
* `authInfo` claims an identity for the session: the app's per-tool scheme and
|
|
47
|
+
* scope enforcement runs against it for real, only token verification is
|
|
48
|
+
* skipped. Omit it to exercise the anonymous path, challenges included.
|
|
49
|
+
*/
|
|
50
|
+
export declare function start<App extends EvalApp>(options: StartOptions & {
|
|
51
|
+
app: App;
|
|
52
|
+
authInfo?: EvalIdentity;
|
|
15
53
|
}): Promise<Chat<App>>;
|
|
54
|
+
export {};
|
|
16
55
|
//# sourceMappingURL=session-registry.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAoB,KAAK,MAAM,EAAE,MAAM,8BAA8B,CAAC;AAC7E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAExC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC;;;;;GAKG;AACH,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5C,oBAAoB,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;CACzC;AAED;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,GAAG,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACjC;AAED,UAAU,YAAY;IACpB,uDAAuD;IACvD,KAAK,EAAE,aAAa,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAUD;;;;;;;;;;;;;GAaG;AACH,wBAAsB,KAAK,CAAC,GAAG,SAAS,OAAO,EAC7C,OAAO,EAAE,YAAY,GAAG;IAAE,GAAG,EAAE,GAAG,CAAC;IAAC,QAAQ,CAAC,EAAE,YAAY,CAAA;CAAE,GAC5D,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAwBpB"}
|
package/dist/session-registry.js
CHANGED
|
@@ -1,25 +1,42 @@
|
|
|
1
|
+
import { createMcpHandler } from "@modelcontextprotocol/server";
|
|
1
2
|
import { inject, onTestFinished } from "vitest";
|
|
2
3
|
import { Chat } from "./chat.js";
|
|
3
|
-
|
|
4
|
+
function sharedDefaults() {
|
|
5
|
+
try {
|
|
6
|
+
return inject("skybridgeEvals");
|
|
7
|
+
}
|
|
8
|
+
catch {
|
|
9
|
+
return undefined;
|
|
10
|
+
}
|
|
11
|
+
}
|
|
4
12
|
/**
|
|
5
|
-
* Opens a fresh MCP session and conversation against the
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
13
|
+
* Opens a fresh MCP session and conversation against the app under test. The
|
|
14
|
+
* app is served in-process by a handler built on `app.createServerInstance`
|
|
15
|
+
* for this conversation alone, and both are closed when the current test
|
|
16
|
+
* finishes, so scenarios never leak sessions and never write teardown, and
|
|
17
|
+
* tests that run concurrently cannot close each other's sessions.
|
|
18
|
+
*
|
|
19
|
+
* The assertions are inferred from the app value itself, so `expect.chat` gets
|
|
20
|
+
* the project's tool names and argument shapes with no type parameter.
|
|
9
21
|
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
22
|
+
* `authInfo` claims an identity for the session: the app's per-tool scheme and
|
|
23
|
+
* scope enforcement runs against it for real, only token verification is
|
|
24
|
+
* skipped. Omit it to exercise the anonymous path, challenges included.
|
|
13
25
|
*/
|
|
14
|
-
export async function start(
|
|
15
|
-
const config =
|
|
16
|
-
const
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
26
|
+
export async function start(options) {
|
|
27
|
+
const config = sharedDefaults();
|
|
28
|
+
const { app, authInfo } = options;
|
|
29
|
+
const handler = createMcpHandler(() => app.createServerInstance());
|
|
30
|
+
const chat = await Chat.open({
|
|
31
|
+
model: options.model,
|
|
32
|
+
temperature: options.temperature ?? config?.temperature,
|
|
33
|
+
systemPrompt: options.systemPrompt ?? config?.systemPrompt,
|
|
34
|
+
maxSteps: options.maxSteps ?? config?.maxSteps,
|
|
35
|
+
}, (url, init) => handler.fetch(new Request(url, init), authInfo === undefined ? undefined : { authInfo }));
|
|
36
|
+
onTestFinished(async () => {
|
|
37
|
+
await chat.close();
|
|
38
|
+
await handler.close();
|
|
21
39
|
});
|
|
22
|
-
onTestFinished(() => chat.close());
|
|
23
40
|
return chat;
|
|
24
41
|
}
|
|
25
42
|
//# sourceMappingURL=session-registry.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-registry.js","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"session-registry.js","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAe,MAAM,8BAA8B,CAAC;AAE7E,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAmCjC,SAAS,cAAc;IACrB,IAAI,CAAC;QACH,OAAO,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CACzB,OAA6D;IAE7D,MAAM,MAAM,GAAG,cAAc,EAAE,CAAC;IAChC,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAClC,MAAM,OAAO,GAAG,gBAAgB,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,oBAAoB,EAAE,CAAC,CAAC;IAEnE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAC1B;QACE,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,MAAM,EAAE,WAAW;QACvD,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,MAAM,EAAE,YAAY;QAC1D,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,MAAM,EAAE,QAAQ;KAC/C,EACD,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CACZ,OAAO,CAAC,KAAK,CACX,IAAI,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,EACtB,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAClD,CACJ,CAAC;IAEF,cAAc,CAAC,KAAK,IAAI,EAAE;QACxB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC,CAAC,CAAC;IACH,OAAO,IAAI,CAAC;AACd,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@skybridge/test",
|
|
3
|
-
"version": "0.0.0-dev.
|
|
3
|
+
"version": "0.0.0-dev.ed005a8",
|
|
4
4
|
"description": "Eval runner for Skybridge MCP apps: does a prompt reach the right tool with the right arguments",
|
|
5
5
|
"license": "ISC",
|
|
6
6
|
"type": "module",
|
|
@@ -12,58 +12,33 @@
|
|
|
12
12
|
".": {
|
|
13
13
|
"types": "./dist/index.d.ts",
|
|
14
14
|
"default": "./dist/index.js"
|
|
15
|
-
},
|
|
16
|
-
"./vitest-config": {
|
|
17
|
-
"types": "./dist/vitest-config.d.ts",
|
|
18
|
-
"default": "./dist/vitest-config.js"
|
|
19
15
|
}
|
|
20
16
|
},
|
|
21
17
|
"files": [
|
|
22
18
|
"dist"
|
|
23
19
|
],
|
|
24
20
|
"dependencies": {
|
|
25
|
-
"@modelcontextprotocol/
|
|
26
|
-
"
|
|
27
|
-
"@skybridge/vite-plugin": "0.0.0-dev.dde647a"
|
|
21
|
+
"@modelcontextprotocol/client": "^2.0.0",
|
|
22
|
+
"@modelcontextprotocol/server": "^2.0.0"
|
|
28
23
|
},
|
|
29
24
|
"peerDependencies": {
|
|
30
|
-
"@
|
|
31
|
-
"
|
|
32
|
-
"@ai-sdk/mistral": "^2.0.0",
|
|
33
|
-
"@ai-sdk/openai": "^2.0.0",
|
|
25
|
+
"@skybridge/vite-plugin": "^2.0.0",
|
|
26
|
+
"ai": "^5.0.0",
|
|
34
27
|
"skybridge": "^2.0.0",
|
|
35
28
|
"vitest": "^4.1.0"
|
|
36
29
|
},
|
|
37
|
-
"peerDependenciesMeta": {
|
|
38
|
-
"@ai-sdk/anthropic": {
|
|
39
|
-
"optional": true
|
|
40
|
-
},
|
|
41
|
-
"@ai-sdk/google": {
|
|
42
|
-
"optional": true
|
|
43
|
-
},
|
|
44
|
-
"@ai-sdk/mistral": {
|
|
45
|
-
"optional": true
|
|
46
|
-
},
|
|
47
|
-
"@ai-sdk/openai": {
|
|
48
|
-
"optional": true
|
|
49
|
-
}
|
|
50
|
-
},
|
|
51
30
|
"devDependencies": {
|
|
52
|
-
"@ai-sdk/anthropic": "^2.0.0",
|
|
53
|
-
"@ai-sdk/google": "^2.0.0",
|
|
54
|
-
"@ai-sdk/mistral": "^2.0.0",
|
|
55
|
-
"@ai-sdk/openai": "^2.0.0",
|
|
56
31
|
"@types/node": "^24.13.3",
|
|
32
|
+
"ai": "^5.0.0",
|
|
57
33
|
"typescript": "^5.9.3",
|
|
58
34
|
"vitest": "^4.1.10",
|
|
59
|
-
"
|
|
35
|
+
"zod": "^4.4.3",
|
|
36
|
+
"@skybridge/vite-plugin": "0.0.0-dev.ed005a8",
|
|
37
|
+
"skybridge": "0.0.0-dev.ed005a8"
|
|
60
38
|
},
|
|
61
39
|
"engines": {
|
|
62
40
|
"node": ">=22.23.1"
|
|
63
41
|
},
|
|
64
|
-
"publishConfig": {
|
|
65
|
-
"tag": "beta"
|
|
66
|
-
},
|
|
67
42
|
"scripts": {
|
|
68
43
|
"build": "tsc -p tsconfig.json",
|
|
69
44
|
"format": "biome check --write --error-on-warnings",
|
package/dist/config.d.ts
DELETED
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
import type { EvalsOptions } from "@skybridge/vite-plugin";
|
|
2
|
-
export type { EvalsOptions } from "@skybridge/vite-plugin";
|
|
3
|
-
declare module "vitest" {
|
|
4
|
-
interface ProvidedContext {
|
|
5
|
-
skybridgeEvals: EvalsOptions;
|
|
6
|
-
skybridgeEvalsUrl: string;
|
|
7
|
-
}
|
|
8
|
-
}
|
|
9
|
-
//# sourceMappingURL=config.d.ts.map
|
package/dist/config.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAE3D,YAAY,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAE3D,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,eAAe;QACvB,cAAc,EAAE,YAAY,CAAC;QAC7B,iBAAiB,EAAE,MAAM,CAAC;KAC3B;CACF"}
|
package/dist/config.js
DELETED
package/dist/config.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":""}
|
package/dist/global-setup.d.ts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"global-setup.d.ts","sourceRoot":"","sources":["../src/global-setup.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAG/C,wBAA8B,KAAK,CAAC,OAAO,EAAE,WAAW,8CA0BvD"}
|
package/dist/global-setup.js
DELETED
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
import { startServer } from "./server-harness.js";
|
|
2
|
-
export default async function setup(project) {
|
|
3
|
-
const options = project.config.provide?.skybridgeEvals;
|
|
4
|
-
if (options === undefined) {
|
|
5
|
-
throw new Error("The evals plugin is not registered. Add `skybridge({ evals: {...} })` to the vitest config.");
|
|
6
|
-
}
|
|
7
|
-
if (options.server !== undefined) {
|
|
8
|
-
project.provide("skybridgeEvalsUrl", options.server);
|
|
9
|
-
return;
|
|
10
|
-
}
|
|
11
|
-
if (options.project === undefined) {
|
|
12
|
-
throw new Error("The evals plugin needs either `server` or `project`");
|
|
13
|
-
}
|
|
14
|
-
const server = await startServer(options.project);
|
|
15
|
-
project.provide("skybridgeEvalsUrl", server.url);
|
|
16
|
-
return async () => {
|
|
17
|
-
const output = server.output().trim();
|
|
18
|
-
await server.stop();
|
|
19
|
-
if (output !== "") {
|
|
20
|
-
console.log(`\n--- server output ---\n${output}\n---`);
|
|
21
|
-
}
|
|
22
|
-
};
|
|
23
|
-
}
|
|
24
|
-
//# sourceMappingURL=global-setup.js.map
|
package/dist/global-setup.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"global-setup.js","sourceRoot":"","sources":["../src/global-setup.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAElD,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,KAAK,CAAC,OAAoB;IACtD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,cAAc,CAAC;IACvD,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,6FAA6F,CAC9F,CAAC;IACJ,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QACjC,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACrD,OAAO;IACT,CAAC;IACD,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAClD,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;IAEjD,OAAO,KAAK,IAAI,EAAE;QAChB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;YAClB,OAAO,CAAC,GAAG,CAAC,4BAA4B,MAAM,OAAO,CAAC,CAAC;QACzD,CAAC;IACH,CAAC,CAAC;AACJ,CAAC"}
|
package/dist/matchers.d.ts
DELETED
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
import type { ToolInput, ToolNames } from "skybridge/server";
|
|
2
|
-
import type { Chat } from "./chat.js";
|
|
3
|
-
/**
|
|
4
|
-
* Assertions about the tool calls a conversation produced. Kept off vitest's
|
|
5
|
-
* global `Matchers` so they never appear on unrelated `expect()` calls, and so
|
|
6
|
-
* this stays the one place later tool-call helpers are added.
|
|
7
|
-
*/
|
|
8
|
-
export interface ChatMatchers<App> {
|
|
9
|
-
toHaveCalledToolOnce<Name extends ToolNames<App>>(name: Name, args?: Partial<ToolInput<App, Name>>): void;
|
|
10
|
-
toHaveCalledToolWith<Name extends ToolNames<App>>(name: Name, args: Partial<ToolInput<App, Name>>): void;
|
|
11
|
-
toNeverHaveCalledTool<Name extends ToolNames<App>>(name: Name): void;
|
|
12
|
-
toHaveFailedToolCall<Name extends ToolNames<App>>(name: Name): void;
|
|
13
|
-
}
|
|
14
|
-
export interface ChatAssertion<App> extends ChatMatchers<App> {
|
|
15
|
-
not: ChatMatchers<App>;
|
|
16
|
-
}
|
|
17
|
-
declare module "vitest" {
|
|
18
|
-
interface ExpectStatic {
|
|
19
|
-
chat<App>(chat: Chat<App>): ChatAssertion<App>;
|
|
20
|
-
}
|
|
21
|
-
}
|
|
22
|
-
//# sourceMappingURL=matchers.d.ts.map
|
package/dist/matchers.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"matchers.d.ts","sourceRoot":"","sources":["../src/matchers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7D,OAAO,KAAK,EAAE,IAAI,EAAY,MAAM,WAAW,CAAC;AAEhD;;;;GAIG;AACH,MAAM,WAAW,YAAY,CAAC,GAAG;IAC/B,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAC9C,IAAI,EAAE,IAAI,EACV,IAAI,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GACnC,IAAI,CAAC;IACR,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAC9C,IAAI,EAAE,IAAI,EACV,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GAClC,IAAI,CAAC;IACR,qBAAqB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,CAAC;IACrE,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,CAAC;CACrE;AAED,MAAM,WAAW,aAAa,CAAC,GAAG,CAAE,SAAQ,YAAY,CAAC,GAAG,CAAC;IAC3D,GAAG,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC;CACxB;AAED,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,YAAY;QACpB,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;KAChD;CACF"}
|
package/dist/matchers.js
DELETED
|
@@ -1,78 +0,0 @@
|
|
|
1
|
-
import { expect } from "vitest";
|
|
2
|
-
function attemptedCalls(chat, name) {
|
|
3
|
-
return chat.toolCalls.filter((call) => String(call.name) === name);
|
|
4
|
-
}
|
|
5
|
-
function acceptedCalls(chat, name) {
|
|
6
|
-
return attemptedCalls(chat, name).filter((call) => call.failed === undefined);
|
|
7
|
-
}
|
|
8
|
-
function observed(chat) {
|
|
9
|
-
if (chat.toolCalls.length === 0) {
|
|
10
|
-
return " (no tool was called)";
|
|
11
|
-
}
|
|
12
|
-
return chat.toolCalls
|
|
13
|
-
.map((call, index) => ` ${index + 1}. ${call.name} ${JSON.stringify(call.arguments)}${call.failed === undefined ? "" : ` (failed: ${call.failed})`}`)
|
|
14
|
-
.join("\n");
|
|
15
|
-
}
|
|
16
|
-
function definitions(chat) {
|
|
17
|
-
return chat.toolDefinitions
|
|
18
|
-
.map((tool) => ` - ${tool.name}: ${tool.description ?? ""}\n arguments: ${JSON.stringify(tool.inputSchema)}`)
|
|
19
|
-
.join("\n");
|
|
20
|
-
}
|
|
21
|
-
function report(chat, summary) {
|
|
22
|
-
return `${summary}
|
|
23
|
-
|
|
24
|
-
Calls the model actually made:
|
|
25
|
-
${observed(chat)}
|
|
26
|
-
|
|
27
|
-
Tool definitions the model was looking at:
|
|
28
|
-
${definitions(chat)}`;
|
|
29
|
-
}
|
|
30
|
-
expect.extend({
|
|
31
|
-
toHaveCalledToolOnce(received, name, args) {
|
|
32
|
-
const matching = acceptedCalls(received, name);
|
|
33
|
-
const argsMatch = args === undefined ||
|
|
34
|
-
(matching.length === 1 &&
|
|
35
|
-
this.equals(matching[0]?.arguments, expect.objectContaining(args)));
|
|
36
|
-
const pass = matching.length === 1 && argsMatch;
|
|
37
|
-
return {
|
|
38
|
-
pass,
|
|
39
|
-
message: () => report(received, pass
|
|
40
|
-
? `Expected "${name}" not to be called exactly once.`
|
|
41
|
-
: matching.length === 1
|
|
42
|
-
? `Expected "${name}" to be called once with ${JSON.stringify(args)}, but it was called with ${JSON.stringify(matching[0]?.arguments)}.`
|
|
43
|
-
: `Expected "${name}" to be called exactly once by a call that succeeded, but ${matching.length} such calls were made.`),
|
|
44
|
-
};
|
|
45
|
-
},
|
|
46
|
-
toHaveCalledToolWith(received, name, args) {
|
|
47
|
-
const pass = acceptedCalls(received, name).some((call) => this.equals(call.arguments, expect.objectContaining(args)));
|
|
48
|
-
return {
|
|
49
|
-
pass,
|
|
50
|
-
message: () => report(received, pass
|
|
51
|
-
? `Expected no call to "${name}" with ${JSON.stringify(args)}.`
|
|
52
|
-
: `Expected "${name}" to be called with ${JSON.stringify(args)}.`),
|
|
53
|
-
};
|
|
54
|
-
},
|
|
55
|
-
toHaveFailedToolCall(received, name) {
|
|
56
|
-
const attempted = attemptedCalls(received, name);
|
|
57
|
-
const failed = attempted.filter((call) => call.failed !== undefined);
|
|
58
|
-
return {
|
|
59
|
-
pass: failed.length > 0,
|
|
60
|
-
message: () => report(received, failed.length > 0
|
|
61
|
-
? `Expected every call to "${name}" to succeed.`
|
|
62
|
-
: attempted.length === 0
|
|
63
|
-
? `Expected "${name}" to be called and fail, but it was never called.`
|
|
64
|
-
: `Expected "${name}" to fail, but all ${attempted.length} calls succeeded.`),
|
|
65
|
-
};
|
|
66
|
-
},
|
|
67
|
-
toNeverHaveCalledTool(received, name) {
|
|
68
|
-
const matching = attemptedCalls(received, name);
|
|
69
|
-
return {
|
|
70
|
-
pass: matching.length === 0,
|
|
71
|
-
message: () => report(received, matching.length === 0
|
|
72
|
-
? `Expected "${name}" to be called at least once.`
|
|
73
|
-
: `Expected "${name}" never to be called, but it was called ${matching.length} times.`),
|
|
74
|
-
};
|
|
75
|
-
},
|
|
76
|
-
});
|
|
77
|
-
expect.chat = (chat) => expect(chat);
|
|
78
|
-
//# sourceMappingURL=matchers.js.map
|
package/dist/matchers.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"matchers.js","sourceRoot":"","sources":["../src/matchers.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AA+BhC,SAAS,cAAc,CACrB,IAAmB,EACnB,IAAY;IAEZ,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;AACrE,CAAC;AAED,SAAS,aAAa,CAAC,IAAmB,EAAE,IAAY;IACtD,OAAO,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,QAAQ,CAAC,IAAmB;IACnC,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,wBAAwB,CAAC;IAClC,CAAC;IACD,OAAO,IAAI,CAAC,SAAS;SAClB,GAAG,CACF,CAAC,IAAuB,EAAE,KAAa,EAAE,EAAE,CACzC,KAAK,KAAK,GAAG,CAAC,KAAK,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,cAAc,IAAI,CAAC,MAAM,GAAG,EAAE,CACnI;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,WAAW,CAAC,IAAmB;IACtC,OAAO,IAAI,CAAC,eAAe;SACxB,GAAG,CACF,CAAC,IAAI,EAAE,EAAE,CACP,OAAO,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,WAAW,IAAI,EAAE,oBAAoB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CACpG;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,MAAM,CAAC,IAAmB,EAAE,OAAe;IAClD,OAAO,GAAG,OAAO;;;EAGjB,QAAQ,CAAC,IAAI,CAAC;;;EAGd,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;AACtB,CAAC;AAED,MAAM,CAAC,MAAM,CAAC;IACZ,oBAAoB,CAClB,QAAuB,EACvB,IAAY,EACZ,IAA8B;QAE9B,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC/C,MAAM,SAAS,GACb,IAAI,KAAK,SAAS;YAClB,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACpB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxE,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC;QAEhD,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,aAAa,IAAI,kCAAkC;gBACrD,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;oBACrB,CAAC,CAAC,aAAa,IAAI,4BAA4B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,4BAA4B,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG;oBACxI,CAAC,CAAC,aAAa,IAAI,6DAA6D,QAAQ,CAAC,MAAM,wBAAwB,CAC5H;SACJ,CAAC;IACJ,CAAC;IAED,oBAAoB,CAClB,QAAuB,EACvB,IAAY,EACZ,IAA6B;QAE7B,MAAM,IAAI,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CACvD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAC3D,CAAC;QAEF,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,wBAAwB,IAAI,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG;gBAC/D,CAAC,CAAC,aAAa,IAAI,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CACpE;SACJ,CAAC;IACJ,CAAC;IAED,oBAAoB,CAAC,QAAuB,EAAE,IAAY;QACxD,MAAM,SAAS,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;QAErE,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC;YACvB,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,MAAM,CAAC,MAAM,GAAG,CAAC;gBACf,CAAC,CAAC,2BAA2B,IAAI,eAAe;gBAChD,CAAC,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;oBACtB,CAAC,CAAC,aAAa,IAAI,mDAAmD;oBACtE,CAAC,CAAC,aAAa,IAAI,sBAAsB,SAAS,CAAC,MAAM,mBAAmB,CACjF;SACJ,CAAC;IACJ,CAAC;IAED,qBAAqB,CAAC,QAAuB,EAAE,IAAY;QACzD,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAEhD,OAAO;YACL,IAAI,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC;YAC3B,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACnB,CAAC,CAAC,aAAa,IAAI,+BAA+B;gBAClD,CAAC,CAAC,aAAa,IAAI,2CAA2C,QAAQ,CAAC,MAAM,SAAS,CACzF;SACJ,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,IAAI,GAAG,CAAM,IAAe,EAAsB,EAAE,CACzD,MAAM,CAAC,IAAI,CAAkC,CAAC"}
|
package/dist/model.d.ts
DELETED
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
import type { LanguageModel } from "ai";
|
|
2
|
-
/**
|
|
3
|
-
* Registers a model instance from a setup file, taking precedence over the
|
|
4
|
-
* configured `model` string. The escape hatch for anything the provider
|
|
5
|
-
* prefixes cannot express, such as a local runtime or a company LLM gateway.
|
|
6
|
-
*/
|
|
7
|
-
export declare function defineEvalModel(model: LanguageModel): void;
|
|
8
|
-
/**
|
|
9
|
-
* Turns a `provider/model-id` string into a model instance, importing only the
|
|
10
|
-
* provider named. Each provider package reads its own conventional API key
|
|
11
|
-
* variable, so no key configuration is needed. An unrecognised prefix is
|
|
12
|
-
* returned untouched for the AI SDK's default provider to resolve.
|
|
13
|
-
*/
|
|
14
|
-
export declare function resolveModel(spec: string | undefined): Promise<LanguageModel>;
|
|
15
|
-
//# sourceMappingURL=model.d.ts.map
|
package/dist/model.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"model.d.ts","sourceRoot":"","sources":["../src/model.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAaxC;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAE1D;AAED;;;;;GAKG;AACH,wBAAsB,YAAY,CAChC,IAAI,EAAE,MAAM,GAAG,SAAS,GACvB,OAAO,CAAC,aAAa,CAAC,CAwBxB"}
|
package/dist/model.js
DELETED
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
const PROVIDERS = {
|
|
2
|
-
anthropic: () => import("@ai-sdk/anthropic").then((m) => m.anthropic),
|
|
3
|
-
google: () => import("@ai-sdk/google").then((m) => m.google),
|
|
4
|
-
mistral: () => import("@ai-sdk/mistral").then((m) => m.mistral),
|
|
5
|
-
openai: () => import("@ai-sdk/openai").then((m) => m.openai),
|
|
6
|
-
};
|
|
7
|
-
let registered;
|
|
8
|
-
/**
|
|
9
|
-
* Registers a model instance from a setup file, taking precedence over the
|
|
10
|
-
* configured `model` string. The escape hatch for anything the provider
|
|
11
|
-
* prefixes cannot express, such as a local runtime or a company LLM gateway.
|
|
12
|
-
*/
|
|
13
|
-
export function defineEvalModel(model) {
|
|
14
|
-
registered = model;
|
|
15
|
-
}
|
|
16
|
-
/**
|
|
17
|
-
* Turns a `provider/model-id` string into a model instance, importing only the
|
|
18
|
-
* provider named. Each provider package reads its own conventional API key
|
|
19
|
-
* variable, so no key configuration is needed. An unrecognised prefix is
|
|
20
|
-
* returned untouched for the AI SDK's default provider to resolve.
|
|
21
|
-
*/
|
|
22
|
-
export async function resolveModel(spec) {
|
|
23
|
-
if (registered !== undefined) {
|
|
24
|
-
return registered;
|
|
25
|
-
}
|
|
26
|
-
if (spec === undefined) {
|
|
27
|
-
throw new Error("No model configured. Set `model` on the evals plugin, or call defineEvalModel() from a setup file.");
|
|
28
|
-
}
|
|
29
|
-
const separator = spec.indexOf("/");
|
|
30
|
-
if (separator === -1) {
|
|
31
|
-
return spec;
|
|
32
|
-
}
|
|
33
|
-
const name = spec.slice(0, separator);
|
|
34
|
-
const load = PROVIDERS[name];
|
|
35
|
-
if (load === undefined) {
|
|
36
|
-
return spec;
|
|
37
|
-
}
|
|
38
|
-
const provider = await load().catch(() => {
|
|
39
|
-
throw new Error(`The model "${spec}" needs @ai-sdk/${name}@^2. Install it as a dev dependency.`);
|
|
40
|
-
});
|
|
41
|
-
return provider(spec.slice(separator + 1));
|
|
42
|
-
}
|
|
43
|
-
//# sourceMappingURL=model.js.map
|
package/dist/model.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"model.js","sourceRoot":"","sources":["../src/model.ts"],"names":[],"mappings":"AAIA,MAAM,SAAS,GAAmD;IAChE,SAAS,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IACrE,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IAC5D,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;IAC/D,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;CAC7D,CAAC;AAEF,IAAI,UAAqC,CAAC;AAE1C;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,KAAoB;IAClD,UAAU,GAAG,KAAK,CAAC;AACrB,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAAwB;IAExB,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,oGAAoG,CACrG,CAAC;IACJ,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACpC,IAAI,SAAS,KAAK,CAAC,CAAC,EAAE,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC7B,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE;QACvC,MAAM,IAAI,KAAK,CACb,cAAc,IAAI,mBAAmB,IAAI,sCAAsC,CAChF,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC;AAC7C,CAAC"}
|
package/dist/server-harness.d.ts
DELETED
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
export interface RunningServer {
|
|
2
|
-
url: string;
|
|
3
|
-
/** Everything the server wrote to stdout/stderr, for failure reports. */
|
|
4
|
-
output: () => string;
|
|
5
|
-
stop: () => Promise<void>;
|
|
6
|
-
}
|
|
7
|
-
/**
|
|
8
|
-
* Spawns the project's server with `__PORT=0` and an IPC channel, and waits
|
|
9
|
-
* for the `skybridge:listening` message carrying the port the OS picked.
|
|
10
|
-
* Binding and reporting both happen in the child, so there is no window in
|
|
11
|
-
* which another process can steal the port.
|
|
12
|
-
*/
|
|
13
|
-
export declare function startServer({ cwd, command, env, }: {
|
|
14
|
-
cwd: string;
|
|
15
|
-
command: string[];
|
|
16
|
-
env?: Record<string, string>;
|
|
17
|
-
}): Promise<RunningServer>;
|
|
18
|
-
//# sourceMappingURL=server-harness.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"server-harness.d.ts","sourceRoot":"","sources":["../src/server-harness.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,aAAa;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,yEAAyE;IACzE,MAAM,EAAE,MAAM,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAiBD;;;;;GAKG;AACH,wBAAsB,WAAW,CAAC,EAChC,GAAG,EACH,OAAO,EACP,GAAQ,GACT,EAAE;IACD,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC9B,GAAG,OAAO,CAAC,aAAa,CAAC,CA4EzB"}
|
package/dist/server-harness.js
DELETED
|
@@ -1,79 +0,0 @@
|
|
|
1
|
-
import { spawn } from "node:child_process";
|
|
2
|
-
import { resolve } from "node:path";
|
|
3
|
-
function hasExited(child) {
|
|
4
|
-
return child.exitCode !== null || child.signalCode !== null;
|
|
5
|
-
}
|
|
6
|
-
function isListeningMessage(message) {
|
|
7
|
-
return (typeof message === "object" &&
|
|
8
|
-
message !== null &&
|
|
9
|
-
message.type === "skybridge:listening" &&
|
|
10
|
-
typeof message.port === "number");
|
|
11
|
-
}
|
|
12
|
-
/**
|
|
13
|
-
* Spawns the project's server with `__PORT=0` and an IPC channel, and waits
|
|
14
|
-
* for the `skybridge:listening` message carrying the port the OS picked.
|
|
15
|
-
* Binding and reporting both happen in the child, so there is no window in
|
|
16
|
-
* which another process can steal the port.
|
|
17
|
-
*/
|
|
18
|
-
export async function startServer({ cwd, command, env = {}, }) {
|
|
19
|
-
const timeoutMs = 60_000;
|
|
20
|
-
const [bin, ...args] = command;
|
|
21
|
-
if (bin === undefined) {
|
|
22
|
-
throw new Error("Empty server command");
|
|
23
|
-
}
|
|
24
|
-
const executable = bin.startsWith(".") ? resolve(cwd, bin) : bin;
|
|
25
|
-
const child = spawn(executable, args, {
|
|
26
|
-
cwd,
|
|
27
|
-
env: {
|
|
28
|
-
...process.env,
|
|
29
|
-
NODE_ENV: "production",
|
|
30
|
-
...env,
|
|
31
|
-
__PORT: "0",
|
|
32
|
-
},
|
|
33
|
-
stdio: ["ignore", "pipe", "pipe", "ipc"],
|
|
34
|
-
});
|
|
35
|
-
let captured = "";
|
|
36
|
-
const output = () => captured;
|
|
37
|
-
child.stdout?.on("data", (chunk) => {
|
|
38
|
-
captured += chunk;
|
|
39
|
-
});
|
|
40
|
-
child.stderr?.on("data", (chunk) => {
|
|
41
|
-
captured += chunk;
|
|
42
|
-
});
|
|
43
|
-
const stop = async () => {
|
|
44
|
-
if (hasExited(child)) {
|
|
45
|
-
return;
|
|
46
|
-
}
|
|
47
|
-
const exited = new Promise((resolveExit) => {
|
|
48
|
-
child.once("exit", () => resolveExit());
|
|
49
|
-
});
|
|
50
|
-
child.kill("SIGTERM");
|
|
51
|
-
const forced = setTimeout(() => child.kill("SIGKILL"), 3_000);
|
|
52
|
-
await exited;
|
|
53
|
-
clearTimeout(forced);
|
|
54
|
-
};
|
|
55
|
-
const port = await new Promise((resolvePort, reject) => {
|
|
56
|
-
const timer = setTimeout(() => {
|
|
57
|
-
reject(new Error(`Server did not report a port in ${timeoutMs}ms. \`command\` must start the server process directly (node, tsx): wrappers such as \`pnpm run\` do not forward the IPC channel the readiness signal uses. It must also run a skybridge version that sends skybridge:listening.\n${output()}`));
|
|
58
|
-
}, timeoutMs);
|
|
59
|
-
child.on("message", (message) => {
|
|
60
|
-
if (isListeningMessage(message)) {
|
|
61
|
-
clearTimeout(timer);
|
|
62
|
-
resolvePort(message.port);
|
|
63
|
-
}
|
|
64
|
-
});
|
|
65
|
-
child.on("error", (error) => {
|
|
66
|
-
clearTimeout(timer);
|
|
67
|
-
reject(new Error(`Failed to spawn ${command.join(" ")}: ${error.message}`));
|
|
68
|
-
});
|
|
69
|
-
child.on("exit", () => {
|
|
70
|
-
clearTimeout(timer);
|
|
71
|
-
reject(new Error(`Server exited with code ${child.exitCode} before reporting its port:\n${output()}`));
|
|
72
|
-
});
|
|
73
|
-
}).catch(async (error) => {
|
|
74
|
-
await stop();
|
|
75
|
-
throw error;
|
|
76
|
-
});
|
|
77
|
-
return { url: `http://127.0.0.1:${port}/mcp`, output, stop };
|
|
78
|
-
}
|
|
79
|
-
//# sourceMappingURL=server-harness.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"server-harness.js","sourceRoot":"","sources":["../src/server-harness.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqB,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AASpC,SAAS,SAAS,CAAC,KAAmB;IACpC,OAAO,KAAK,CAAC,QAAQ,KAAK,IAAI,IAAI,KAAK,CAAC,UAAU,KAAK,IAAI,CAAC;AAC9D,CAAC;AAED,SAAS,kBAAkB,CACzB,OAAgB;IAEhB,OAAO,CACL,OAAO,OAAO,KAAK,QAAQ;QAC3B,OAAO,KAAK,IAAI;QACf,OAA8B,CAAC,IAAI,KAAK,qBAAqB;QAC9D,OAAQ,OAA8B,CAAC,IAAI,KAAK,QAAQ,CACzD,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,EAChC,GAAG,EACH,OAAO,EACP,GAAG,GAAG,EAAE,GAKT;IACC,MAAM,SAAS,GAAG,MAAM,CAAC;IACzB,MAAM,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC;IAC/B,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAC1C,CAAC;IAED,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAEjE,MAAM,KAAK,GAAG,KAAK,CAAC,UAAU,EAAE,IAAI,EAAE;QACpC,GAAG;QACH,GAAG,EAAE;YACH,GAAG,OAAO,CAAC,GAAG;YACd,QAAQ,EAAE,YAAY;YACtB,GAAG,GAAG;YACN,MAAM,EAAE,GAAG;SACZ;QACD,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC;KACzC,CAAC,CAAC;IAEH,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,MAAM,MAAM,GAAG,GAAG,EAAE,CAAC,QAAQ,CAAC;IAC9B,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;QACjC,QAAQ,IAAI,KAAK,CAAC;IACpB,CAAC,CAAC,CAAC;IACH,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;QACjC,QAAQ,IAAI,KAAK,CAAC;IACpB,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,KAAK,IAAI,EAAE;QACtB,IAAI,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;YACrB,OAAO;QACT,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,OAAO,CAAO,CAAC,WAAW,EAAE,EAAE;YAC/C,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtB,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,CAAC,CAAC;QAC9D,MAAM,MAAM,CAAC;QACb,YAAY,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,MAAM,IAAI,OAAO,CAAS,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE;QAC7D,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,MAAM,CACJ,IAAI,KAAK,CACP,mCAAmC,SAAS,qOAAqO,MAAM,EAAE,EAAE,CAC5R,CACF,CAAC;QACJ,CAAC,EAAE,SAAS,CAAC,CAAC;QACd,KAAK,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE;YAC9B,IAAI,kBAAkB,CAAC,OAAO,CAAC,EAAE,CAAC;gBAChC,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YAC1B,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CACJ,IAAI,KAAK,CAAC,mBAAmB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CACpE,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;YACpB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CACJ,IAAI,KAAK,CACP,2BAA2B,KAAK,CAAC,QAAQ,gCAAgC,MAAM,EAAE,EAAE,CACpF,CACF,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,KAAc,EAAE,EAAE;QAChC,MAAM,IAAI,EAAE,CAAC;QACb,MAAM,KAAK,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,GAAG,EAAE,oBAAoB,IAAI,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;AAC/D,CAAC"}
|
package/dist/vitest-config.d.ts
DELETED
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
import type { ViteUserConfig } from "vitest/config";
|
|
2
|
-
import type { EvalsOptions } from "./config.js";
|
|
3
|
-
/**
|
|
4
|
-
* The vitest settings the eval runner needs, merged in by the `skybridge()`
|
|
5
|
-
* Vite plugin. Returned as plain config rather than a plugin so the paths stay
|
|
6
|
-
* resolved against this package, which owns the files they point at.
|
|
7
|
-
*/
|
|
8
|
-
export declare function evalsConfig(options: EvalsOptions): ViteUserConfig["test"];
|
|
9
|
-
//# sourceMappingURL=vitest-config.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"vitest-config.d.ts","sourceRoot":"","sources":["../src/vitest-config.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAMhD;;;;GAIG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,YAAY,GAAG,cAAc,CAAC,MAAM,CAAC,CAOzE"}
|
package/dist/vitest-config.js
DELETED
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
import { fileURLToPath } from "node:url";
|
|
2
|
-
function here(file) {
|
|
3
|
-
return fileURLToPath(new URL(file, import.meta.url));
|
|
4
|
-
}
|
|
5
|
-
/**
|
|
6
|
-
* The vitest settings the eval runner needs, merged in by the `skybridge()`
|
|
7
|
-
* Vite plugin. Returned as plain config rather than a plugin so the paths stay
|
|
8
|
-
* resolved against this package, which owns the files they point at.
|
|
9
|
-
*/
|
|
10
|
-
export function evalsConfig(options) {
|
|
11
|
-
return {
|
|
12
|
-
testTimeout: 120_000,
|
|
13
|
-
setupFiles: [here("./matchers.js")],
|
|
14
|
-
globalSetup: [here("./global-setup.js")],
|
|
15
|
-
provide: { skybridgeEvals: options },
|
|
16
|
-
};
|
|
17
|
-
}
|
|
18
|
-
//# sourceMappingURL=vitest-config.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"vitest-config.js","sourceRoot":"","sources":["../src/vitest-config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAIzC,SAAS,IAAI,CAAC,IAAY;IACxB,OAAO,aAAa,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AACvD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAAC,OAAqB;IAC/C,OAAO;QACL,WAAW,EAAE,OAAO;QACpB,UAAU,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACnC,WAAW,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QACxC,OAAO,EAAE,EAAE,cAAc,EAAE,OAAO,EAAE;KACrC,CAAC;AACJ,CAAC"}
|