@skybridge/test 0.0.0-dev.09043e8 → 0.0.0-dev.0ad9d4b
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -47
- package/dist/chat.d.ts +2 -0
- package/dist/chat.d.ts.map +1 -1
- package/dist/chat.js +7 -0
- package/dist/chat.js.map +1 -1
- package/dist/index.d.ts +0 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +0 -1
- package/dist/index.js.map +1 -1
- package/dist/session-registry.d.ts +5 -1
- package/dist/session-registry.d.ts.map +1 -1
- package/dist/session-registry.js +7 -8
- package/dist/session-registry.js.map +1 -1
- package/package.json +5 -33
- package/dist/model.d.ts +0 -15
- package/dist/model.d.ts.map +0 -1
- package/dist/model.js +0 -43
- package/dist/model.js.map +0 -1
package/README.md
CHANGED
|
@@ -4,19 +4,19 @@ Evals for [Skybridge](https://skybridge.tech) MCP apps: they answer whether a
|
|
|
4
4
|
prompt reaches the right tool with the right arguments, which no unit test can
|
|
5
5
|
check.
|
|
6
6
|
|
|
7
|
-
**Beta.** The API is still settling
|
|
8
|
-
dist-tag and may change between releases.
|
|
7
|
+
**Beta.** The API is still settling and may change between releases.
|
|
9
8
|
|
|
10
9
|
Add the `evals` option to the Skybridge Vite plugin, then write scenarios with
|
|
11
10
|
vitest's own `it` and `expect`.
|
|
12
11
|
|
|
13
12
|
```ts
|
|
13
|
+
import { anthropic } from "@ai-sdk/anthropic";
|
|
14
14
|
import { expect, it } from "vitest";
|
|
15
15
|
import { start } from "@skybridge/test";
|
|
16
16
|
import type { AppType } from "../src/server.js";
|
|
17
17
|
|
|
18
18
|
it("finds products by category", async () => {
|
|
19
|
-
const chat = await start<AppType>();
|
|
19
|
+
const chat = await start<AppType>({ model: anthropic("claude-sonnet-4-5") });
|
|
20
20
|
await chat.send("I am looking for ski goggles");
|
|
21
21
|
|
|
22
22
|
expect.chat(chat).toHaveCalledToolWith("search-products", {
|
|
@@ -27,66 +27,46 @@ it("finds products by category", async () => {
|
|
|
27
27
|
|
|
28
28
|
## Choosing a model
|
|
29
29
|
|
|
30
|
-
`
|
|
31
|
-
|
|
32
|
-
from its own environment variable.
|
|
33
|
-
|
|
34
|
-
| prefix | package | key |
|
|
35
|
-
| --- | --- | --- |
|
|
36
|
-
| `anthropic` | `@ai-sdk/anthropic` | `ANTHROPIC_API_KEY` |
|
|
37
|
-
| `openai` | `@ai-sdk/openai` | `OPENAI_API_KEY` |
|
|
38
|
-
| `mistral` | `@ai-sdk/mistral` | `MISTRAL_API_KEY` |
|
|
39
|
-
| `google` | `@ai-sdk/google` | `GOOGLE_GENERATIVE_AI_API_KEY` |
|
|
30
|
+
`start` takes any AI SDK model instance, so the provider and its credentials
|
|
31
|
+
are the project's own. Install the provider package you want alongside `ai`.
|
|
40
32
|
|
|
41
33
|
```ts
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
model: "anthropic/claude-sonnet-4-5",
|
|
45
|
-
project: { cwd: ".", command: ["node", "dist/server.js"] },
|
|
46
|
-
},
|
|
47
|
-
});
|
|
48
|
-
```
|
|
49
|
-
|
|
50
|
-
`command` has to start the server process directly, since wrappers such as
|
|
51
|
-
`pnpm run` do not forward the channel the runner uses to learn the port.
|
|
52
|
-
|
|
53
|
-
Scenarios wait on a real model, so raise vitest's `testTimeout` well above its
|
|
54
|
-
default: `test: { testTimeout: 120_000 }`.
|
|
55
|
-
|
|
56
|
-
Any other prefix is passed to the AI SDK untouched, so a string such as
|
|
57
|
-
`openai/gpt-5` also resolves through the Vercel AI Gateway if you have
|
|
58
|
-
`AI_GATEWAY_API_KEY` set and no provider package installed.
|
|
34
|
+
import { anthropic } from "@ai-sdk/anthropic";
|
|
35
|
+
import { start } from "@skybridge/test";
|
|
59
36
|
|
|
60
|
-
|
|
37
|
+
const chat = await start<AppType>({ model: anthropic("claude-sonnet-4-5") });
|
|
38
|
+
```
|
|
61
39
|
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
precedence over `model`, which can then be left out.
|
|
40
|
+
A local runtime or a company gateway works the same way, through the AI SDK's
|
|
41
|
+
own `createOpenAICompatible` or `customProvider`.
|
|
65
42
|
|
|
66
43
|
```ts
|
|
67
|
-
// eval-model.ts
|
|
68
44
|
import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
|
|
69
|
-
import { defineEvalModel } from "@skybridge/test";
|
|
70
45
|
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
46
|
+
const ollama = createOpenAICompatible({
|
|
47
|
+
name: "ollama",
|
|
48
|
+
baseURL: "http://localhost:11434/v1",
|
|
49
|
+
});
|
|
50
|
+
const chat = await start<AppType>({ model: ollama("llama3.1") });
|
|
74
51
|
```
|
|
75
52
|
|
|
53
|
+
## Booting the server
|
|
54
|
+
|
|
76
55
|
```ts
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
plugins: [skybridge({ evals: { project: { ... } } })],
|
|
80
|
-
test: { setupFiles: ["./eval-model.ts"] },
|
|
56
|
+
skybridge({
|
|
57
|
+
evals: { project: { cwd: ".", command: ["node", "dist/server.js"] } },
|
|
81
58
|
});
|
|
82
59
|
```
|
|
83
60
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
61
|
+
`command` has to start the server process directly, since wrappers such as
|
|
62
|
+
`pnpm run` do not forward the channel the runner uses to learn the port.
|
|
63
|
+
|
|
64
|
+
Assertions on what the assistant said work the same way, though model prose
|
|
65
|
+
varies between runs, so keep them loose.
|
|
87
66
|
|
|
88
67
|
```ts
|
|
89
|
-
expect.chat(chat).
|
|
68
|
+
expect.chat(chat).toHaveSaid("in stock");
|
|
69
|
+
expect.chat(chat).not.toHaveSaid("refund");
|
|
90
70
|
```
|
|
91
71
|
|
|
92
72
|
See the [documentation](https://docs.skybridge.tech) for setup and the full
|
package/dist/chat.d.ts
CHANGED
|
@@ -13,6 +13,8 @@ interface HostConfig {
|
|
|
13
13
|
export declare class Chat<App = unknown> {
|
|
14
14
|
/** Every call the model made, in order, as it crossed the wire. */
|
|
15
15
|
readonly toolCalls: ToolCall<App>[];
|
|
16
|
+
/** What the assistant said, one entry per turn, intermediate ones included. */
|
|
17
|
+
readonly assistantTurns: string[];
|
|
16
18
|
private readonly messages;
|
|
17
19
|
private readonly failures;
|
|
18
20
|
private readonly client;
|
package/dist/chat.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC7E,OAAO,EAIL,KAAK,aAAa,EAGnB,MAAM,IAAI,CAAC;AAEZ,UAAU,UAAU;IAClB,KAAK,EAAE,aAAa,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAID;;;GAGG;AACH,qBAAa,IAAI,CAAC,GAAG,GAAG,OAAO;IAC7B,mEAAmE;IACnE,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAM;IAEzC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA6B;IACtD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAa;IAClC,OAAO,CAAC,KAAK,CAAwB;IAErC,OAAO;WAUM,IAAI,CAAC,GAAG,EACnB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,UAAU,GACf,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAerB,0EAA0E;IAC1E,IAAI,eAAe,IAAI,cAAc,EAAE,CAEtC;IAED,mDAAmD;IAC7C,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC7E,OAAO,EAIL,KAAK,aAAa,EAGnB,MAAM,IAAI,CAAC;AAEZ,UAAU,UAAU;IAClB,KAAK,EAAE,aAAa,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAID;;;GAGG;AACH,qBAAa,IAAI,CAAC,GAAG,GAAG,OAAO;IAC7B,mEAAmE;IACnE,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAM;IAEzC,+EAA+E;IAC/E,QAAQ,CAAC,cAAc,EAAE,MAAM,EAAE,CAAM;IAEvC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA6B;IACtD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAa;IAClC,OAAO,CAAC,KAAK,CAAwB;IAErC,OAAO;WAUM,IAAI,CAAC,GAAG,EACnB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,UAAU,GACf,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAerB,0EAA0E;IAC1E,IAAI,eAAe,IAAI,cAAc,EAAE,CAEtC;IAED,mDAAmD;IAC7C,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAkCrC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,OAAO,CAAC,UAAU;IAalB,OAAO,CAAC,OAAO;CAmChB"}
|
package/dist/chat.js
CHANGED
|
@@ -9,6 +9,8 @@ const DEFAULT_MAX_STEPS = 8;
|
|
|
9
9
|
export class Chat {
|
|
10
10
|
/** Every call the model made, in order, as it crossed the wire. */
|
|
11
11
|
toolCalls = [];
|
|
12
|
+
/** What the assistant said, one entry per turn, intermediate ones included. */
|
|
13
|
+
assistantTurns = [];
|
|
12
14
|
messages = [];
|
|
13
15
|
failures = new Map();
|
|
14
16
|
client;
|
|
@@ -61,6 +63,11 @@ export class Chat {
|
|
|
61
63
|
});
|
|
62
64
|
}
|
|
63
65
|
}
|
|
66
|
+
for (const step of result.steps) {
|
|
67
|
+
if (step.text !== "") {
|
|
68
|
+
this.assistantTurns.push(step.text);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
64
71
|
this.messages.push(...result.response.messages);
|
|
65
72
|
return result.text;
|
|
66
73
|
}
|
package/dist/chat.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,6BAA6B,EAAE,MAAM,oDAAoD,CAAC;AAEnG,OAAO,EACL,WAAW,EACX,YAAY,EACZ,UAAU,EAGV,WAAW,GACZ,MAAM,IAAI,CAAC;AASZ,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B;;;GAGG;AACH,MAAM,OAAO,IAAI;IACf,mEAAmE;IAC1D,SAAS,GAAoB,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,6BAA6B,EAAE,MAAM,oDAAoD,CAAC;AAEnG,OAAO,EACL,WAAW,EACX,YAAY,EACZ,UAAU,EAGV,WAAW,GACZ,MAAM,IAAI,CAAC;AASZ,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B;;;GAGG;AACH,MAAM,OAAO,IAAI;IACf,mEAAmE;IAC1D,SAAS,GAAoB,EAAE,CAAC;IAEzC,+EAA+E;IACtE,cAAc,GAAa,EAAE,CAAC;IAEtB,QAAQ,GAAmB,EAAE,CAAC;IAC9B,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrC,MAAM,CAAS;IACf,SAAS,CAAgC;IACzC,IAAI,CAAa;IAC1B,KAAK,GAAqB,EAAE,CAAC;IAErC,YACE,MAAc,EACd,SAAwC,EACxC,IAAgB;QAEhB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,IAAI,CACf,SAAiB,EACjB,IAAgB;QAEhB,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,IAAI,6BAA6B,CAAC,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;QACxE,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAM,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QACpD,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACrB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YACxB,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,0EAA0E;IAC1E,IAAI,eAAe;QACjB,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED,mDAAmD;IACnD,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAEtD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;YACtB,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC;YACvC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,KAAK,SAAS;gBACtC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE;YACrB,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;SAC/D,CAAC,CAAC;QAEH,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAChC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAClC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;oBAClB,IAAI,EAAE,IAAI,CAAC,QAAQ;oBACnB,SAAS,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAA4B;oBACxD,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;iBACR,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,IAAI,CAAC,IAAI,KAAK,EAAE,EAAE,CAAC;gBACrB,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAChD,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;IAEO,UAAU,CAAC,IAA+C;QAGhE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QAC9B,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC1B,OAAO,EAAE,MAAM,EAAE,2CAA2C,EAAE,CAAC;QACjE,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAEO,OAAO;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;YAC5C,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,UAAU,EAAE,GAAG,UAAU,CAAC,WAAW,CAAC;YACpE,OAAO;gBACL,UAAU,CAAC,IAAI;gBACf,WAAW,CAAC;oBACV,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,EAAE;oBACzC,WAAW,EAAE,UAAU,CAAC,UAAU,CAAC;oBACnC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE;wBACvC,IAAI,MAA+C,CAAC;wBACpD,IAAI,CAAC;4BACH,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;gCAClC,IAAI,EAAE,UAAU,CAAC,IAAI;gCACrB,SAAS,EAAE,CAAC,KAAK,IAAI,EAAE,CAA4B;6BACpD,CAAC,CAAC;wBACL,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BACf,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,sCAAsC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC/F,CAAC;4BACF,MAAM,KAAK,CAAC;wBACd,CAAC;wBACD,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;4BAC5B,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,+BAA+B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAChE,CAAC;wBACJ,CAAC;wBACD,OAAO,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC;oBAClC,CAAC;iBACF,CAAC;aACM,CAAC;QACb,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;CACF"}
|
package/dist/index.d.ts
CHANGED
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,aAAa,EACb,YAAY,GACb,MAAM,8BAA8B,CAAC;AACtC,YAAY,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,aAAa,EACb,YAAY,GACb,MAAM,8BAA8B,CAAC;AACtC,YAAY,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
|
package/dist/index.js
CHANGED
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { LanguageModel } from "ai";
|
|
1
2
|
import { Chat } from "./chat.js";
|
|
2
3
|
/**
|
|
3
4
|
* Opens a fresh MCP session and conversation against the server the plugin
|
|
@@ -9,8 +10,11 @@ import { Chat } from "./chat.js";
|
|
|
9
10
|
* `start<AppType>()` returns a `Chat<AppType>`, and `expect.chat` infers the
|
|
10
11
|
* tool names and argument shapes from it.
|
|
11
12
|
*/
|
|
12
|
-
export declare function start<App>(
|
|
13
|
+
export declare function start<App>(options: {
|
|
14
|
+
/** Any AI SDK model instance, built by the project. */
|
|
15
|
+
model: LanguageModel;
|
|
13
16
|
systemPrompt?: string;
|
|
14
17
|
temperature?: number;
|
|
18
|
+
maxSteps?: number;
|
|
15
19
|
}): Promise<Chat<App>>;
|
|
16
20
|
//# sourceMappingURL=session-registry.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAExC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC;;;;;;;;;GASG;AACH,wBAAsB,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE;IACxC,uDAAuD;IACvD,KAAK,EAAE,aAAa,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAgBrB"}
|
package/dist/session-registry.js
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { inject, onTestFinished } from "vitest";
|
|
2
2
|
import { Chat } from "./chat.js";
|
|
3
|
-
import { resolveModel } from "./model.js";
|
|
4
3
|
/**
|
|
5
4
|
* Opens a fresh MCP session and conversation against the server the plugin
|
|
6
5
|
* started. The session is closed when the current test finishes, so scenarios
|
|
@@ -11,17 +10,17 @@ import { resolveModel } from "./model.js";
|
|
|
11
10
|
* `start<AppType>()` returns a `Chat<AppType>`, and `expect.chat` infers the
|
|
12
11
|
* tool names and argument shapes from it.
|
|
13
12
|
*/
|
|
14
|
-
export async function start(
|
|
15
|
-
const config = inject("skybridgeEvals");
|
|
13
|
+
export async function start(options) {
|
|
16
14
|
const url = inject("skybridgeEvalsUrl");
|
|
17
|
-
if (
|
|
15
|
+
if (url === undefined) {
|
|
18
16
|
throw new Error("No eval server is running. Add `skybridge({ evals: {...} })` to the vitest config.");
|
|
19
17
|
}
|
|
18
|
+
const config = inject("skybridgeEvals");
|
|
20
19
|
const chat = await Chat.open(url, {
|
|
21
|
-
model:
|
|
22
|
-
temperature:
|
|
23
|
-
systemPrompt:
|
|
24
|
-
maxSteps:
|
|
20
|
+
model: options.model,
|
|
21
|
+
temperature: options.temperature ?? config?.temperature,
|
|
22
|
+
systemPrompt: options.systemPrompt ?? config?.systemPrompt,
|
|
23
|
+
maxSteps: options.maxSteps ?? config?.maxSteps,
|
|
25
24
|
});
|
|
26
25
|
onTestFinished(() => chat.close());
|
|
27
26
|
return chat;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-registry.js","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"session-registry.js","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CAAM,OAMhC;IACC,MAAM,GAAG,GAAG,MAAM,CAAC,mBAAmB,CAAC,CAAC;IACxC,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CACb,oFAAoF,CACrF,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAM,GAAG,EAAE;QACrC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,MAAM,EAAE,WAAW;QACvD,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,MAAM,EAAE,YAAY;QAC1D,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,MAAM,EAAE,QAAQ;KAC/C,CAAC,CAAC;IACH,cAAc,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;IACnC,OAAO,IAAI,CAAC;AACd,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@skybridge/test",
|
|
3
|
-
"version": "0.0.0-dev.
|
|
3
|
+
"version": "0.0.0-dev.0ad9d4b",
|
|
4
4
|
"description": "Eval runner for Skybridge MCP apps: does a prompt reach the right tool with the right arguments",
|
|
5
5
|
"license": "ISC",
|
|
6
6
|
"type": "module",
|
|
@@ -12,10 +12,6 @@
|
|
|
12
12
|
".": {
|
|
13
13
|
"types": "./dist/index.d.ts",
|
|
14
14
|
"default": "./dist/index.js"
|
|
15
|
-
},
|
|
16
|
-
"./vitest-config": {
|
|
17
|
-
"types": "./dist/vitest-config.d.ts",
|
|
18
|
-
"default": "./dist/vitest-config.js"
|
|
19
15
|
}
|
|
20
16
|
},
|
|
21
17
|
"files": [
|
|
@@ -23,47 +19,23 @@
|
|
|
23
19
|
],
|
|
24
20
|
"dependencies": {
|
|
25
21
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
26
|
-
"
|
|
27
|
-
"@skybridge/vite-plugin": "0.0.0-dev.09043e8"
|
|
22
|
+
"@skybridge/vite-plugin": "0.0.0-dev.0ad9d4b"
|
|
28
23
|
},
|
|
29
24
|
"peerDependencies": {
|
|
30
|
-
"
|
|
31
|
-
"@ai-sdk/google": "^2.0.0",
|
|
32
|
-
"@ai-sdk/mistral": "^2.0.0",
|
|
33
|
-
"@ai-sdk/openai": "^2.0.0",
|
|
25
|
+
"ai": "^5.0.0",
|
|
34
26
|
"skybridge": "^2.0.0",
|
|
35
27
|
"vitest": "^4.1.0"
|
|
36
28
|
},
|
|
37
|
-
"peerDependenciesMeta": {
|
|
38
|
-
"@ai-sdk/anthropic": {
|
|
39
|
-
"optional": true
|
|
40
|
-
},
|
|
41
|
-
"@ai-sdk/google": {
|
|
42
|
-
"optional": true
|
|
43
|
-
},
|
|
44
|
-
"@ai-sdk/mistral": {
|
|
45
|
-
"optional": true
|
|
46
|
-
},
|
|
47
|
-
"@ai-sdk/openai": {
|
|
48
|
-
"optional": true
|
|
49
|
-
}
|
|
50
|
-
},
|
|
51
29
|
"devDependencies": {
|
|
52
|
-
"@ai-sdk/anthropic": "^2.0.0",
|
|
53
|
-
"@ai-sdk/google": "^2.0.0",
|
|
54
|
-
"@ai-sdk/mistral": "^2.0.0",
|
|
55
|
-
"@ai-sdk/openai": "^2.0.0",
|
|
56
30
|
"@types/node": "^24.13.3",
|
|
31
|
+
"ai": "^5.0.0",
|
|
57
32
|
"typescript": "^5.9.3",
|
|
58
33
|
"vitest": "^4.1.10",
|
|
59
|
-
"skybridge": "0.0.0-dev.
|
|
34
|
+
"skybridge": "0.0.0-dev.0ad9d4b"
|
|
60
35
|
},
|
|
61
36
|
"engines": {
|
|
62
37
|
"node": ">=22.23.1"
|
|
63
38
|
},
|
|
64
|
-
"publishConfig": {
|
|
65
|
-
"tag": "beta"
|
|
66
|
-
},
|
|
67
39
|
"scripts": {
|
|
68
40
|
"build": "tsc -p tsconfig.json",
|
|
69
41
|
"format": "biome check --write --error-on-warnings",
|
package/dist/model.d.ts
DELETED
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
import type { LanguageModel } from "ai";
|
|
2
|
-
/**
|
|
3
|
-
* Registers a model instance from a setup file, taking precedence over the
|
|
4
|
-
* configured `model` string. The escape hatch for anything the provider
|
|
5
|
-
* prefixes cannot express, such as a local runtime or a company LLM gateway.
|
|
6
|
-
*/
|
|
7
|
-
export declare function defineEvalModel(model: LanguageModel): void;
|
|
8
|
-
/**
|
|
9
|
-
* Turns a `provider/model-id` string into a model instance, importing only the
|
|
10
|
-
* provider named. Each provider package reads its own conventional API key
|
|
11
|
-
* variable, so no key configuration is needed. An unrecognised prefix is
|
|
12
|
-
* returned untouched for the AI SDK's default provider to resolve.
|
|
13
|
-
*/
|
|
14
|
-
export declare function resolveModel(spec: string | undefined): Promise<LanguageModel>;
|
|
15
|
-
//# sourceMappingURL=model.d.ts.map
|
package/dist/model.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"model.d.ts","sourceRoot":"","sources":["../src/model.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAaxC;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAE1D;AAED;;;;;GAKG;AACH,wBAAsB,YAAY,CAChC,IAAI,EAAE,MAAM,GAAG,SAAS,GACvB,OAAO,CAAC,aAAa,CAAC,CAwBxB"}
|
package/dist/model.js
DELETED
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
const PROVIDERS = {
|
|
2
|
-
anthropic: () => import("@ai-sdk/anthropic").then((m) => m.anthropic),
|
|
3
|
-
google: () => import("@ai-sdk/google").then((m) => m.google),
|
|
4
|
-
mistral: () => import("@ai-sdk/mistral").then((m) => m.mistral),
|
|
5
|
-
openai: () => import("@ai-sdk/openai").then((m) => m.openai),
|
|
6
|
-
};
|
|
7
|
-
let registered;
|
|
8
|
-
/**
|
|
9
|
-
* Registers a model instance from a setup file, taking precedence over the
|
|
10
|
-
* configured `model` string. The escape hatch for anything the provider
|
|
11
|
-
* prefixes cannot express, such as a local runtime or a company LLM gateway.
|
|
12
|
-
*/
|
|
13
|
-
export function defineEvalModel(model) {
|
|
14
|
-
registered = model;
|
|
15
|
-
}
|
|
16
|
-
/**
|
|
17
|
-
* Turns a `provider/model-id` string into a model instance, importing only the
|
|
18
|
-
* provider named. Each provider package reads its own conventional API key
|
|
19
|
-
* variable, so no key configuration is needed. An unrecognised prefix is
|
|
20
|
-
* returned untouched for the AI SDK's default provider to resolve.
|
|
21
|
-
*/
|
|
22
|
-
export async function resolveModel(spec) {
|
|
23
|
-
if (registered !== undefined) {
|
|
24
|
-
return registered;
|
|
25
|
-
}
|
|
26
|
-
if (spec === undefined) {
|
|
27
|
-
throw new Error("No model configured. Set `model` on the evals plugin, or call defineEvalModel() from a setup file.");
|
|
28
|
-
}
|
|
29
|
-
const separator = spec.indexOf("/");
|
|
30
|
-
if (separator === -1) {
|
|
31
|
-
return spec;
|
|
32
|
-
}
|
|
33
|
-
const name = spec.slice(0, separator);
|
|
34
|
-
const load = PROVIDERS[name];
|
|
35
|
-
if (load === undefined) {
|
|
36
|
-
return spec;
|
|
37
|
-
}
|
|
38
|
-
const provider = await load().catch(() => {
|
|
39
|
-
throw new Error(`The model "${spec}" needs @ai-sdk/${name}@^2. Install it as a dev dependency.`);
|
|
40
|
-
});
|
|
41
|
-
return provider(spec.slice(separator + 1));
|
|
42
|
-
}
|
|
43
|
-
//# sourceMappingURL=model.js.map
|
package/dist/model.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"model.js","sourceRoot":"","sources":["../src/model.ts"],"names":[],"mappings":"AAIA,MAAM,SAAS,GAAmD;IAChE,SAAS,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IACrE,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IAC5D,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;IAC/D,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;CAC7D,CAAC;AAEF,IAAI,UAAqC,CAAC;AAE1C;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,KAAoB;IAClD,UAAU,GAAG,KAAK,CAAC;AACrB,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAAwB;IAExB,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,oGAAoG,CACrG,CAAC;IACJ,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACpC,IAAI,SAAS,KAAK,CAAC,CAAC,EAAE,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC7B,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE;QACvC,MAAM,IAAI,KAAK,CACb,cAAc,IAAI,mBAAmB,IAAI,sCAAsC,CAChF,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC;AAC7C,CAAC"}
|