@skybridge/test 0.0.0-dev.7aada9d → 0.0.0-dev.9a3a28d
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +90 -0
- package/dist/chat.d.ts +5 -24
- package/dist/chat.d.ts.map +1 -1
- package/dist/chat.js +27 -6
- package/dist/chat.js.map +1 -1
- package/dist/index.d.ts +2 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/model.d.ts +10 -4
- package/dist/model.d.ts.map +1 -1
- package/dist/model.js +31 -16
- package/dist/model.js.map +1 -1
- package/dist/session-registry.d.ts +3 -3
- package/dist/session-registry.d.ts.map +1 -1
- package/dist/session-registry.js +10 -9
- package/dist/session-registry.js.map +1 -1
- package/package.json +33 -10
- package/dist/config.d.ts +0 -40
- package/dist/config.d.ts.map +0 -1
- package/dist/config.js +0 -2
- package/dist/config.js.map +0 -1
- package/dist/global-setup.d.ts +0 -3
- package/dist/global-setup.d.ts.map +0 -1
- package/dist/global-setup.js +0 -39
- package/dist/global-setup.js.map +0 -1
- package/dist/matchers.d.ts +0 -21
- package/dist/matchers.d.ts.map +0 -1
- package/dist/matchers.js +0 -76
- package/dist/matchers.js.map +0 -1
- package/dist/plugin.d.ts +0 -10
- package/dist/plugin.d.ts.map +0 -1
- package/dist/plugin.js +0 -32
- package/dist/plugin.js.map +0 -1
- package/dist/server-harness.d.ts +0 -19
- package/dist/server-harness.d.ts.map +0 -1
- package/dist/server-harness.js +0 -78
- package/dist/server-harness.js.map +0 -1
- package/dist/setup.d.ts +0 -2
- package/dist/setup.d.ts.map +0 -1
- package/dist/setup.js +0 -7
- package/dist/setup.js.map +0 -1
- package/dist/telemetry.d.ts +0 -10
- package/dist/telemetry.d.ts.map +0 -1
- package/dist/telemetry.js +0 -77
- package/dist/telemetry.js.map +0 -1
package/README.md
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
# @skybridge/test
|
|
2
|
+
|
|
3
|
+
Evals for [Skybridge](https://skybridge.tech) MCP apps: they answer whether a
|
|
4
|
+
prompt reaches the right tool with the right arguments, which no unit test can
|
|
5
|
+
check.
|
|
6
|
+
|
|
7
|
+
**Beta.** The API is still settling, so it publishes under the `beta`
|
|
8
|
+
dist-tag and may change between releases.
|
|
9
|
+
|
|
10
|
+
Add the `evals` option to the Skybridge Vite plugin, then write scenarios with
|
|
11
|
+
vitest's own `it` and `expect`.
|
|
12
|
+
|
|
13
|
+
```ts
|
|
14
|
+
import { expect, it } from "vitest";
|
|
15
|
+
import { start } from "@skybridge/test";
|
|
16
|
+
import type { AppType } from "../src/server.js";
|
|
17
|
+
|
|
18
|
+
it("finds products by category", async () => {
|
|
19
|
+
const chat = await start<AppType>();
|
|
20
|
+
await chat.send("I am looking for ski goggles");
|
|
21
|
+
|
|
22
|
+
expect.chat(chat).toHaveCalledToolWith("search-products", {
|
|
23
|
+
category: "goggles",
|
|
24
|
+
});
|
|
25
|
+
});
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Choosing a model
|
|
29
|
+
|
|
30
|
+
`model` is a `provider/model-id` string. Only the provider you name is
|
|
31
|
+
imported, so install just that one as a dev dependency, and the key is read
|
|
32
|
+
from its own environment variable.
|
|
33
|
+
|
|
34
|
+
| prefix | package | key |
|
|
35
|
+
| --- | --- | --- |
|
|
36
|
+
| `anthropic` | `@ai-sdk/anthropic` | `ANTHROPIC_API_KEY` |
|
|
37
|
+
| `openai` | `@ai-sdk/openai` | `OPENAI_API_KEY` |
|
|
38
|
+
| `mistral` | `@ai-sdk/mistral` | `MISTRAL_API_KEY` |
|
|
39
|
+
| `google` | `@ai-sdk/google` | `GOOGLE_GENERATIVE_AI_API_KEY` |
|
|
40
|
+
|
|
41
|
+
```ts
|
|
42
|
+
skybridge({
|
|
43
|
+
evals: {
|
|
44
|
+
model: "anthropic/claude-sonnet-4-5",
|
|
45
|
+
project: { cwd: ".", command: ["node", "dist/server.js"] },
|
|
46
|
+
},
|
|
47
|
+
});
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
`command` has to start the server process directly, since wrappers such as
|
|
51
|
+
`pnpm run` do not forward the channel the runner uses to learn the port.
|
|
52
|
+
|
|
53
|
+
Any other prefix is passed to the AI SDK untouched, so a string such as
|
|
54
|
+
`openai/gpt-5` also resolves through the Vercel AI Gateway if you have
|
|
55
|
+
`AI_GATEWAY_API_KEY` set and no provider package installed.
|
|
56
|
+
|
|
57
|
+
## Models the prefixes cannot reach
|
|
58
|
+
|
|
59
|
+
For a local runtime, a company gateway, or a model that needs middleware,
|
|
60
|
+
build the model yourself and register it from a vitest setup file. It takes
|
|
61
|
+
precedence over `model`, which can then be left out.
|
|
62
|
+
|
|
63
|
+
```ts
|
|
64
|
+
// eval-model.ts
|
|
65
|
+
import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
|
|
66
|
+
import { defineEvalModel } from "@skybridge/test";
|
|
67
|
+
|
|
68
|
+
defineEvalModel(
|
|
69
|
+
createOpenAICompatible({ baseURL: "http://localhost:11434/v1" })("llama3.1"),
|
|
70
|
+
);
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
```ts
|
|
74
|
+
// vitest.config.ts
|
|
75
|
+
export default defineConfig({
|
|
76
|
+
plugins: [skybridge({ evals: { project: { ... } } })],
|
|
77
|
+
test: { setupFiles: ["./eval-model.ts"] },
|
|
78
|
+
});
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Tool calls the server rejected do not count towards a positive assertion, so a
|
|
82
|
+
scenario cannot pass on a call that errored. To assert an error on purpose, use
|
|
83
|
+
`toHaveFailedToolCall`.
|
|
84
|
+
|
|
85
|
+
```ts
|
|
86
|
+
expect.chat(chat).toHaveFailedToolCall("create-checkout");
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
See the [documentation](https://docs.skybridge.tech) for setup and the full
|
|
90
|
+
matcher list.
|
package/dist/chat.d.ts
CHANGED
|
@@ -1,32 +1,11 @@
|
|
|
1
|
+
import type { ToolCall, ToolDefinition } from "@skybridge/vite-plugin/evals";
|
|
1
2
|
import { type LanguageModel } from "ai";
|
|
2
|
-
|
|
3
|
-
/**
|
|
4
|
-
* One recorded call, typed against the project's own registry: `name` is a
|
|
5
|
-
* union of the project's tool names and `arguments` is discriminated on it.
|
|
6
|
-
*/
|
|
7
|
-
export type ToolCall<App> = {
|
|
8
|
-
[Name in ToolNames<App>]: {
|
|
9
|
-
name: Name;
|
|
10
|
-
arguments: ToolInput<App, Name>;
|
|
11
|
-
invalid?: boolean;
|
|
12
|
-
};
|
|
13
|
-
}[ToolNames<App>];
|
|
14
|
-
/**
|
|
15
|
-
* How the host is configured. `model` is an AI SDK model instance, so the
|
|
16
|
-
* provider and its credentials are chosen by the project rather than by a
|
|
17
|
-
* base URL the runner has to know about.
|
|
18
|
-
*/
|
|
19
|
-
export interface HostConfig {
|
|
3
|
+
interface HostConfig {
|
|
20
4
|
model: LanguageModel;
|
|
21
5
|
temperature?: number;
|
|
22
6
|
systemPrompt?: string;
|
|
23
7
|
maxSteps?: number;
|
|
24
8
|
}
|
|
25
|
-
interface McpToolDefinition {
|
|
26
|
-
name: string;
|
|
27
|
-
description?: string;
|
|
28
|
-
inputSchema: Record<string, unknown>;
|
|
29
|
-
}
|
|
30
9
|
/**
|
|
31
10
|
* One conversation against one MCP session. A scenario asserts on
|
|
32
11
|
* {@link Chat.toolCalls}, the sequence of calls the model made.
|
|
@@ -35,6 +14,7 @@ export declare class Chat<App = unknown> {
|
|
|
35
14
|
/** Every call the model made, in order, as it crossed the wire. */
|
|
36
15
|
readonly toolCalls: ToolCall<App>[];
|
|
37
16
|
private readonly messages;
|
|
17
|
+
private readonly failures;
|
|
38
18
|
private readonly client;
|
|
39
19
|
private readonly transport;
|
|
40
20
|
private readonly host;
|
|
@@ -42,10 +22,11 @@ export declare class Chat<App = unknown> {
|
|
|
42
22
|
private constructor();
|
|
43
23
|
static open<App>(serverUrl: string, host: HostConfig): Promise<Chat<App>>;
|
|
44
24
|
/** The tool definitions the model was looking at, for failure reports. */
|
|
45
|
-
get toolDefinitions():
|
|
25
|
+
get toolDefinitions(): ToolDefinition[];
|
|
46
26
|
/** Takes a turn and returns the model's answer. */
|
|
47
27
|
send(prompt: string): Promise<string>;
|
|
48
28
|
close(): Promise<void>;
|
|
29
|
+
private failureFor;
|
|
49
30
|
private toolSet;
|
|
50
31
|
}
|
|
51
32
|
export {};
|
package/dist/chat.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAEA,OAAO,
|
|
1
|
+
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC7E,OAAO,EAIL,KAAK,aAAa,EAGnB,MAAM,IAAI,CAAC;AAEZ,UAAU,UAAU;IAClB,KAAK,EAAE,aAAa,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAID;;;GAGG;AACH,qBAAa,IAAI,CAAC,GAAG,GAAG,OAAO;IAC7B,mEAAmE;IACnE,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAM;IAEzC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA6B;IACtD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAa;IAClC,OAAO,CAAC,KAAK,CAAwB;IAErC,OAAO;WAUM,IAAI,CAAC,GAAG,EACnB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,UAAU,GACf,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAerB,0EAA0E;IAC1E,IAAI,eAAe,IAAI,cAAc,EAAE,CAEtC;IAED,mDAAmD;IAC7C,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IA4BrC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,OAAO,CAAC,UAAU;IAalB,OAAO,CAAC,OAAO;CAmChB"}
|
package/dist/chat.js
CHANGED
|
@@ -10,6 +10,7 @@ export class Chat {
|
|
|
10
10
|
/** Every call the model made, in order, as it crossed the wire. */
|
|
11
11
|
toolCalls = [];
|
|
12
12
|
messages = [];
|
|
13
|
+
failures = new Map();
|
|
13
14
|
client;
|
|
14
15
|
transport;
|
|
15
16
|
host;
|
|
@@ -56,7 +57,7 @@ export class Chat {
|
|
|
56
57
|
this.toolCalls.push({
|
|
57
58
|
name: call.toolName,
|
|
58
59
|
arguments: (call.input ?? {}),
|
|
59
|
-
...(call
|
|
60
|
+
...this.failureFor(call),
|
|
60
61
|
});
|
|
61
62
|
}
|
|
62
63
|
}
|
|
@@ -66,6 +67,16 @@ export class Chat {
|
|
|
66
67
|
async close() {
|
|
67
68
|
await this.transport.close();
|
|
68
69
|
}
|
|
70
|
+
failureFor(call) {
|
|
71
|
+
const reported = this.failures.get(call.toolCallId);
|
|
72
|
+
if (reported !== undefined) {
|
|
73
|
+
return { failed: reported };
|
|
74
|
+
}
|
|
75
|
+
if (call.invalid === true) {
|
|
76
|
+
return { failed: "the model's arguments could not be parsed" };
|
|
77
|
+
}
|
|
78
|
+
return {};
|
|
79
|
+
}
|
|
69
80
|
toolSet() {
|
|
70
81
|
const entries = this.tools.map((definition) => {
|
|
71
82
|
const { $schema: _ignored, ...parameters } = definition.inputSchema;
|
|
@@ -74,11 +85,21 @@ export class Chat {
|
|
|
74
85
|
dynamicTool({
|
|
75
86
|
description: definition.description ?? "",
|
|
76
87
|
inputSchema: jsonSchema(parameters),
|
|
77
|
-
execute: async (input) => {
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
88
|
+
execute: async (input, { toolCallId }) => {
|
|
89
|
+
let result;
|
|
90
|
+
try {
|
|
91
|
+
result = await this.client.callTool({
|
|
92
|
+
name: definition.name,
|
|
93
|
+
arguments: (input ?? {}),
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
catch (error) {
|
|
97
|
+
this.failures.set(toolCallId, `the call never reached the server: ${error instanceof Error ? error.message : String(error)}`);
|
|
98
|
+
throw error;
|
|
99
|
+
}
|
|
100
|
+
if (result.isError === true) {
|
|
101
|
+
this.failures.set(toolCallId, `the tool returned an error: ${JSON.stringify(result.content)}`);
|
|
102
|
+
}
|
|
82
103
|
return result.content ?? result;
|
|
83
104
|
},
|
|
84
105
|
}),
|
package/dist/chat.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,6BAA6B,EAAE,MAAM,oDAAoD,CAAC;
|
|
1
|
+
{"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,6BAA6B,EAAE,MAAM,oDAAoD,CAAC;AAEnG,OAAO,EACL,WAAW,EACX,YAAY,EACZ,UAAU,EAGV,WAAW,GACZ,MAAM,IAAI,CAAC;AASZ,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B;;;GAGG;AACH,MAAM,OAAO,IAAI;IACf,mEAAmE;IAC1D,SAAS,GAAoB,EAAE,CAAC;IAExB,QAAQ,GAAmB,EAAE,CAAC;IAC9B,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrC,MAAM,CAAS;IACf,SAAS,CAAgC;IACzC,IAAI,CAAa;IAC1B,KAAK,GAAqB,EAAE,CAAC;IAErC,YACE,MAAc,EACd,SAAwC,EACxC,IAAgB;QAEhB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,IAAI,CACf,SAAiB,EACjB,IAAgB;QAEhB,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,IAAI,6BAA6B,CAAC,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;QACxE,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAM,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QACpD,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACrB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YACxB,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,0EAA0E;IAC1E,IAAI,eAAe;QACjB,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED,mDAAmD;IACnD,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAEtD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;YACtB,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC;YACvC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,KAAK,SAAS;gBACtC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE;YACrB,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;SAC/D,CAAC,CAAC;QAEH,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAChC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAClC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;oBAClB,IAAI,EAAE,IAAI,CAAC,QAAQ;oBACnB,SAAS,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAA4B;oBACxD,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;iBACR,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAChD,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;IAEO,UAAU,CAAC,IAA+C;QAGhE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QAC9B,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC1B,OAAO,EAAE,MAAM,EAAE,2CAA2C,EAAE,CAAC;QACjE,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAEO,OAAO;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;YAC5C,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,UAAU,EAAE,GAAG,UAAU,CAAC,WAAW,CAAC;YACpE,OAAO;gBACL,UAAU,CAAC,IAAI;gBACf,WAAW,CAAC;oBACV,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,EAAE;oBACzC,WAAW,EAAE,UAAU,CAAC,UAAU,CAAC;oBACnC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE;wBACvC,IAAI,MAA+C,CAAC;wBACpD,IAAI,CAAC;4BACH,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;gCAClC,IAAI,EAAE,UAAU,CAAC,IAAI;gCACrB,SAAS,EAAE,CAAC,KAAK,IAAI,EAAE,CAA4B;6BACpD,CAAC,CAAC;wBACL,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BACf,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,sCAAsC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC/F,CAAC;4BACF,MAAM,KAAK,CAAC;wBACd,CAAC;wBACD,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;4BAC5B,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,+BAA+B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAChE,CAAC;wBACJ,CAAC;wBACD,OAAO,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC;oBAClC,CAAC;iBACF,CAAC;aACM,CAAC;QACb,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;CACF"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
export type {
|
|
2
|
-
export type {
|
|
3
|
-
export type { ChatAssertion, ChatMatchers } from "./matchers.js";
|
|
1
|
+
export type { ChatAssertion, ChatMatchers, } from "@skybridge/vite-plugin/evals";
|
|
2
|
+
export type { Chat } from "./chat.js";
|
|
4
3
|
export { defineEvalModel } from "./model.js";
|
|
5
4
|
export { start } from "./session-registry.js";
|
|
6
5
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,aAAa,EACb,YAAY,GACb,MAAM,8BAA8B,CAAC;AACtC,YAAY,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
|
package/dist/model.d.ts
CHANGED
|
@@ -1,9 +1,15 @@
|
|
|
1
1
|
import type { LanguageModel } from "ai";
|
|
2
|
-
import type { ModelDescriptor } from "./config.js";
|
|
3
2
|
/**
|
|
4
|
-
* Registers a model instance from a setup file
|
|
5
|
-
*
|
|
3
|
+
* Registers a model instance from a setup file, taking precedence over the
|
|
4
|
+
* configured `model` string. The escape hatch for anything the provider
|
|
5
|
+
* prefixes cannot express, such as a local runtime or a company LLM gateway.
|
|
6
6
|
*/
|
|
7
7
|
export declare function defineEvalModel(model: LanguageModel): void;
|
|
8
|
-
|
|
8
|
+
/**
|
|
9
|
+
* Turns a `provider/model-id` string into a model instance, importing only the
|
|
10
|
+
* provider named. Each provider package reads its own conventional API key
|
|
11
|
+
* variable, so no key configuration is needed. An unrecognised prefix is
|
|
12
|
+
* returned untouched for the AI SDK's default provider to resolve.
|
|
13
|
+
*/
|
|
14
|
+
export declare function resolveModel(spec: string | undefined): Promise<LanguageModel>;
|
|
9
15
|
//# sourceMappingURL=model.d.ts.map
|
package/dist/model.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"model.d.ts","sourceRoot":"","sources":["../src/model.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"model.d.ts","sourceRoot":"","sources":["../src/model.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAaxC;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAE1D;AAED;;;;;GAKG;AACH,wBAAsB,YAAY,CAChC,IAAI,EAAE,MAAM,GAAG,SAAS,GACvB,OAAO,CAAC,aAAa,CAAC,CAwBxB"}
|
package/dist/model.js
CHANGED
|
@@ -1,28 +1,43 @@
|
|
|
1
|
+
const PROVIDERS = {
|
|
2
|
+
anthropic: () => import("@ai-sdk/anthropic").then((m) => m.anthropic),
|
|
3
|
+
google: () => import("@ai-sdk/google").then((m) => m.google),
|
|
4
|
+
mistral: () => import("@ai-sdk/mistral").then((m) => m.mistral),
|
|
5
|
+
openai: () => import("@ai-sdk/openai").then((m) => m.openai),
|
|
6
|
+
};
|
|
1
7
|
let registered;
|
|
2
8
|
/**
|
|
3
|
-
* Registers a model instance from a setup file
|
|
4
|
-
*
|
|
9
|
+
* Registers a model instance from a setup file, taking precedence over the
|
|
10
|
+
* configured `model` string. The escape hatch for anything the provider
|
|
11
|
+
* prefixes cannot express, such as a local runtime or a company LLM gateway.
|
|
5
12
|
*/
|
|
6
13
|
export function defineEvalModel(model) {
|
|
7
14
|
registered = model;
|
|
8
15
|
}
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
return key;
|
|
17
|
-
}
|
|
18
|
-
export async function resolveModel(descriptor) {
|
|
16
|
+
/**
|
|
17
|
+
* Turns a `provider/model-id` string into a model instance, importing only the
|
|
18
|
+
* provider named. Each provider package reads its own conventional API key
|
|
19
|
+
* variable, so no key configuration is needed. An unrecognised prefix is
|
|
20
|
+
* returned untouched for the AI SDK's default provider to resolve.
|
|
21
|
+
*/
|
|
22
|
+
export async function resolveModel(spec) {
|
|
19
23
|
if (registered !== undefined) {
|
|
20
24
|
return registered;
|
|
21
25
|
}
|
|
22
|
-
if (
|
|
23
|
-
throw new Error("No model configured.
|
|
26
|
+
if (spec === undefined) {
|
|
27
|
+
throw new Error("No model configured. Set `model` on the evals plugin, or call defineEvalModel() from a setup file.");
|
|
28
|
+
}
|
|
29
|
+
const separator = spec.indexOf("/");
|
|
30
|
+
if (separator === -1) {
|
|
31
|
+
return spec;
|
|
32
|
+
}
|
|
33
|
+
const name = spec.slice(0, separator);
|
|
34
|
+
const load = PROVIDERS[name];
|
|
35
|
+
if (load === undefined) {
|
|
36
|
+
return spec;
|
|
24
37
|
}
|
|
25
|
-
const
|
|
26
|
-
|
|
38
|
+
const provider = await load().catch(() => {
|
|
39
|
+
throw new Error(`The model "${spec}" needs @ai-sdk/${name}@^2. Install it as a dev dependency.`);
|
|
40
|
+
});
|
|
41
|
+
return provider(spec.slice(separator + 1));
|
|
27
42
|
}
|
|
28
43
|
//# sourceMappingURL=model.js.map
|
package/dist/model.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"model.js","sourceRoot":"","sources":["../src/model.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"model.js","sourceRoot":"","sources":["../src/model.ts"],"names":[],"mappings":"AAIA,MAAM,SAAS,GAAmD;IAChE,SAAS,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IACrE,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IAC5D,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;IAC/D,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;CAC7D,CAAC;AAEF,IAAI,UAAqC,CAAC;AAE1C;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,KAAoB;IAClD,UAAU,GAAG,KAAK,CAAC;AACrB,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAAwB;IAExB,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,oGAAoG,CACrG,CAAC;IACJ,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACpC,IAAI,SAAS,KAAK,CAAC,CAAC,EAAE,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC7B,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE;QACvC,MAAM,IAAI,KAAK,CACb,cAAc,IAAI,mBAAmB,IAAI,sCAAsC,CAChF,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC;AAC7C,CAAC"}
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { Chat } from "./chat.js";
|
|
2
2
|
/**
|
|
3
3
|
* Opens a fresh MCP session and conversation against the server the plugin
|
|
4
|
-
* started.
|
|
5
|
-
*
|
|
4
|
+
* started. The session is closed when the current test finishes, so scenarios
|
|
5
|
+
* never leak sessions and never write teardown, and tests that run
|
|
6
|
+
* concurrently cannot close each other's sessions.
|
|
6
7
|
*
|
|
7
8
|
* The type parameter pins the assertions to the project's registry:
|
|
8
9
|
* `start<AppType>()` returns a `Chat<AppType>`, and `expect.chat` infers the
|
|
@@ -12,5 +13,4 @@ export declare function start<App>(overrides?: {
|
|
|
12
13
|
systemPrompt?: string;
|
|
13
14
|
temperature?: number;
|
|
14
15
|
}): Promise<Chat<App>>;
|
|
15
|
-
export declare function closeOpenedSessions(): Promise<void>;
|
|
16
16
|
//# sourceMappingURL=session-registry.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC;;;;;;;;;GASG;AACH,wBAAsB,KAAK,CAAC,GAAG,EAAE,SAAS,CAAC,EAAE;IAC3C,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAgBrB"}
|
package/dist/session-registry.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { inject } from "vitest";
|
|
1
|
+
import { inject, onTestFinished } from "vitest";
|
|
2
2
|
import { Chat } from "./chat.js";
|
|
3
3
|
import { resolveModel } from "./model.js";
|
|
4
|
-
const opened = [];
|
|
5
4
|
/**
|
|
6
5
|
* Opens a fresh MCP session and conversation against the server the plugin
|
|
7
|
-
* started.
|
|
8
|
-
*
|
|
6
|
+
* started. The session is closed when the current test finishes, so scenarios
|
|
7
|
+
* never leak sessions and never write teardown, and tests that run
|
|
8
|
+
* concurrently cannot close each other's sessions.
|
|
9
9
|
*
|
|
10
10
|
* The type parameter pins the assertions to the project's registry:
|
|
11
11
|
* `start<AppType>()` returns a `Chat<AppType>`, and `expect.chat` infers the
|
|
@@ -13,16 +13,17 @@ const opened = [];
|
|
|
13
13
|
*/
|
|
14
14
|
export async function start(overrides) {
|
|
15
15
|
const config = inject("skybridgeEvals");
|
|
16
|
-
const
|
|
16
|
+
const url = inject("skybridgeEvalsUrl");
|
|
17
|
+
if (config === undefined || url === undefined) {
|
|
18
|
+
throw new Error("No eval server is running. Add `skybridge({ evals: {...} })` to the vitest config.");
|
|
19
|
+
}
|
|
20
|
+
const chat = await Chat.open(url, {
|
|
17
21
|
model: await resolveModel(config.model),
|
|
18
22
|
temperature: overrides?.temperature ?? config.temperature,
|
|
19
23
|
systemPrompt: overrides?.systemPrompt ?? config.systemPrompt,
|
|
20
24
|
maxSteps: config.maxSteps,
|
|
21
25
|
});
|
|
22
|
-
|
|
26
|
+
onTestFinished(() => chat.close());
|
|
23
27
|
return chat;
|
|
24
28
|
}
|
|
25
|
-
export async function closeOpenedSessions() {
|
|
26
|
-
await Promise.all(opened.splice(0).map((chat) => chat.close()));
|
|
27
|
-
}
|
|
28
29
|
//# sourceMappingURL=session-registry.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-registry.js","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;
|
|
1
|
+
{"version":3,"file":"session-registry.js","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE1C;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CAAM,SAGhC;IACC,MAAM,MAAM,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;IACxC,MAAM,GAAG,GAAG,MAAM,CAAC,mBAAmB,CAAC,CAAC;IACxC,IAAI,MAAM,KAAK,SAAS,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CACb,oFAAoF,CACrF,CAAC;IACJ,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAM,GAAG,EAAE;QACrC,KAAK,EAAE,MAAM,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC;QACvC,WAAW,EAAE,SAAS,EAAE,WAAW,IAAI,MAAM,CAAC,WAAW;QACzD,YAAY,EAAE,SAAS,EAAE,YAAY,IAAI,MAAM,CAAC,YAAY;QAC5D,QAAQ,EAAE,MAAM,CAAC,QAAQ;KAC1B,CAAC,CAAC;IACH,cAAc,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;IACnC,OAAO,IAAI,CAAC;AACd,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@skybridge/test",
|
|
3
|
-
"version": "0.0.0-dev.
|
|
3
|
+
"version": "0.0.0-dev.9a3a28d",
|
|
4
4
|
"description": "Eval runner for Skybridge MCP apps: does a prompt reach the right tool with the right arguments",
|
|
5
5
|
"license": "ISC",
|
|
6
6
|
"type": "module",
|
|
@@ -13,34 +13,57 @@
|
|
|
13
13
|
"types": "./dist/index.d.ts",
|
|
14
14
|
"default": "./dist/index.js"
|
|
15
15
|
},
|
|
16
|
-
"./
|
|
17
|
-
"types": "./dist/
|
|
18
|
-
"default": "./dist/
|
|
16
|
+
"./vitest-config": {
|
|
17
|
+
"types": "./dist/vitest-config.d.ts",
|
|
18
|
+
"default": "./dist/vitest-config.js"
|
|
19
19
|
}
|
|
20
20
|
},
|
|
21
21
|
"files": [
|
|
22
22
|
"dist"
|
|
23
23
|
],
|
|
24
24
|
"dependencies": {
|
|
25
|
-
"@ai-sdk/anthropic": "^2.0.0",
|
|
26
25
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
27
26
|
"ai": "^5.0.0",
|
|
28
|
-
"
|
|
29
|
-
"posthog-node": "^5.45.2"
|
|
27
|
+
"@skybridge/vite-plugin": "0.0.0-dev.9a3a28d"
|
|
30
28
|
},
|
|
31
29
|
"peerDependencies": {
|
|
32
|
-
"
|
|
33
|
-
"
|
|
30
|
+
"@ai-sdk/anthropic": "^2.0.0",
|
|
31
|
+
"@ai-sdk/google": "^2.0.0",
|
|
32
|
+
"@ai-sdk/mistral": "^2.0.0",
|
|
33
|
+
"@ai-sdk/openai": "^2.0.0",
|
|
34
|
+
"skybridge": "^2.0.0",
|
|
35
|
+
"vitest": "^4.1.0"
|
|
36
|
+
},
|
|
37
|
+
"peerDependenciesMeta": {
|
|
38
|
+
"@ai-sdk/anthropic": {
|
|
39
|
+
"optional": true
|
|
40
|
+
},
|
|
41
|
+
"@ai-sdk/google": {
|
|
42
|
+
"optional": true
|
|
43
|
+
},
|
|
44
|
+
"@ai-sdk/mistral": {
|
|
45
|
+
"optional": true
|
|
46
|
+
},
|
|
47
|
+
"@ai-sdk/openai": {
|
|
48
|
+
"optional": true
|
|
49
|
+
}
|
|
34
50
|
},
|
|
35
51
|
"devDependencies": {
|
|
52
|
+
"@ai-sdk/anthropic": "^2.0.0",
|
|
53
|
+
"@ai-sdk/google": "^2.0.0",
|
|
54
|
+
"@ai-sdk/mistral": "^2.0.0",
|
|
55
|
+
"@ai-sdk/openai": "^2.0.0",
|
|
36
56
|
"@types/node": "^24.13.3",
|
|
37
57
|
"typescript": "^5.9.3",
|
|
38
58
|
"vitest": "^4.1.10",
|
|
39
|
-
"skybridge": "0.0.0-dev.
|
|
59
|
+
"skybridge": "0.0.0-dev.9a3a28d"
|
|
40
60
|
},
|
|
41
61
|
"engines": {
|
|
42
62
|
"node": ">=22.23.1"
|
|
43
63
|
},
|
|
64
|
+
"publishConfig": {
|
|
65
|
+
"tag": "beta"
|
|
66
|
+
},
|
|
44
67
|
"scripts": {
|
|
45
68
|
"build": "tsc -p tsconfig.json",
|
|
46
69
|
"format": "biome check --write --error-on-warnings",
|
package/dist/config.d.ts
DELETED
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
/** A provider we can construct ourselves, so no extra file is needed. */
|
|
2
|
-
export interface ModelDescriptor {
|
|
3
|
-
provider: "anthropic";
|
|
4
|
-
name: string;
|
|
5
|
-
/** Env var holding the key. Never the key itself. */
|
|
6
|
-
apiKeyEnv?: string;
|
|
7
|
-
}
|
|
8
|
-
/** Escape hatch: a module that calls `defineEvalModel` with its own instance. */
|
|
9
|
-
export interface ModelFile {
|
|
10
|
-
file: string;
|
|
11
|
-
}
|
|
12
|
-
export interface EvalsOptions {
|
|
13
|
-
model: ModelDescriptor | ModelFile;
|
|
14
|
-
temperature?: number;
|
|
15
|
-
systemPrompt?: string;
|
|
16
|
-
maxSteps?: number;
|
|
17
|
-
server?: string;
|
|
18
|
-
/**
|
|
19
|
-
* How the runner starts the project's server. `command` must start the
|
|
20
|
-
* server process directly (`node`, `tsx`), not through a wrapper such as
|
|
21
|
-
* `pnpm run`: readiness and the bound port arrive over the spawned
|
|
22
|
-
* process's IPC channel, which wrappers do not forward.
|
|
23
|
-
*/
|
|
24
|
-
project?: {
|
|
25
|
-
cwd: string;
|
|
26
|
-
command: string[];
|
|
27
|
-
env?: Record<string, string>;
|
|
28
|
-
};
|
|
29
|
-
}
|
|
30
|
-
/** The serializable half, the only part that crosses into the workers. */
|
|
31
|
-
export type ProvidedEvalsOptions = Omit<EvalsOptions, "model"> & {
|
|
32
|
-
model?: ModelDescriptor;
|
|
33
|
-
};
|
|
34
|
-
declare module "vitest" {
|
|
35
|
-
interface ProvidedContext {
|
|
36
|
-
skybridgeEvals: ProvidedEvalsOptions;
|
|
37
|
-
skybridgeEvalsUrl: string;
|
|
38
|
-
}
|
|
39
|
-
}
|
|
40
|
-
//# sourceMappingURL=config.d.ts.map
|
package/dist/config.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,yEAAyE;AACzE,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,WAAW,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,qDAAqD;IACrD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,iFAAiF;AACjF,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,eAAe,GAAG,SAAS,CAAC;IACnC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;OAKG;IACH,OAAO,CAAC,EAAE;QACR,GAAG,EAAE,MAAM,CAAC;QACZ,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAC9B,CAAC;CACH;AAED,0EAA0E;AAC1E,MAAM,MAAM,oBAAoB,GAAG,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,GAAG;IAC/D,KAAK,CAAC,EAAE,eAAe,CAAC;CACzB,CAAC;AAEF,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,eAAe;QACvB,cAAc,EAAE,oBAAoB,CAAC;QACrC,iBAAiB,EAAE,MAAM,CAAC;KAC3B;CACF"}
|
package/dist/config.js
DELETED
package/dist/config.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":""}
|
package/dist/global-setup.d.ts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"global-setup.d.ts","sourceRoot":"","sources":["../src/global-setup.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAK/C,wBAA8B,KAAK,CAAC,OAAO,EAAE,WAAW,gCAwCvD"}
|
package/dist/global-setup.js
DELETED
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
import { startServer } from "./server-harness.js";
|
|
2
|
-
import { captureEvalRun } from "./telemetry.js";
|
|
3
|
-
export default async function setup(project) {
|
|
4
|
-
const options = project.config.provide
|
|
5
|
-
?.skybridgeEvals;
|
|
6
|
-
const startedAt = Date.now();
|
|
7
|
-
if (options.server !== undefined) {
|
|
8
|
-
project.provide("skybridgeEvalsUrl", options.server);
|
|
9
|
-
return async () => {
|
|
10
|
-
await captureEvalRun({
|
|
11
|
-
durationMs: Date.now() - startedAt,
|
|
12
|
-
serverStarted: true,
|
|
13
|
-
});
|
|
14
|
-
};
|
|
15
|
-
}
|
|
16
|
-
if (options.project === undefined) {
|
|
17
|
-
throw new Error("The evals plugin needs either `server` or `project`");
|
|
18
|
-
}
|
|
19
|
-
const server = await startServer(options.project).catch(async (error) => {
|
|
20
|
-
await captureEvalRun({
|
|
21
|
-
durationMs: Date.now() - startedAt,
|
|
22
|
-
serverStarted: false,
|
|
23
|
-
});
|
|
24
|
-
throw error;
|
|
25
|
-
});
|
|
26
|
-
project.provide("skybridgeEvalsUrl", server.url);
|
|
27
|
-
return async () => {
|
|
28
|
-
const output = server.output().trim();
|
|
29
|
-
await server.stop();
|
|
30
|
-
if (output !== "") {
|
|
31
|
-
console.log(`\n--- server output ---\n${output}\n---`);
|
|
32
|
-
}
|
|
33
|
-
await captureEvalRun({
|
|
34
|
-
durationMs: Date.now() - startedAt,
|
|
35
|
-
serverStarted: true,
|
|
36
|
-
});
|
|
37
|
-
};
|
|
38
|
-
}
|
|
39
|
-
//# sourceMappingURL=global-setup.js.map
|
package/dist/global-setup.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"global-setup.js","sourceRoot":"","sources":["../src/global-setup.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAEhD,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,KAAK,CAAC,OAAoB;IACtD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO;QACpC,EAAE,cAAsC,CAAC;IAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QACjC,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACrD,OAAO,KAAK,IAAI,EAAE;YAChB,MAAM,cAAc,CAAC;gBACnB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAClC,aAAa,EAAE,IAAI;aACpB,CAAC,CAAC;QACL,CAAC,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,KAAK,CACrD,KAAK,EAAE,KAAc,EAAE,EAAE;QACvB,MAAM,cAAc,CAAC;YACnB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,aAAa,EAAE,KAAK;SACrB,CAAC,CAAC;QACH,MAAM,KAAK,CAAC;IACd,CAAC,CACF,CAAC;IACF,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;IAEjD,OAAO,KAAK,IAAI,EAAE;QAChB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;YAClB,OAAO,CAAC,GAAG,CAAC,4BAA4B,MAAM,OAAO,CAAC,CAAC;QACzD,CAAC;QACD,MAAM,cAAc,CAAC;YACnB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,aAAa,EAAE,IAAI;SACpB,CAAC,CAAC;IACL,CAAC,CAAC;AACJ,CAAC"}
|
package/dist/matchers.d.ts
DELETED
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
import type { ToolInput, ToolNames } from "skybridge/server";
|
|
2
|
-
import type { Chat } from "./chat.js";
|
|
3
|
-
/**
|
|
4
|
-
* Assertions about the tool calls a conversation produced. Kept off vitest's
|
|
5
|
-
* global `Matchers` so they never appear on unrelated `expect()` calls, and so
|
|
6
|
-
* this stays the one place later tool-call helpers are added.
|
|
7
|
-
*/
|
|
8
|
-
export interface ChatMatchers<App> {
|
|
9
|
-
toHaveCalledToolOnce<Name extends ToolNames<App>>(name: Name, args?: Partial<ToolInput<App, Name>>): void;
|
|
10
|
-
toHaveCalledToolWith<Name extends ToolNames<App>>(name: Name, args: Partial<ToolInput<App, Name>>): void;
|
|
11
|
-
toNeverHaveCalledTool<Name extends ToolNames<App>>(name: Name): void;
|
|
12
|
-
}
|
|
13
|
-
export interface ChatAssertion<App> extends ChatMatchers<App> {
|
|
14
|
-
not: ChatMatchers<App>;
|
|
15
|
-
}
|
|
16
|
-
declare module "vitest" {
|
|
17
|
-
interface ExpectStatic {
|
|
18
|
-
chat<App>(chat: Chat<App>): ChatAssertion<App>;
|
|
19
|
-
}
|
|
20
|
-
}
|
|
21
|
-
//# sourceMappingURL=matchers.d.ts.map
|
package/dist/matchers.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"matchers.d.ts","sourceRoot":"","sources":["../src/matchers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7D,OAAO,KAAK,EAAE,IAAI,EAAY,MAAM,WAAW,CAAC;AAEhD;;;;GAIG;AACH,MAAM,WAAW,YAAY,CAAC,GAAG;IAC/B,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAC9C,IAAI,EAAE,IAAI,EACV,IAAI,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GACnC,IAAI,CAAC;IACR,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAC9C,IAAI,EAAE,IAAI,EACV,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GAClC,IAAI,CAAC;IACR,qBAAqB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,CAAC;CACtE;AAED,MAAM,WAAW,aAAa,CAAC,GAAG,CAAE,SAAQ,YAAY,CAAC,GAAG,CAAC;IAC3D,GAAG,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC;CACxB;AAED,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,YAAY;QACpB,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;KAChD;CACF"}
|
package/dist/matchers.js
DELETED
|
@@ -1,76 +0,0 @@
|
|
|
1
|
-
import { expect } from "vitest";
|
|
2
|
-
/**
|
|
3
|
-
* Every attempt at a tool, including calls the tool's schema rejected. Used by
|
|
4
|
-
* the negative matcher: an attempt the model should not have made is a failure
|
|
5
|
-
* whether or not its arguments parsed.
|
|
6
|
-
*/
|
|
7
|
-
function attemptsAt(chat, name) {
|
|
8
|
-
return chat.toolCalls.filter((call) => String(call.name) === name);
|
|
9
|
-
}
|
|
10
|
-
/**
|
|
11
|
-
* Only the calls that actually reached the server. Positive matchers use this,
|
|
12
|
-
* so a rejected call never satisfies an assertion that the tool ran; the
|
|
13
|
-
* rejected attempt still shows up in the failure report.
|
|
14
|
-
*/
|
|
15
|
-
function callsTo(chat, name) {
|
|
16
|
-
return attemptsAt(chat, name).filter((call) => call.invalid !== true);
|
|
17
|
-
}
|
|
18
|
-
function observed(chat) {
|
|
19
|
-
if (chat.toolCalls.length === 0) {
|
|
20
|
-
return " (no tool was called)";
|
|
21
|
-
}
|
|
22
|
-
return chat.toolCalls
|
|
23
|
-
.map((call, index) => ` ${index + 1}. ${String(call.name)} ${JSON.stringify(call.arguments)}${call.invalid === true ? " (arguments rejected by the tool schema)" : ""}`)
|
|
24
|
-
.join("\n");
|
|
25
|
-
}
|
|
26
|
-
function definitions(chat) {
|
|
27
|
-
return chat.toolDefinitions
|
|
28
|
-
.map((tool) => ` - ${tool.name}: ${tool.description ?? ""}\n arguments: ${JSON.stringify(tool.inputSchema)}`)
|
|
29
|
-
.join("\n");
|
|
30
|
-
}
|
|
31
|
-
function report(chat, summary) {
|
|
32
|
-
return `${summary}
|
|
33
|
-
|
|
34
|
-
Calls the model actually made:
|
|
35
|
-
${observed(chat)}
|
|
36
|
-
|
|
37
|
-
Tool definitions the model was looking at:
|
|
38
|
-
${definitions(chat)}`;
|
|
39
|
-
}
|
|
40
|
-
expect.extend({
|
|
41
|
-
toHaveCalledToolOnce(received, name, args) {
|
|
42
|
-
const matching = callsTo(received, name);
|
|
43
|
-
const argsMatch = args === undefined ||
|
|
44
|
-
(matching.length === 1 &&
|
|
45
|
-
this.equals(matching[0]?.arguments, expect.objectContaining(args)));
|
|
46
|
-
const pass = matching.length === 1 && argsMatch;
|
|
47
|
-
return {
|
|
48
|
-
pass,
|
|
49
|
-
message: () => report(received, pass
|
|
50
|
-
? `Expected "${name}" not to be called exactly once.`
|
|
51
|
-
: matching.length === 1
|
|
52
|
-
? `Expected "${name}" to be called once with ${JSON.stringify(args)}, but it was called with ${JSON.stringify(matching[0]?.arguments)}.`
|
|
53
|
-
: `Expected "${name}" to be called exactly once by a call the tool schema accepted, but ${matching.length} such calls were made.`),
|
|
54
|
-
};
|
|
55
|
-
},
|
|
56
|
-
toHaveCalledToolWith(received, name, args) {
|
|
57
|
-
const pass = callsTo(received, name).some((call) => this.equals(call.arguments, expect.objectContaining(args)));
|
|
58
|
-
return {
|
|
59
|
-
pass,
|
|
60
|
-
message: () => report(received, pass
|
|
61
|
-
? `Expected no call to "${name}" with ${JSON.stringify(args)}.`
|
|
62
|
-
: `Expected "${name}" to be called with ${JSON.stringify(args)}.`),
|
|
63
|
-
};
|
|
64
|
-
},
|
|
65
|
-
toNeverHaveCalledTool(received, name) {
|
|
66
|
-
const matching = attemptsAt(received, name);
|
|
67
|
-
return {
|
|
68
|
-
pass: matching.length === 0,
|
|
69
|
-
message: () => report(received, matching.length === 0
|
|
70
|
-
? `Expected "${name}" to be called at least once.`
|
|
71
|
-
: `Expected "${name}" never to be called, but it was called ${matching.length} times.`),
|
|
72
|
-
};
|
|
73
|
-
},
|
|
74
|
-
});
|
|
75
|
-
expect.chat = ((chat) => expect(chat));
|
|
76
|
-
//# sourceMappingURL=matchers.js.map
|
package/dist/matchers.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"matchers.js","sourceRoot":"","sources":["../src/matchers.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AA8BhC;;;;GAIG;AACH,SAAS,UAAU,CAAC,IAAmB,EAAE,IAAY;IACnD,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;AACrE,CAAC;AAED;;;;GAIG;AACH,SAAS,OAAO,CAAC,IAAmB,EAAE,IAAY;IAChD,OAAO,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC;AACxE,CAAC;AAED,SAAS,QAAQ,CAAC,IAAmB;IACnC,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,wBAAwB,CAAC;IAClC,CAAC;IACD,OAAO,IAAI,CAAC,SAAS;SAClB,GAAG,CACF,CAAC,IAAuB,EAAE,KAAa,EAAE,EAAE,CACzC,KAAK,KAAK,GAAG,CAAC,KAAK,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,2CAA2C,CAAC,CAAC,CAAC,EAAE,EAAE,CACtJ;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,WAAW,CAAC,IAAmB;IACtC,OAAO,IAAI,CAAC,eAAe;SACxB,GAAG,CACF,CAAC,IAAI,EAAE,EAAE,CACP,OAAO,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,WAAW,IAAI,EAAE,oBAAoB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CACpG;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,MAAM,CAAC,IAAmB,EAAE,OAAe;IAClD,OAAO,GAAG,OAAO;;;EAGjB,QAAQ,CAAC,IAAI,CAAC;;;EAGd,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;AACtB,CAAC;AAED,MAAM,CAAC,MAAM,CAAC;IACZ,oBAAoB,CAClB,QAAuB,EACvB,IAAY,EACZ,IAA8B;QAE9B,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QACzC,MAAM,SAAS,GACb,IAAI,KAAK,SAAS;YAClB,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACpB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxE,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC;QAEhD,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,aAAa,IAAI,kCAAkC;gBACrD,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;oBACrB,CAAC,CAAC,aAAa,IAAI,4BAA4B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,4BAA4B,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG;oBACxI,CAAC,CAAC,aAAa,IAAI,uEAAuE,QAAQ,CAAC,MAAM,wBAAwB,CACtI;SACJ,CAAC;IACJ,CAAC;IAED,oBAAoB,CAClB,QAAuB,EACvB,IAAY,EACZ,IAA6B;QAE7B,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CACjD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAC3D,CAAC;QAEF,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,wBAAwB,IAAI,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG;gBAC/D,CAAC,CAAC,aAAa,IAAI,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CACpE;SACJ,CAAC;IACJ,CAAC;IAED,qBAAqB,CAAC,QAAuB,EAAE,IAAY;QACzD,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAE5C,OAAO;YACL,IAAI,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC;YAC3B,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACnB,CAAC,CAAC,aAAa,IAAI,+BAA+B;gBAClD,CAAC,CAAC,aAAa,IAAI,2CAA2C,QAAQ,CAAC,MAAM,SAAS,CACzF;SACJ,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,IAAI,GAAG,CAAC,CAAM,IAAe,EAAE,EAAE,CACtC,MAAM,CAAC,IAAI,CAAC,CAAkC,CAAC"}
|
package/dist/plugin.d.ts
DELETED
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
import type { Plugin } from "vitest/config";
|
|
2
|
-
import type { EvalsOptions } from "./config.js";
|
|
3
|
-
/**
|
|
4
|
-
* Wires the eval runner into vitest. Usually reached through the `evals`
|
|
5
|
-
* option of the `skybridge()` Vite plugin rather than registered directly.
|
|
6
|
-
* Everything the runner needs is configured here, so a scenario file imports
|
|
7
|
-
* nothing but `start` and its own `AppType`.
|
|
8
|
-
*/
|
|
9
|
-
export declare function evals(options: EvalsOptions): Plugin;
|
|
10
|
-
//# sourceMappingURL=plugin.d.ts.map
|
package/dist/plugin.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"plugin.d.ts","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,KAAK,EAAE,YAAY,EAAwB,MAAM,aAAa,CAAC;AAMtE;;;;;GAKG;AACH,wBAAgB,KAAK,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM,CAuBnD"}
|
package/dist/plugin.js
DELETED
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
import { fileURLToPath } from "node:url";
|
|
2
|
-
function here(file) {
|
|
3
|
-
return fileURLToPath(new URL(file, import.meta.url));
|
|
4
|
-
}
|
|
5
|
-
/**
|
|
6
|
-
* Wires the eval runner into vitest. Usually reached through the `evals`
|
|
7
|
-
* option of the `skybridge()` Vite plugin rather than registered directly.
|
|
8
|
-
* Everything the runner needs is configured here, so a scenario file imports
|
|
9
|
-
* nothing but `start` and its own `AppType`.
|
|
10
|
-
*/
|
|
11
|
-
export function evals(options) {
|
|
12
|
-
const { model, ...rest } = options;
|
|
13
|
-
const provided = "file" in model ? rest : { ...rest, model };
|
|
14
|
-
const setupFiles = [here("./setup.js")];
|
|
15
|
-
if ("file" in model) {
|
|
16
|
-
setupFiles.push(model.file);
|
|
17
|
-
}
|
|
18
|
-
return {
|
|
19
|
-
name: "skybridge-evals",
|
|
20
|
-
config() {
|
|
21
|
-
return {
|
|
22
|
-
test: {
|
|
23
|
-
testTimeout: 120_000,
|
|
24
|
-
setupFiles,
|
|
25
|
-
globalSetup: [here("./global-setup.js")],
|
|
26
|
-
provide: { skybridgeEvals: provided },
|
|
27
|
-
},
|
|
28
|
-
};
|
|
29
|
-
},
|
|
30
|
-
};
|
|
31
|
-
}
|
|
32
|
-
//# sourceMappingURL=plugin.js.map
|
package/dist/plugin.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"plugin.js","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAIzC,SAAS,IAAI,CAAC,IAAY;IACxB,OAAO,aAAa,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AACvD,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,KAAK,CAAC,OAAqB;IACzC,MAAM,EAAE,KAAK,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;IACnC,MAAM,QAAQ,GACZ,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,CAAC;IAE9C,MAAM,UAAU,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;IACxC,IAAI,MAAM,IAAI,KAAK,EAAE,CAAC;QACpB,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAED,OAAO;QACL,IAAI,EAAE,iBAAiB;QACvB,MAAM;YACJ,OAAO;gBACL,IAAI,EAAE;oBACJ,WAAW,EAAE,OAAO;oBACpB,UAAU;oBACV,WAAW,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;oBACxC,OAAO,EAAE,EAAE,cAAc,EAAE,QAAQ,EAAE;iBACtC;aACF,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC"}
|
package/dist/server-harness.d.ts
DELETED
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
export interface RunningServer {
|
|
2
|
-
url: string;
|
|
3
|
-
/** Everything the server wrote to stdout/stderr, for failure reports. */
|
|
4
|
-
output: () => string;
|
|
5
|
-
stop: () => Promise<void>;
|
|
6
|
-
}
|
|
7
|
-
/**
|
|
8
|
-
* Spawns the project's server with `__PORT=0` and an IPC channel, and waits
|
|
9
|
-
* for the `skybridge:listening` message carrying the port the OS picked.
|
|
10
|
-
* Binding and reporting both happen in the child, so there is no window in
|
|
11
|
-
* which another process can steal the port.
|
|
12
|
-
*/
|
|
13
|
-
export declare function startServer({ cwd, command, env, timeoutMs, }: {
|
|
14
|
-
cwd: string;
|
|
15
|
-
command: string[];
|
|
16
|
-
env?: Record<string, string>;
|
|
17
|
-
timeoutMs?: number;
|
|
18
|
-
}): Promise<RunningServer>;
|
|
19
|
-
//# sourceMappingURL=server-harness.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"server-harness.d.ts","sourceRoot":"","sources":["../src/server-harness.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,aAAa;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,yEAAyE;IACzE,MAAM,EAAE,MAAM,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAiBD;;;;;GAKG;AACH,wBAAsB,WAAW,CAAC,EAChC,GAAG,EACH,OAAO,EACP,GAAQ,EACR,SAAkB,GACnB,EAAE;IACD,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,OAAO,CAAC,aAAa,CAAC,CA2EzB"}
|
package/dist/server-harness.js
DELETED
|
@@ -1,78 +0,0 @@
|
|
|
1
|
-
import { spawn } from "node:child_process";
|
|
2
|
-
import { resolve } from "node:path";
|
|
3
|
-
function hasExited(child) {
|
|
4
|
-
return child.exitCode !== null || child.signalCode !== null;
|
|
5
|
-
}
|
|
6
|
-
function isListeningMessage(message) {
|
|
7
|
-
return (typeof message === "object" &&
|
|
8
|
-
message !== null &&
|
|
9
|
-
message.type === "skybridge:listening" &&
|
|
10
|
-
typeof message.port === "number");
|
|
11
|
-
}
|
|
12
|
-
/**
|
|
13
|
-
* Spawns the project's server with `__PORT=0` and an IPC channel, and waits
|
|
14
|
-
* for the `skybridge:listening` message carrying the port the OS picked.
|
|
15
|
-
* Binding and reporting both happen in the child, so there is no window in
|
|
16
|
-
* which another process can steal the port.
|
|
17
|
-
*/
|
|
18
|
-
export async function startServer({ cwd, command, env = {}, timeoutMs = 60_000, }) {
|
|
19
|
-
const [bin, ...args] = command;
|
|
20
|
-
if (bin === undefined) {
|
|
21
|
-
throw new Error("Empty server command");
|
|
22
|
-
}
|
|
23
|
-
const executable = bin.startsWith(".") ? resolve(cwd, bin) : bin;
|
|
24
|
-
const child = spawn(executable, args, {
|
|
25
|
-
cwd,
|
|
26
|
-
env: {
|
|
27
|
-
...process.env,
|
|
28
|
-
NODE_ENV: "production",
|
|
29
|
-
...env,
|
|
30
|
-
__PORT: "0",
|
|
31
|
-
},
|
|
32
|
-
stdio: ["ignore", "pipe", "pipe", "ipc"],
|
|
33
|
-
});
|
|
34
|
-
let captured = "";
|
|
35
|
-
const output = () => captured;
|
|
36
|
-
child.stdout?.on("data", (chunk) => {
|
|
37
|
-
captured += chunk;
|
|
38
|
-
});
|
|
39
|
-
child.stderr?.on("data", (chunk) => {
|
|
40
|
-
captured += chunk;
|
|
41
|
-
});
|
|
42
|
-
const stop = async () => {
|
|
43
|
-
if (hasExited(child)) {
|
|
44
|
-
return;
|
|
45
|
-
}
|
|
46
|
-
const exited = new Promise((resolveExit) => {
|
|
47
|
-
child.once("exit", () => resolveExit());
|
|
48
|
-
});
|
|
49
|
-
child.kill("SIGTERM");
|
|
50
|
-
const forced = setTimeout(() => child.kill("SIGKILL"), 3_000);
|
|
51
|
-
await exited;
|
|
52
|
-
clearTimeout(forced);
|
|
53
|
-
};
|
|
54
|
-
const port = await new Promise((resolvePort, reject) => {
|
|
55
|
-
const timer = setTimeout(() => {
|
|
56
|
-
reject(new Error(`Server did not report its port in ${timeoutMs}ms. It must call \`server.run()\` on a skybridge version that sends the skybridge:listening readiness message.\n${output()}`));
|
|
57
|
-
}, timeoutMs);
|
|
58
|
-
child.on("message", (message) => {
|
|
59
|
-
if (isListeningMessage(message)) {
|
|
60
|
-
clearTimeout(timer);
|
|
61
|
-
resolvePort(message.port);
|
|
62
|
-
}
|
|
63
|
-
});
|
|
64
|
-
child.on("error", (error) => {
|
|
65
|
-
clearTimeout(timer);
|
|
66
|
-
reject(new Error(`Failed to spawn ${command.join(" ")}: ${error.message}`));
|
|
67
|
-
});
|
|
68
|
-
child.on("exit", () => {
|
|
69
|
-
clearTimeout(timer);
|
|
70
|
-
reject(new Error(`Server exited with code ${child.exitCode} before reporting its port:\n${output()}`));
|
|
71
|
-
});
|
|
72
|
-
}).catch(async (error) => {
|
|
73
|
-
await stop();
|
|
74
|
-
throw error;
|
|
75
|
-
});
|
|
76
|
-
return { url: `http://127.0.0.1:${port}/mcp`, output, stop };
|
|
77
|
-
}
|
|
78
|
-
//# sourceMappingURL=server-harness.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"server-harness.js","sourceRoot":"","sources":["../src/server-harness.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqB,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AASpC,SAAS,SAAS,CAAC,KAAmB;IACpC,OAAO,KAAK,CAAC,QAAQ,KAAK,IAAI,IAAI,KAAK,CAAC,UAAU,KAAK,IAAI,CAAC;AAC9D,CAAC;AAED,SAAS,kBAAkB,CACzB,OAAgB;IAEhB,OAAO,CACL,OAAO,OAAO,KAAK,QAAQ;QAC3B,OAAO,KAAK,IAAI;QACf,OAA8B,CAAC,IAAI,KAAK,qBAAqB;QAC9D,OAAQ,OAA8B,CAAC,IAAI,KAAK,QAAQ,CACzD,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,EAChC,GAAG,EACH,OAAO,EACP,GAAG,GAAG,EAAE,EACR,SAAS,GAAG,MAAM,GAMnB;IACC,MAAM,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC;IAC/B,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAC1C,CAAC;IAED,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAEjE,MAAM,KAAK,GAAG,KAAK,CAAC,UAAU,EAAE,IAAI,EAAE;QACpC,GAAG;QACH,GAAG,EAAE;YACH,GAAG,OAAO,CAAC,GAAG;YACd,QAAQ,EAAE,YAAY;YACtB,GAAG,GAAG;YACN,MAAM,EAAE,GAAG;SACZ;QACD,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC;KACzC,CAAC,CAAC;IAEH,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,MAAM,MAAM,GAAG,GAAG,EAAE,CAAC,QAAQ,CAAC;IAC9B,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;QACjC,QAAQ,IAAI,KAAK,CAAC;IACpB,CAAC,CAAC,CAAC;IACH,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;QACjC,QAAQ,IAAI,KAAK,CAAC;IACpB,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,KAAK,IAAI,EAAE;QACtB,IAAI,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;YACrB,OAAO;QACT,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,OAAO,CAAO,CAAC,WAAW,EAAE,EAAE;YAC/C,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtB,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,CAAC,CAAC;QAC9D,MAAM,MAAM,CAAC;QACb,YAAY,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,MAAM,IAAI,OAAO,CAAS,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE;QAC7D,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,MAAM,CACJ,IAAI,KAAK,CACP,qCAAqC,SAAS,mHAAmH,MAAM,EAAE,EAAE,CAC5K,CACF,CAAC;QACJ,CAAC,EAAE,SAAS,CAAC,CAAC;QACd,KAAK,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE;YAC9B,IAAI,kBAAkB,CAAC,OAAO,CAAC,EAAE,CAAC;gBAChC,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YAC1B,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CACJ,IAAI,KAAK,CAAC,mBAAmB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CACpE,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;YACpB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CACJ,IAAI,KAAK,CACP,2BAA2B,KAAK,CAAC,QAAQ,gCAAgC,MAAM,EAAE,EAAE,CACpF,CACF,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,KAAc,EAAE,EAAE;QAChC,MAAM,IAAI,EAAE,CAAC;QACb,MAAM,KAAK,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,GAAG,EAAE,oBAAoB,IAAI,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;AAC/D,CAAC"}
|
package/dist/setup.d.ts
DELETED
package/dist/setup.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"setup.d.ts","sourceRoot":"","sources":["../src/setup.ts"],"names":[],"mappings":"AACA,OAAO,eAAe,CAAC"}
|
package/dist/setup.js
DELETED
package/dist/setup.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"setup.js","sourceRoot":"","sources":["../src/setup.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,eAAe,CAAC;AACvB,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAE5D,SAAS,CAAC,KAAK,IAAI,EAAE;IACnB,MAAM,mBAAmB,EAAE,CAAC;AAC9B,CAAC,CAAC,CAAC"}
|
package/dist/telemetry.d.ts
DELETED
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* One event per eval run, honoring the same opt-outs, machine id file, and
|
|
3
|
-
* env-var semantics as the skybridge CLI. Silently a no-op when telemetry is
|
|
4
|
-
* disabled.
|
|
5
|
-
*/
|
|
6
|
-
export declare function captureEvalRun(properties: {
|
|
7
|
-
durationMs: number;
|
|
8
|
-
serverStarted: boolean;
|
|
9
|
-
}): Promise<void>;
|
|
10
|
-
//# sourceMappingURL=telemetry.d.ts.map
|
package/dist/telemetry.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../src/telemetry.ts"],"names":[],"mappings":"AAyDA;;;;GAIG;AACH,wBAAsB,cAAc,CAAC,UAAU,EAAE;IAC/C,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,OAAO,CAAC;CACxB,GAAG,OAAO,CAAC,IAAI,CAAC,CA8BhB"}
|
package/dist/telemetry.js
DELETED
|
@@ -1,77 +0,0 @@
|
|
|
1
|
-
import { randomUUID } from "node:crypto";
|
|
2
|
-
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
3
|
-
import { homedir } from "node:os";
|
|
4
|
-
import { join } from "node:path";
|
|
5
|
-
import ci from "ci-info";
|
|
6
|
-
import { PostHog } from "posthog-node";
|
|
7
|
-
const POSTHOG_API_KEY = "phc_rQdkCYr0DO4NcZBQXZnUwsHAbau9zuNwKIpil9FQP6v";
|
|
8
|
-
const POSTHOG_HOST = "https://us.i.posthog.com";
|
|
9
|
-
const GLOBAL_CONFIG_DIR = join(homedir(), ".skybridge");
|
|
10
|
-
const GLOBAL_CONFIG_FILE = join(GLOBAL_CONFIG_DIR, "config.json");
|
|
11
|
-
function isFlagSet(value) {
|
|
12
|
-
return value === "1" || value === "true";
|
|
13
|
-
}
|
|
14
|
-
function readGlobalConfig() {
|
|
15
|
-
try {
|
|
16
|
-
if (existsSync(GLOBAL_CONFIG_FILE)) {
|
|
17
|
-
return JSON.parse(readFileSync(GLOBAL_CONFIG_FILE, "utf-8"));
|
|
18
|
-
}
|
|
19
|
-
}
|
|
20
|
-
catch {
|
|
21
|
-
return null;
|
|
22
|
-
}
|
|
23
|
-
return null;
|
|
24
|
-
}
|
|
25
|
-
function getMachineId() {
|
|
26
|
-
const config = readGlobalConfig();
|
|
27
|
-
if (config?.telemetry?.enabled === false) {
|
|
28
|
-
return undefined;
|
|
29
|
-
}
|
|
30
|
-
if (config?.machineId !== undefined) {
|
|
31
|
-
return config.machineId;
|
|
32
|
-
}
|
|
33
|
-
try {
|
|
34
|
-
const machineId = randomUUID();
|
|
35
|
-
mkdirSync(GLOBAL_CONFIG_DIR, { recursive: true });
|
|
36
|
-
writeFileSync(GLOBAL_CONFIG_FILE, JSON.stringify({ ...config, machineId, telemetry: { enabled: true } }, null, 2));
|
|
37
|
-
return machineId;
|
|
38
|
-
}
|
|
39
|
-
catch {
|
|
40
|
-
return undefined;
|
|
41
|
-
}
|
|
42
|
-
}
|
|
43
|
-
/**
|
|
44
|
-
* One event per eval run, honoring the same opt-outs, machine id file, and
|
|
45
|
-
* env-var semantics as the skybridge CLI. Silently a no-op when telemetry is
|
|
46
|
-
* disabled.
|
|
47
|
-
*/
|
|
48
|
-
export async function captureEvalRun(properties) {
|
|
49
|
-
if (isFlagSet(process.env.SKYBRIDGE_TELEMETRY_DISABLED) ||
|
|
50
|
-
isFlagSet(process.env.DO_NOT_TRACK)) {
|
|
51
|
-
return;
|
|
52
|
-
}
|
|
53
|
-
const machineId = getMachineId();
|
|
54
|
-
if (machineId === undefined) {
|
|
55
|
-
return;
|
|
56
|
-
}
|
|
57
|
-
try {
|
|
58
|
-
const client = new PostHog(POSTHOG_API_KEY, {
|
|
59
|
-
host: POSTHOG_HOST,
|
|
60
|
-
flushAt: 1,
|
|
61
|
-
flushInterval: 0,
|
|
62
|
-
});
|
|
63
|
-
client.capture({
|
|
64
|
-
distinctId: machineId,
|
|
65
|
-
event: "evals run",
|
|
66
|
-
properties: { ...properties, isCI: ci.isCI },
|
|
67
|
-
});
|
|
68
|
-
await Promise.race([
|
|
69
|
-
client.shutdown(),
|
|
70
|
-
new Promise((resolve) => setTimeout(resolve, 2_000)),
|
|
71
|
-
]);
|
|
72
|
-
}
|
|
73
|
-
catch {
|
|
74
|
-
return;
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
//# sourceMappingURL=telemetry.js.map
|
package/dist/telemetry.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"telemetry.js","sourceRoot":"","sources":["../src/telemetry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,MAAM,eAAe,GAAG,iDAAiD,CAAC;AAC1E,MAAM,YAAY,GAAG,0BAA0B,CAAC;AAChD,MAAM,iBAAiB,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,YAAY,CAAC,CAAC;AACxD,MAAM,kBAAkB,GAAG,IAAI,CAAC,iBAAiB,EAAE,aAAa,CAAC,CAAC;AAOlE,SAAS,SAAS,CAAC,KAAyB;IAC1C,OAAO,KAAK,KAAK,GAAG,IAAI,KAAK,KAAK,MAAM,CAAC;AAC3C,CAAC;AAED,SAAS,gBAAgB;IACvB,IAAI,CAAC;QACH,IAAI,UAAU,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,YAAY;IACnB,MAAM,MAAM,GAAG,gBAAgB,EAAE,CAAC;IAClC,IAAI,MAAM,EAAE,SAAS,EAAE,OAAO,KAAK,KAAK,EAAE,CAAC;QACzC,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,MAAM,EAAE,SAAS,KAAK,SAAS,EAAE,CAAC;QACpC,OAAO,MAAM,CAAC,SAAS,CAAC;IAC1B,CAAC;IACD,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,UAAU,EAAE,CAAC;QAC/B,SAAS,CAAC,iBAAiB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAClD,aAAa,CACX,kBAAkB,EAClB,IAAI,CAAC,SAAS,CACZ,EAAE,GAAG,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EACtD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;QACF,OAAO,SAAS,CAAC;IACnB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,UAGpC;IACC,IACE,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC;QACnD,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,EACnC,CAAC;QACD,OAAO;IACT,CAAC;IACD,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;IACjC,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QAC5B,OAAO;IACT,CAAC;IAED,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,OAAO,CAAC,eAAe,EAAE;YAC1C,IAAI,EAAE,YAAY;YAClB,OAAO,EAAE,CAAC;YACV,aAAa,EAAE,CAAC;SACjB,CAAC,CAAC;QACH,MAAM,CAAC,OAAO,CAAC;YACb,UAAU,EAAE,SAAS;YACrB,KAAK,EAAE,WAAW;YAClB,UAAU,EAAE,EAAE,GAAG,UAAU,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE;SAC7C,CAAC,CAAC;QACH,MAAM,OAAO,CAAC,IAAI,CAAC;YACjB,MAAM,CAAC,QAAQ,EAAE;YACjB,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;SACrD,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;IACT,CAAC;AACH,CAAC"}
|