@skybridge/test 0.0.0-dev.50c10e1 → 0.0.0-dev.874c2eb

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -25,5 +25,58 @@ it("finds products by category", async () => {
25
25
  });
26
26
  ```
27
27
 
28
+ ## Choosing a model
29
+
30
+ `model` is a `provider/model-id` string. Only the provider you name is
31
+ imported, so install just that one as a dev dependency, and the key is read
32
+ from its own environment variable.
33
+
34
+ | prefix | package | key |
35
+ | --- | --- | --- |
36
+ | `anthropic` | `@ai-sdk/anthropic` | `ANTHROPIC_API_KEY` |
37
+ | `openai` | `@ai-sdk/openai` | `OPENAI_API_KEY` |
38
+ | `mistral` | `@ai-sdk/mistral` | `MISTRAL_API_KEY` |
39
+ | `google` | `@ai-sdk/google` | `GOOGLE_GENERATIVE_AI_API_KEY` |
40
+
41
+ ```ts
42
+ skybridge({
43
+ evals: {
44
+ model: "anthropic/claude-sonnet-4-5",
45
+ project: { cwd: ".", command: ["node", "dist/server.js"] },
46
+ },
47
+ });
48
+ ```
49
+
50
+ `command` has to start the server process directly, since wrappers such as
51
+ `pnpm run` do not forward the channel the runner uses to learn the port.
52
+
53
+ Any other prefix is passed to the AI SDK untouched, so a string such as
54
+ `openai/gpt-5` also resolves through the Vercel AI Gateway if you have
55
+ `AI_GATEWAY_API_KEY` set and no provider package installed.
56
+
57
+ ## Models the prefixes cannot reach
58
+
59
+ For a local runtime, a company gateway, or a model that needs middleware,
60
+ build the model yourself and register it from a vitest setup file. It takes
61
+ precedence over `model`, which can then be left out.
62
+
63
+ ```ts
64
+ // eval-model.ts
65
+ import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
66
+ import { defineEvalModel } from "@skybridge/test";
67
+
68
+ defineEvalModel(
69
+ createOpenAICompatible({ baseURL: "http://localhost:11434/v1" })("llama3.1"),
70
+ );
71
+ ```
72
+
73
+ ```ts
74
+ // vitest.config.ts
75
+ export default defineConfig({
76
+ plugins: [skybridge({ evals: { project: { ... } } })],
77
+ test: { setupFiles: ["./eval-model.ts"] },
78
+ });
79
+ ```
80
+
28
81
  See the [documentation](https://docs.skybridge.tech) for setup and the full
29
82
  matcher list.
package/dist/chat.d.ts CHANGED
@@ -8,15 +8,10 @@ export type ToolCall<App> = {
8
8
  [Name in ToolNames<App>]: {
9
9
  name: Name;
10
10
  arguments: ToolInput<App, Name>;
11
- invalid?: boolean;
11
+ failed?: string;
12
12
  };
13
13
  }[ToolNames<App>];
14
- /**
15
- * How the host is configured. `model` is an AI SDK model instance, so the
16
- * provider and its credentials are chosen by the project rather than by a
17
- * base URL the runner has to know about.
18
- */
19
- export interface HostConfig {
14
+ interface HostConfig {
20
15
  model: LanguageModel;
21
16
  temperature?: number;
22
17
  systemPrompt?: string;
@@ -35,6 +30,7 @@ export declare class Chat<App = unknown> {
35
30
  /** Every call the model made, in order, as it crossed the wire. */
36
31
  readonly toolCalls: ToolCall<App>[];
37
32
  private readonly messages;
33
+ private readonly failures;
38
34
  private readonly client;
39
35
  private readonly transport;
40
36
  private readonly host;
@@ -46,6 +42,7 @@ export declare class Chat<App = unknown> {
46
42
  /** Takes a turn and returns the model's answer. */
47
43
  send(prompt: string): Promise<string>;
48
44
  close(): Promise<void>;
45
+ private failureFor;
49
46
  private toolSet;
50
47
  }
51
48
  export {};
@@ -1 +1 @@
1
- {"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAEA,OAAO,EAIL,KAAK,aAAa,EAGnB,MAAM,IAAI,CAAC;AACZ,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7D;;;GAGG;AACH,MAAM,MAAM,QAAQ,CAAC,GAAG,IAAI;KACzB,IAAI,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG;QACxB,IAAI,EAAE,IAAI,CAAC;QACX,SAAS,EAAE,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAChC,OAAO,CAAC,EAAE,OAAO,CAAC;KACnB;CACF,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;AAElB;;;;GAIG;AACH,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,aAAa,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,UAAU,iBAAiB;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACtC;AAID;;;GAGG;AACH,qBAAa,IAAI,CAAC,GAAG,GAAG,OAAO;IAC7B,mEAAmE;IACnE,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAM;IAEzC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAa;IAClC,OAAO,CAAC,KAAK,CAA2B;IAExC,OAAO;WAUM,IAAI,CAAC,GAAG,EACnB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,UAAU,GACf,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAerB,0EAA0E;IAC1E,IAAI,eAAe,IAAI,iBAAiB,EAAE,CAEzC;IAED,mDAAmD;IAC7C,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IA4BrC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,OAAO,CAAC,OAAO;CAoBhB"}
1
+ {"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAEA,OAAO,EAIL,KAAK,aAAa,EAGnB,MAAM,IAAI,CAAC;AACZ,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7D;;;GAGG;AACH,MAAM,MAAM,QAAQ,CAAC,GAAG,IAAI;KACzB,IAAI,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG;QACxB,IAAI,EAAE,IAAI,CAAC;QACX,SAAS,EAAE,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAChC,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB;CACF,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;AAElB,UAAU,UAAU;IAClB,KAAK,EAAE,aAAa,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,UAAU,iBAAiB;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACtC;AAID;;;GAGG;AACH,qBAAa,IAAI,CAAC,GAAG,GAAG,OAAO;IAC7B,mEAAmE;IACnE,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAM;IAEzC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA6B;IACtD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAa;IAClC,OAAO,CAAC,KAAK,CAA2B;IAExC,OAAO;WAUM,IAAI,CAAC,GAAG,EACnB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,UAAU,GACf,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAerB,0EAA0E;IAC1E,IAAI,eAAe,IAAI,iBAAiB,EAAE,CAEzC;IAED,mDAAmD;IAC7C,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IA4BrC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,OAAO,CAAC,UAAU;IAalB,OAAO,CAAC,OAAO;CAmChB"}
package/dist/chat.js CHANGED
@@ -10,6 +10,7 @@ export class Chat {
10
10
  /** Every call the model made, in order, as it crossed the wire. */
11
11
  toolCalls = [];
12
12
  messages = [];
13
+ failures = new Map();
13
14
  client;
14
15
  transport;
15
16
  host;
@@ -56,7 +57,7 @@ export class Chat {
56
57
  this.toolCalls.push({
57
58
  name: call.toolName,
58
59
  arguments: (call.input ?? {}),
59
- ...(call.invalid === true ? { invalid: true } : {}),
60
+ ...this.failureFor(call),
60
61
  });
61
62
  }
62
63
  }
@@ -66,6 +67,16 @@ export class Chat {
66
67
  async close() {
67
68
  await this.transport.close();
68
69
  }
70
+ failureFor(call) {
71
+ const reported = this.failures.get(call.toolCallId);
72
+ if (reported !== undefined) {
73
+ return { failed: reported };
74
+ }
75
+ if (call.invalid === true) {
76
+ return { failed: "the model's arguments could not be parsed" };
77
+ }
78
+ return {};
79
+ }
69
80
  toolSet() {
70
81
  const entries = this.tools.map((definition) => {
71
82
  const { $schema: _ignored, ...parameters } = definition.inputSchema;
@@ -74,11 +85,21 @@ export class Chat {
74
85
  dynamicTool({
75
86
  description: definition.description ?? "",
76
87
  inputSchema: jsonSchema(parameters),
77
- execute: async (input) => {
78
- const result = await this.client.callTool({
79
- name: definition.name,
80
- arguments: (input ?? {}),
81
- });
88
+ execute: async (input, { toolCallId }) => {
89
+ let result;
90
+ try {
91
+ result = await this.client.callTool({
92
+ name: definition.name,
93
+ arguments: (input ?? {}),
94
+ });
95
+ }
96
+ catch (error) {
97
+ this.failures.set(toolCallId, `the call never reached the server: ${error instanceof Error ? error.message : String(error)}`);
98
+ throw error;
99
+ }
100
+ if (result.isError === true) {
101
+ this.failures.set(toolCallId, `the tool returned an error: ${JSON.stringify(result.content)}`);
102
+ }
82
103
  return result.content ?? result;
83
104
  },
84
105
  }),
package/dist/chat.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,6BAA6B,EAAE,MAAM,oDAAoD,CAAC;AACnG,OAAO,EACL,WAAW,EACX,YAAY,EACZ,UAAU,EAGV,WAAW,GACZ,MAAM,IAAI,CAAC;AAiCZ,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B;;;GAGG;AACH,MAAM,OAAO,IAAI;IACf,mEAAmE;IAC1D,SAAS,GAAoB,EAAE,CAAC;IAExB,QAAQ,GAAmB,EAAE,CAAC;IAC9B,MAAM,CAAS;IACf,SAAS,CAAgC;IACzC,IAAI,CAAa;IAC1B,KAAK,GAAwB,EAAE,CAAC;IAExC,YACE,MAAc,EACd,SAAwC,EACxC,IAAgB;QAEhB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,IAAI,CACf,SAAiB,EACjB,IAAgB;QAEhB,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,IAAI,6BAA6B,CAAC,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;QACxE,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAM,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QACpD,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,GAAG,KAAuC,CAAC;QACvD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YACxB,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,0EAA0E;IAC1E,IAAI,eAAe;QACjB,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED,mDAAmD;IACnD,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAEtD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;YACtB,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC;YACvC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,KAAK,SAAS;gBACtC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE;YACrB,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;SAC/D,CAAC,CAAC;QAEH,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAChC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAClC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;oBAClB,IAAI,EAAE,IAAI,CAAC,QAAQ;oBACnB,SAAS,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAA4B;oBACxD,GAAG,CAAC,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACnC,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAChD,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;IAEO,OAAO;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;YAC5C,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,UAAU,EAAE,GAAG,UAAU,CAAC,WAAW,CAAC;YACpE,OAAO;gBACL,UAAU,CAAC,IAAI;gBACf,WAAW,CAAC;oBACV,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,EAAE;oBACzC,WAAW,EAAE,UAAU,CAAC,UAAU,CAAC;oBACnC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;wBACvB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;4BACxC,IAAI,EAAE,UAAU,CAAC,IAAI;4BACrB,SAAS,EAAE,CAAC,KAAK,IAAI,EAAE,CAA4B;yBACpD,CAAC,CAAC;wBACH,OAAO,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC;oBAClC,CAAC;iBACF,CAAC;aACM,CAAC;QACb,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;CACF"}
1
+ {"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,6BAA6B,EAAE,MAAM,oDAAoD,CAAC;AACnG,OAAO,EACL,WAAW,EACX,YAAY,EACZ,UAAU,EAGV,WAAW,GACZ,MAAM,IAAI,CAAC;AA4BZ,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B;;;GAGG;AACH,MAAM,OAAO,IAAI;IACf,mEAAmE;IAC1D,SAAS,GAAoB,EAAE,CAAC;IAExB,QAAQ,GAAmB,EAAE,CAAC;IAC9B,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrC,MAAM,CAAS;IACf,SAAS,CAAgC;IACzC,IAAI,CAAa;IAC1B,KAAK,GAAwB,EAAE,CAAC;IAExC,YACE,MAAc,EACd,SAAwC,EACxC,IAAgB;QAEhB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,IAAI,CACf,SAAiB,EACjB,IAAgB;QAEhB,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,IAAI,6BAA6B,CAAC,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;QACxE,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAM,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QACpD,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACrB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YACxB,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,0EAA0E;IAC1E,IAAI,eAAe;QACjB,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED,mDAAmD;IACnD,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAEtD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;YACtB,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC;YACvC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,KAAK,SAAS;gBACtC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE;YACrB,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;SAC/D,CAAC,CAAC;QAEH,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAChC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAClC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;oBAClB,IAAI,EAAE,IAAI,CAAC,QAAQ;oBACnB,SAAS,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAA4B;oBACxD,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;iBACR,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAChD,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;IAEO,UAAU,CAAC,IAA+C;QAGhE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QAC9B,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC1B,OAAO,EAAE,MAAM,EAAE,2CAA2C,EAAE,CAAC;QACjE,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAEO,OAAO;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;YAC5C,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,UAAU,EAAE,GAAG,UAAU,CAAC,WAAW,CAAC;YACpE,OAAO;gBACL,UAAU,CAAC,IAAI;gBACf,WAAW,CAAC;oBACV,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,EAAE;oBACzC,WAAW,EAAE,UAAU,CAAC,UAAU,CAAC;oBACnC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE;wBACvC,IAAI,MAA+C,CAAC;wBACpD,IAAI,CAAC;4BACH,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;gCAClC,IAAI,EAAE,UAAU,CAAC,IAAI;gCACrB,SAAS,EAAE,CAAC,KAAK,IAAI,EAAE,CAA4B;6BACpD,CAAC,CAAC;wBACL,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BACf,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,sCAAsC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC/F,CAAC;4BACF,MAAM,KAAK,CAAC;wBACd,CAAC;wBACD,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;4BAC5B,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,+BAA+B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAChE,CAAC;wBACJ,CAAC;wBACD,OAAO,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC;oBAClC,CAAC;iBACF,CAAC;aACM,CAAC;QACb,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;CACF"}
package/dist/config.d.ts CHANGED
@@ -1,12 +1,8 @@
1
- import type { EvalsOptions, ModelDescriptor } from "@skybridge/vite-plugin";
2
- export type { EvalsOptions, ModelDescriptor, ModelFile, } from "@skybridge/vite-plugin";
3
- /** The serializable half, the only part that crosses into the workers. */
4
- export type ProvidedEvalsOptions = Omit<EvalsOptions, "model"> & {
5
- model?: ModelDescriptor;
6
- };
1
+ import type { EvalsOptions } from "@skybridge/vite-plugin";
2
+ export type { EvalsOptions } from "@skybridge/vite-plugin";
7
3
  declare module "vitest" {
8
4
  interface ProvidedContext {
9
- skybridgeEvals: ProvidedEvalsOptions;
5
+ skybridgeEvals: EvalsOptions;
10
6
  skybridgeEvalsUrl: string;
11
7
  }
12
8
  }
@@ -1 +1 @@
1
- {"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAE5E,YAAY,EACV,YAAY,EACZ,eAAe,EACf,SAAS,GACV,MAAM,wBAAwB,CAAC;AAEhC,0EAA0E;AAC1E,MAAM,MAAM,oBAAoB,GAAG,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,GAAG;IAC/D,KAAK,CAAC,EAAE,eAAe,CAAC;CACzB,CAAC;AAEF,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,eAAe;QACvB,cAAc,EAAE,oBAAoB,CAAC;QACrC,iBAAiB,EAAE,MAAM,CAAC;KAC3B;CACF"}
1
+ {"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAE3D,YAAY,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAE3D,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,eAAe;QACvB,cAAc,EAAE,YAAY,CAAC;QAC7B,iBAAiB,EAAE,MAAM,CAAC;KAC3B;CACF"}
@@ -1 +1 @@
1
- {"version":3,"file":"global-setup.d.ts","sourceRoot":"","sources":["../src/global-setup.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAI/C,wBAA8B,KAAK,CAAC,OAAO,EAAE,WAAW,8CAsBvD"}
1
+ {"version":3,"file":"global-setup.d.ts","sourceRoot":"","sources":["../src/global-setup.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAG/C,wBAA8B,KAAK,CAAC,OAAO,EAAE,WAAW,8CA0BvD"}
@@ -1,7 +1,9 @@
1
1
  import { startServer } from "./server-harness.js";
2
2
  export default async function setup(project) {
3
- const options = project.config.provide
4
- ?.skybridgeEvals;
3
+ const options = project.config.provide?.skybridgeEvals;
4
+ if (options === undefined) {
5
+ throw new Error("The evals plugin is not registered. Add `skybridge({ evals: {...} })` to the vitest config.");
6
+ }
5
7
  if (options.server !== undefined) {
6
8
  project.provide("skybridgeEvalsUrl", options.server);
7
9
  return;
@@ -1 +1 @@
1
- {"version":3,"file":"global-setup.js","sourceRoot":"","sources":["../src/global-setup.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAElD,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,KAAK,CAAC,OAAoB;IACtD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO;QACpC,EAAE,cAAsC,CAAC;IAE3C,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QACjC,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACrD,OAAO;IACT,CAAC;IACD,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAClD,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;IAEjD,OAAO,KAAK,IAAI,EAAE;QAChB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;YAClB,OAAO,CAAC,GAAG,CAAC,4BAA4B,MAAM,OAAO,CAAC,CAAC;QACzD,CAAC;IACH,CAAC,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"global-setup.js","sourceRoot":"","sources":["../src/global-setup.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAElD,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,KAAK,CAAC,OAAoB;IACtD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,cAAc,CAAC;IACvD,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,6FAA6F,CAC9F,CAAC;IACJ,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QACjC,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACrD,OAAO;IACT,CAAC;IACD,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAClD,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;IAEjD,OAAO,KAAK,IAAI,EAAE;QAChB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;YAClB,OAAO,CAAC,GAAG,CAAC,4BAA4B,MAAM,OAAO,CAAC,CAAC;QACzD,CAAC;IACH,CAAC,CAAC;AACJ,CAAC"}
package/dist/index.d.ts CHANGED
@@ -1,5 +1,4 @@
1
1
  export type { Chat } from "./chat.js";
2
- export type { EvalsOptions, ModelDescriptor } from "./config.js";
3
2
  export type { ChatAssertion, ChatMatchers } from "./matchers.js";
4
3
  export { defineEvalModel } from "./model.js";
5
4
  export { start } from "./session-registry.js";
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACtC,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AACjE,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AACjE,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACtC,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AACjE,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
package/dist/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
package/dist/matchers.js CHANGED
@@ -3,14 +3,14 @@ function attemptedCalls(chat, name) {
3
3
  return chat.toolCalls.filter((call) => String(call.name) === name);
4
4
  }
5
5
  function acceptedCalls(chat, name) {
6
- return attemptedCalls(chat, name).filter((call) => call.invalid !== true);
6
+ return attemptedCalls(chat, name).filter((call) => call.failed === undefined);
7
7
  }
8
8
  function observed(chat) {
9
9
  if (chat.toolCalls.length === 0) {
10
10
  return " (no tool was called)";
11
11
  }
12
12
  return chat.toolCalls
13
- .map((call, index) => ` ${index + 1}. ${String(call.name)} ${JSON.stringify(call.arguments)}${call.invalid === true ? " (arguments rejected by the tool schema)" : ""}`)
13
+ .map((call, index) => ` ${index + 1}. ${call.name} ${JSON.stringify(call.arguments)}${call.failed === undefined ? "" : ` (failed: ${call.failed})`}`)
14
14
  .join("\n");
15
15
  }
16
16
  function definitions(chat) {
@@ -40,7 +40,7 @@ expect.extend({
40
40
  ? `Expected "${name}" not to be called exactly once.`
41
41
  : matching.length === 1
42
42
  ? `Expected "${name}" to be called once with ${JSON.stringify(args)}, but it was called with ${JSON.stringify(matching[0]?.arguments)}.`
43
- : `Expected "${name}" to be called exactly once by a call the tool schema accepted, but ${matching.length} such calls were made.`),
43
+ : `Expected "${name}" to be called exactly once by a call that succeeded, but ${matching.length} such calls were made.`),
44
44
  };
45
45
  },
46
46
  toHaveCalledToolWith(received, name, args) {
@@ -62,5 +62,5 @@ expect.extend({
62
62
  };
63
63
  },
64
64
  });
65
- expect.chat = ((chat) => expect(chat));
65
+ expect.chat = (chat) => expect(chat);
66
66
  //# sourceMappingURL=matchers.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"matchers.js","sourceRoot":"","sources":["../src/matchers.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AA8BhC,SAAS,cAAc,CACrB,IAAmB,EACnB,IAAY;IAEZ,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;AACrE,CAAC;AAED,SAAS,aAAa,CAAC,IAAmB,EAAE,IAAY;IACtD,OAAO,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC;AAC5E,CAAC;AAED,SAAS,QAAQ,CAAC,IAAmB;IACnC,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,wBAAwB,CAAC;IAClC,CAAC;IACD,OAAO,IAAI,CAAC,SAAS;SAClB,GAAG,CACF,CAAC,IAAuB,EAAE,KAAa,EAAE,EAAE,CACzC,KAAK,KAAK,GAAG,CAAC,KAAK,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,2CAA2C,CAAC,CAAC,CAAC,EAAE,EAAE,CACtJ;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,WAAW,CAAC,IAAmB;IACtC,OAAO,IAAI,CAAC,eAAe;SACxB,GAAG,CACF,CAAC,IAAI,EAAE,EAAE,CACP,OAAO,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,WAAW,IAAI,EAAE,oBAAoB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CACpG;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,MAAM,CAAC,IAAmB,EAAE,OAAe;IAClD,OAAO,GAAG,OAAO;;;EAGjB,QAAQ,CAAC,IAAI,CAAC;;;EAGd,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;AACtB,CAAC;AAED,MAAM,CAAC,MAAM,CAAC;IACZ,oBAAoB,CAClB,QAAuB,EACvB,IAAY,EACZ,IAA8B;QAE9B,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC/C,MAAM,SAAS,GACb,IAAI,KAAK,SAAS;YAClB,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACpB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxE,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC;QAEhD,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,aAAa,IAAI,kCAAkC;gBACrD,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;oBACrB,CAAC,CAAC,aAAa,IAAI,4BAA4B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,4BAA4B,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG;oBACxI,CAAC,CAAC,aAAa,IAAI,uEAAuE,QAAQ,CAAC,MAAM,wBAAwB,CACtI;SACJ,CAAC;IACJ,CAAC;IAED,oBAAoB,CAClB,QAAuB,EACvB,IAAY,EACZ,IAA6B;QAE7B,MAAM,IAAI,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CACvD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAC3D,CAAC;QAEF,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,wBAAwB,IAAI,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG;gBAC/D,CAAC,CAAC,aAAa,IAAI,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CACpE;SACJ,CAAC;IACJ,CAAC;IAED,qBAAqB,CAAC,QAAuB,EAAE,IAAY;QACzD,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAEhD,OAAO;YACL,IAAI,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC;YAC3B,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACnB,CAAC,CAAC,aAAa,IAAI,+BAA+B;gBAClD,CAAC,CAAC,aAAa,IAAI,2CAA2C,QAAQ,CAAC,MAAM,SAAS,CACzF;SACJ,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,IAAI,GAAG,CAAC,CAAM,IAAe,EAAE,EAAE,CACtC,MAAM,CAAC,IAAI,CAAC,CAAkC,CAAC"}
1
+ {"version":3,"file":"matchers.js","sourceRoot":"","sources":["../src/matchers.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AA8BhC,SAAS,cAAc,CACrB,IAAmB,EACnB,IAAY;IAEZ,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;AACrE,CAAC;AAED,SAAS,aAAa,CAAC,IAAmB,EAAE,IAAY;IACtD,OAAO,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,QAAQ,CAAC,IAAmB;IACnC,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,wBAAwB,CAAC;IAClC,CAAC;IACD,OAAO,IAAI,CAAC,SAAS;SAClB,GAAG,CACF,CAAC,IAAuB,EAAE,KAAa,EAAE,EAAE,CACzC,KAAK,KAAK,GAAG,CAAC,KAAK,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,cAAc,IAAI,CAAC,MAAM,GAAG,EAAE,CACnI;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,WAAW,CAAC,IAAmB;IACtC,OAAO,IAAI,CAAC,eAAe;SACxB,GAAG,CACF,CAAC,IAAI,EAAE,EAAE,CACP,OAAO,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,WAAW,IAAI,EAAE,oBAAoB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CACpG;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,MAAM,CAAC,IAAmB,EAAE,OAAe;IAClD,OAAO,GAAG,OAAO;;;EAGjB,QAAQ,CAAC,IAAI,CAAC;;;EAGd,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;AACtB,CAAC;AAED,MAAM,CAAC,MAAM,CAAC;IACZ,oBAAoB,CAClB,QAAuB,EACvB,IAAY,EACZ,IAA8B;QAE9B,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC/C,MAAM,SAAS,GACb,IAAI,KAAK,SAAS;YAClB,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACpB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxE,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC;QAEhD,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,aAAa,IAAI,kCAAkC;gBACrD,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;oBACrB,CAAC,CAAC,aAAa,IAAI,4BAA4B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,4BAA4B,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG;oBACxI,CAAC,CAAC,aAAa,IAAI,6DAA6D,QAAQ,CAAC,MAAM,wBAAwB,CAC5H;SACJ,CAAC;IACJ,CAAC;IAED,oBAAoB,CAClB,QAAuB,EACvB,IAAY,EACZ,IAA6B;QAE7B,MAAM,IAAI,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CACvD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAC3D,CAAC;QAEF,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,wBAAwB,IAAI,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG;gBAC/D,CAAC,CAAC,aAAa,IAAI,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CACpE;SACJ,CAAC;IACJ,CAAC;IAED,qBAAqB,CAAC,QAAuB,EAAE,IAAY;QACzD,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAEhD,OAAO;YACL,IAAI,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC;YAC3B,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACnB,CAAC,CAAC,aAAa,IAAI,+BAA+B;gBAClD,CAAC,CAAC,aAAa,IAAI,2CAA2C,QAAQ,CAAC,MAAM,SAAS,CACzF;SACJ,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,IAAI,GAAG,CAAM,IAAe,EAAsB,EAAE,CACzD,MAAM,CAAC,IAAI,CAAkC,CAAC"}
package/dist/model.d.ts CHANGED
@@ -1,9 +1,15 @@
1
1
  import type { LanguageModel } from "ai";
2
- import type { ModelDescriptor } from "./config.js";
3
2
  /**
4
- * Registers a model instance from a setup file. The escape hatch for providers
5
- * the descriptor cannot construct, such as a company LLM gateway.
3
+ * Registers a model instance from a setup file, taking precedence over the
4
+ * configured `model` string. The escape hatch for anything the provider
5
+ * prefixes cannot express, such as a local runtime or a company LLM gateway.
6
6
  */
7
7
  export declare function defineEvalModel(model: LanguageModel): void;
8
- export declare function resolveModel(descriptor: ModelDescriptor | undefined): Promise<LanguageModel>;
8
+ /**
9
+ * Turns a `provider/model-id` string into a model instance, importing only the
10
+ * provider named. Each provider package reads its own conventional API key
11
+ * variable, so no key configuration is needed. An unrecognised prefix is
12
+ * returned untouched for the AI SDK's default provider to resolve.
13
+ */
14
+ export declare function resolveModel(spec: string | undefined): Promise<LanguageModel>;
9
15
  //# sourceMappingURL=model.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"model.d.ts","sourceRoot":"","sources":["../src/model.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACxC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAInD;;;GAGG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAE1D;AAaD,wBAAsB,YAAY,CAChC,UAAU,EAAE,eAAe,GAAG,SAAS,GACtC,OAAO,CAAC,aAAa,CAAC,CAWxB"}
1
+ {"version":3,"file":"model.d.ts","sourceRoot":"","sources":["../src/model.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAaxC;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAE1D;AAED;;;;;GAKG;AACH,wBAAsB,YAAY,CAChC,IAAI,EAAE,MAAM,GAAG,SAAS,GACvB,OAAO,CAAC,aAAa,CAAC,CAwBxB"}
package/dist/model.js CHANGED
@@ -1,28 +1,43 @@
1
+ const PROVIDERS = {
2
+ anthropic: () => import("@ai-sdk/anthropic").then((m) => m.anthropic),
3
+ google: () => import("@ai-sdk/google").then((m) => m.google),
4
+ mistral: () => import("@ai-sdk/mistral").then((m) => m.mistral),
5
+ openai: () => import("@ai-sdk/openai").then((m) => m.openai),
6
+ };
1
7
  let registered;
2
8
  /**
3
- * Registers a model instance from a setup file. The escape hatch for providers
4
- * the descriptor cannot construct, such as a company LLM gateway.
9
+ * Registers a model instance from a setup file, taking precedence over the
10
+ * configured `model` string. The escape hatch for anything the provider
11
+ * prefixes cannot express, such as a local runtime or a company LLM gateway.
5
12
  */
6
13
  export function defineEvalModel(model) {
7
14
  registered = model;
8
15
  }
9
- const DEFAULT_KEY_ENV = { anthropic: "ANTHROPIC_API_KEY" };
10
- function requireKey(descriptor) {
11
- const name = descriptor.apiKeyEnv ?? DEFAULT_KEY_ENV[descriptor.provider];
12
- const key = process.env[name];
13
- if (key === undefined || key === "") {
14
- throw new Error(`${name} is not set`);
15
- }
16
- return key;
17
- }
18
- export async function resolveModel(descriptor) {
16
+ /**
17
+ * Turns a `provider/model-id` string into a model instance, importing only the
18
+ * provider named. Each provider package reads its own conventional API key
19
+ * variable, so no key configuration is needed. An unrecognised prefix is
20
+ * returned untouched for the AI SDK's default provider to resolve.
21
+ */
22
+ export async function resolveModel(spec) {
19
23
  if (registered !== undefined) {
20
24
  return registered;
21
25
  }
22
- if (descriptor === undefined) {
23
- throw new Error("No model configured. Pass `model` to the evals plugin, or call defineEvalModel() from a setup file.");
26
+ if (spec === undefined) {
27
+ throw new Error("No model configured. Set `model` on the evals plugin, or call defineEvalModel() from a setup file.");
28
+ }
29
+ const separator = spec.indexOf("/");
30
+ if (separator === -1) {
31
+ return spec;
32
+ }
33
+ const name = spec.slice(0, separator);
34
+ const load = PROVIDERS[name];
35
+ if (load === undefined) {
36
+ return spec;
24
37
  }
25
- const { createAnthropic } = await import("@ai-sdk/anthropic");
26
- return createAnthropic({ apiKey: requireKey(descriptor) })(descriptor.name);
38
+ const provider = await load().catch(() => {
39
+ throw new Error(`The model "${spec}" needs @ai-sdk/${name}@^2. Install it as a dev dependency.`);
40
+ });
41
+ return provider(spec.slice(separator + 1));
27
42
  }
28
43
  //# sourceMappingURL=model.js.map
package/dist/model.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"model.js","sourceRoot":"","sources":["../src/model.ts"],"names":[],"mappings":"AAGA,IAAI,UAAqC,CAAC;AAE1C;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,KAAoB;IAClD,UAAU,GAAG,KAAK,CAAC;AACrB,CAAC;AAED,MAAM,eAAe,GAAG,EAAE,SAAS,EAAE,mBAAmB,EAAW,CAAC;AAEpE,SAAS,UAAU,CAAC,UAA2B;IAC7C,MAAM,IAAI,GAAG,UAAU,CAAC,SAAS,IAAI,eAAe,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IAC1E,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC9B,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,EAAE,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,aAAa,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,UAAuC;IAEvC,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CACb,qGAAqG,CACtG,CAAC;IACJ,CAAC;IACD,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;IAC9D,OAAO,eAAe,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;AAC9E,CAAC"}
1
+ {"version":3,"file":"model.js","sourceRoot":"","sources":["../src/model.ts"],"names":[],"mappings":"AAIA,MAAM,SAAS,GAAmD;IAChE,SAAS,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IACrE,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IAC5D,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;IAC/D,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;CAC7D,CAAC;AAEF,IAAI,UAAqC,CAAC;AAE1C;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,KAAoB;IAClD,UAAU,GAAG,KAAK,CAAC;AACrB,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAAwB;IAExB,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,oGAAoG,CACrG,CAAC;IACJ,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACpC,IAAI,SAAS,KAAK,CAAC,CAAC,EAAE,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC7B,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE;QACvC,MAAM,IAAI,KAAK,CACb,cAAc,IAAI,mBAAmB,IAAI,sCAAsC,CAChF,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC;AAC7C,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"plugin.d.ts","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,KAAK,EAAE,YAAY,EAAwB,MAAM,aAAa,CAAC;AAMtE;;;;;GAKG;AACH,wBAAgB,KAAK,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM,CAuBnD"}
1
+ {"version":3,"file":"plugin.d.ts","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAMhD;;;;;GAKG;AACH,wBAAgB,KAAK,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM,CAcnD"}
package/dist/plugin.js CHANGED
@@ -9,21 +9,15 @@ function here(file) {
9
9
  * nothing but `start` and its own `AppType`.
10
10
  */
11
11
  export function evals(options) {
12
- const { model, ...rest } = options;
13
- const provided = "file" in model ? rest : { ...rest, model };
14
- const setupFiles = [here("./setup.js")];
15
- if ("file" in model) {
16
- setupFiles.push(model.file);
17
- }
18
12
  return {
19
13
  name: "skybridge-evals",
20
14
  config() {
21
15
  return {
22
16
  test: {
23
17
  testTimeout: 120_000,
24
- setupFiles,
18
+ setupFiles: [here("./matchers.js")],
25
19
  globalSetup: [here("./global-setup.js")],
26
- provide: { skybridgeEvals: provided },
20
+ provide: { skybridgeEvals: options },
27
21
  },
28
22
  };
29
23
  },
@@ -1 +1 @@
1
- {"version":3,"file":"plugin.js","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAIzC,SAAS,IAAI,CAAC,IAAY;IACxB,OAAO,aAAa,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AACvD,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,KAAK,CAAC,OAAqB;IACzC,MAAM,EAAE,KAAK,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;IACnC,MAAM,QAAQ,GACZ,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,CAAC;IAE9C,MAAM,UAAU,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;IACxC,IAAI,MAAM,IAAI,KAAK,EAAE,CAAC;QACpB,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAED,OAAO;QACL,IAAI,EAAE,iBAAiB;QACvB,MAAM;YACJ,OAAO;gBACL,IAAI,EAAE;oBACJ,WAAW,EAAE,OAAO;oBACpB,UAAU;oBACV,WAAW,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;oBACxC,OAAO,EAAE,EAAE,cAAc,EAAE,QAAQ,EAAE;iBACtC;aACF,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"plugin.js","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAIzC,SAAS,IAAI,CAAC,IAAY;IACxB,OAAO,aAAa,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AACvD,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,KAAK,CAAC,OAAqB;IACzC,OAAO;QACL,IAAI,EAAE,iBAAiB;QACvB,MAAM;YACJ,OAAO;gBACL,IAAI,EAAE;oBACJ,WAAW,EAAE,OAAO;oBACpB,UAAU,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;oBACnC,WAAW,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;oBACxC,OAAO,EAAE,EAAE,cAAc,EAAE,OAAO,EAAE;iBACrC;aACF,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC"}
@@ -10,10 +10,9 @@ export interface RunningServer {
10
10
  * Binding and reporting both happen in the child, so there is no window in
11
11
  * which another process can steal the port.
12
12
  */
13
- export declare function startServer({ cwd, command, env, timeoutMs, }: {
13
+ export declare function startServer({ cwd, command, env, }: {
14
14
  cwd: string;
15
15
  command: string[];
16
16
  env?: Record<string, string>;
17
- timeoutMs?: number;
18
17
  }): Promise<RunningServer>;
19
18
  //# sourceMappingURL=server-harness.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"server-harness.d.ts","sourceRoot":"","sources":["../src/server-harness.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,aAAa;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,yEAAyE;IACzE,MAAM,EAAE,MAAM,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAiBD;;;;;GAKG;AACH,wBAAsB,WAAW,CAAC,EAChC,GAAG,EACH,OAAO,EACP,GAAQ,EACR,SAAkB,GACnB,EAAE;IACD,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,OAAO,CAAC,aAAa,CAAC,CA2EzB"}
1
+ {"version":3,"file":"server-harness.d.ts","sourceRoot":"","sources":["../src/server-harness.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,aAAa;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,yEAAyE;IACzE,MAAM,EAAE,MAAM,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAiBD;;;;;GAKG;AACH,wBAAsB,WAAW,CAAC,EAChC,GAAG,EACH,OAAO,EACP,GAAQ,GACT,EAAE;IACD,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC9B,GAAG,OAAO,CAAC,aAAa,CAAC,CA4EzB"}
@@ -15,7 +15,8 @@ function isListeningMessage(message) {
15
15
  * Binding and reporting both happen in the child, so there is no window in
16
16
  * which another process can steal the port.
17
17
  */
18
- export async function startServer({ cwd, command, env = {}, timeoutMs = 60_000, }) {
18
+ export async function startServer({ cwd, command, env = {}, }) {
19
+ const timeoutMs = 60_000;
19
20
  const [bin, ...args] = command;
20
21
  if (bin === undefined) {
21
22
  throw new Error("Empty server command");
@@ -53,7 +54,7 @@ export async function startServer({ cwd, command, env = {}, timeoutMs = 60_000,
53
54
  };
54
55
  const port = await new Promise((resolvePort, reject) => {
55
56
  const timer = setTimeout(() => {
56
- reject(new Error(`Server did not report its port in ${timeoutMs}ms. It must call \`server.run()\` on a skybridge version that sends the skybridge:listening readiness message.\n${output()}`));
57
+ reject(new Error(`Server did not report a port in ${timeoutMs}ms. \`command\` must start the server process directly (node, tsx): wrappers such as \`pnpm run\` do not forward the IPC channel the readiness signal uses. It must also run a skybridge version that sends skybridge:listening.\n${output()}`));
57
58
  }, timeoutMs);
58
59
  child.on("message", (message) => {
59
60
  if (isListeningMessage(message)) {
@@ -1 +1 @@
1
- {"version":3,"file":"server-harness.js","sourceRoot":"","sources":["../src/server-harness.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqB,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AASpC,SAAS,SAAS,CAAC,KAAmB;IACpC,OAAO,KAAK,CAAC,QAAQ,KAAK,IAAI,IAAI,KAAK,CAAC,UAAU,KAAK,IAAI,CAAC;AAC9D,CAAC;AAED,SAAS,kBAAkB,CACzB,OAAgB;IAEhB,OAAO,CACL,OAAO,OAAO,KAAK,QAAQ;QAC3B,OAAO,KAAK,IAAI;QACf,OAA8B,CAAC,IAAI,KAAK,qBAAqB;QAC9D,OAAQ,OAA8B,CAAC,IAAI,KAAK,QAAQ,CACzD,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,EAChC,GAAG,EACH,OAAO,EACP,GAAG,GAAG,EAAE,EACR,SAAS,GAAG,MAAM,GAMnB;IACC,MAAM,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC;IAC/B,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAC1C,CAAC;IAED,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAEjE,MAAM,KAAK,GAAG,KAAK,CAAC,UAAU,EAAE,IAAI,EAAE;QACpC,GAAG;QACH,GAAG,EAAE;YACH,GAAG,OAAO,CAAC,GAAG;YACd,QAAQ,EAAE,YAAY;YACtB,GAAG,GAAG;YACN,MAAM,EAAE,GAAG;SACZ;QACD,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC;KACzC,CAAC,CAAC;IAEH,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,MAAM,MAAM,GAAG,GAAG,EAAE,CAAC,QAAQ,CAAC;IAC9B,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;QACjC,QAAQ,IAAI,KAAK,CAAC;IACpB,CAAC,CAAC,CAAC;IACH,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;QACjC,QAAQ,IAAI,KAAK,CAAC;IACpB,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,KAAK,IAAI,EAAE;QACtB,IAAI,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;YACrB,OAAO;QACT,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,OAAO,CAAO,CAAC,WAAW,EAAE,EAAE;YAC/C,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtB,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,CAAC,CAAC;QAC9D,MAAM,MAAM,CAAC;QACb,YAAY,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,MAAM,IAAI,OAAO,CAAS,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE;QAC7D,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,MAAM,CACJ,IAAI,KAAK,CACP,qCAAqC,SAAS,mHAAmH,MAAM,EAAE,EAAE,CAC5K,CACF,CAAC;QACJ,CAAC,EAAE,SAAS,CAAC,CAAC;QACd,KAAK,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE;YAC9B,IAAI,kBAAkB,CAAC,OAAO,CAAC,EAAE,CAAC;gBAChC,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YAC1B,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CACJ,IAAI,KAAK,CAAC,mBAAmB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CACpE,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;YACpB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CACJ,IAAI,KAAK,CACP,2BAA2B,KAAK,CAAC,QAAQ,gCAAgC,MAAM,EAAE,EAAE,CACpF,CACF,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,KAAc,EAAE,EAAE;QAChC,MAAM,IAAI,EAAE,CAAC;QACb,MAAM,KAAK,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,GAAG,EAAE,oBAAoB,IAAI,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;AAC/D,CAAC"}
1
+ {"version":3,"file":"server-harness.js","sourceRoot":"","sources":["../src/server-harness.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqB,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AASpC,SAAS,SAAS,CAAC,KAAmB;IACpC,OAAO,KAAK,CAAC,QAAQ,KAAK,IAAI,IAAI,KAAK,CAAC,UAAU,KAAK,IAAI,CAAC;AAC9D,CAAC;AAED,SAAS,kBAAkB,CACzB,OAAgB;IAEhB,OAAO,CACL,OAAO,OAAO,KAAK,QAAQ;QAC3B,OAAO,KAAK,IAAI;QACf,OAA8B,CAAC,IAAI,KAAK,qBAAqB;QAC9D,OAAQ,OAA8B,CAAC,IAAI,KAAK,QAAQ,CACzD,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,EAChC,GAAG,EACH,OAAO,EACP,GAAG,GAAG,EAAE,GAKT;IACC,MAAM,SAAS,GAAG,MAAM,CAAC;IACzB,MAAM,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC;IAC/B,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAC1C,CAAC;IAED,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAEjE,MAAM,KAAK,GAAG,KAAK,CAAC,UAAU,EAAE,IAAI,EAAE;QACpC,GAAG;QACH,GAAG,EAAE;YACH,GAAG,OAAO,CAAC,GAAG;YACd,QAAQ,EAAE,YAAY;YACtB,GAAG,GAAG;YACN,MAAM,EAAE,GAAG;SACZ;QACD,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC;KACzC,CAAC,CAAC;IAEH,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,MAAM,MAAM,GAAG,GAAG,EAAE,CAAC,QAAQ,CAAC;IAC9B,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;QACjC,QAAQ,IAAI,KAAK,CAAC;IACpB,CAAC,CAAC,CAAC;IACH,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;QACjC,QAAQ,IAAI,KAAK,CAAC;IACpB,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,KAAK,IAAI,EAAE;QACtB,IAAI,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;YACrB,OAAO;QACT,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,OAAO,CAAO,CAAC,WAAW,EAAE,EAAE;YAC/C,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtB,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,CAAC,CAAC;QAC9D,MAAM,MAAM,CAAC;QACb,YAAY,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,MAAM,IAAI,OAAO,CAAS,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE;QAC7D,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,MAAM,CACJ,IAAI,KAAK,CACP,mCAAmC,SAAS,qOAAqO,MAAM,EAAE,EAAE,CAC5R,CACF,CAAC;QACJ,CAAC,EAAE,SAAS,CAAC,CAAC;QACd,KAAK,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE;YAC9B,IAAI,kBAAkB,CAAC,OAAO,CAAC,EAAE,CAAC;gBAChC,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YAC1B,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CACJ,IAAI,KAAK,CAAC,mBAAmB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CACpE,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;YACpB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CACJ,IAAI,KAAK,CACP,2BAA2B,KAAK,CAAC,QAAQ,gCAAgC,MAAM,EAAE,EAAE,CACpF,CACF,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,KAAc,EAAE,EAAE;QAChC,MAAM,IAAI,EAAE,CAAC;QACb,MAAM,KAAK,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,GAAG,EAAE,oBAAoB,IAAI,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;AAC/D,CAAC"}
@@ -1,8 +1,9 @@
1
1
  import { Chat } from "./chat.js";
2
2
  /**
3
3
  * Opens a fresh MCP session and conversation against the server the plugin
4
- * started. Every chat opened during a test is closed by the plugin's
5
- * `afterEach`, so scenarios never leak sessions and never write teardown.
4
+ * started. The session is closed when the current test finishes, so scenarios
5
+ * never leak sessions and never write teardown, and tests that run
6
+ * concurrently cannot close each other's sessions.
6
7
  *
7
8
  * The type parameter pins the assertions to the project's registry:
8
9
  * `start<AppType>()` returns a `Chat<AppType>`, and `expect.chat` infers the
@@ -12,5 +13,4 @@ export declare function start<App>(overrides?: {
12
13
  systemPrompt?: string;
13
14
  temperature?: number;
14
15
  }): Promise<Chat<App>>;
15
- export declare function closeOpenedSessions(): Promise<void>;
16
16
  //# sourceMappingURL=session-registry.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAKjC;;;;;;;;GAQG;AACH,wBAAsB,KAAK,CAAC,GAAG,EAAE,SAAS,CAAC,EAAE;IAC3C,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAUrB;AAED,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,IAAI,CAAC,CAEzD"}
1
+ {"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC;;;;;;;;;GASG;AACH,wBAAsB,KAAK,CAAC,GAAG,EAAE,SAAS,CAAC,EAAE;IAC3C,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAUrB"}
@@ -1,11 +1,11 @@
1
- import { inject } from "vitest";
1
+ import { inject, onTestFinished } from "vitest";
2
2
  import { Chat } from "./chat.js";
3
3
  import { resolveModel } from "./model.js";
4
- const opened = [];
5
4
  /**
6
5
  * Opens a fresh MCP session and conversation against the server the plugin
7
- * started. Every chat opened during a test is closed by the plugin's
8
- * `afterEach`, so scenarios never leak sessions and never write teardown.
6
+ * started. The session is closed when the current test finishes, so scenarios
7
+ * never leak sessions and never write teardown, and tests that run
8
+ * concurrently cannot close each other's sessions.
9
9
  *
10
10
  * The type parameter pins the assertions to the project's registry:
11
11
  * `start<AppType>()` returns a `Chat<AppType>`, and `expect.chat` infers the
@@ -19,10 +19,7 @@ export async function start(overrides) {
19
19
  systemPrompt: overrides?.systemPrompt ?? config.systemPrompt,
20
20
  maxSteps: config.maxSteps,
21
21
  });
22
- opened.push(chat);
22
+ onTestFinished(() => chat.close());
23
23
  return chat;
24
24
  }
25
- export async function closeOpenedSessions() {
26
- await Promise.all(opened.splice(0).map((chat) => chat.close()));
27
- }
28
25
  //# sourceMappingURL=session-registry.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"session-registry.js","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE1C,MAAM,MAAM,GAAoB,EAAE,CAAC;AAEnC;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CAAM,SAGhC;IACC,MAAM,MAAM,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAM,MAAM,CAAC,mBAAmB,CAAC,EAAE;QAC7D,KAAK,EAAE,MAAM,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC;QACvC,WAAW,EAAE,SAAS,EAAE,WAAW,IAAI,MAAM,CAAC,WAAW;QACzD,YAAY,EAAE,SAAS,EAAE,YAAY,IAAI,MAAM,CAAC,YAAY;QAC5D,QAAQ,EAAE,MAAM,CAAC,QAAQ;KAC1B,CAAC,CAAC;IACH,MAAM,CAAC,IAAI,CAAC,IAAqB,CAAC,CAAC;IACnC,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB;IACvC,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAClE,CAAC"}
1
+ {"version":3,"file":"session-registry.js","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE1C;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CAAM,SAGhC;IACC,MAAM,MAAM,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAM,MAAM,CAAC,mBAAmB,CAAC,EAAE;QAC7D,KAAK,EAAE,MAAM,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC;QACvC,WAAW,EAAE,SAAS,EAAE,WAAW,IAAI,MAAM,CAAC,WAAW;QACzD,YAAY,EAAE,SAAS,EAAE,YAAY,IAAI,MAAM,CAAC,YAAY;QAC5D,QAAQ,EAAE,MAAM,CAAC,QAAQ;KAC1B,CAAC,CAAC;IACH,cAAc,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;IACnC,OAAO,IAAI,CAAC;AACd,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@skybridge/test",
3
- "version": "0.0.0-dev.50c10e1",
3
+ "version": "0.0.0-dev.874c2eb",
4
4
  "description": "Eval runner for Skybridge MCP apps: does a prompt reach the right tool with the right arguments",
5
5
  "license": "ISC",
6
6
  "type": "module",
@@ -22,20 +22,41 @@
22
22
  "dist"
23
23
  ],
24
24
  "dependencies": {
25
- "@ai-sdk/anthropic": "^2.0.0",
26
25
  "@modelcontextprotocol/sdk": "^1.29.0",
27
26
  "ai": "^5.0.0",
28
- "@skybridge/vite-plugin": "0.0.0-dev.50c10e1"
27
+ "@skybridge/vite-plugin": "0.0.0-dev.874c2eb"
29
28
  },
30
29
  "peerDependencies": {
31
- "skybridge": "^1.1.0",
30
+ "@ai-sdk/anthropic": "^2.0.0",
31
+ "@ai-sdk/google": "^2.0.0",
32
+ "@ai-sdk/mistral": "^2.0.0",
33
+ "@ai-sdk/openai": "^2.0.0",
34
+ "skybridge": "^2.0.0",
32
35
  "vitest": "^4.1.0"
33
36
  },
37
+ "peerDependenciesMeta": {
38
+ "@ai-sdk/anthropic": {
39
+ "optional": true
40
+ },
41
+ "@ai-sdk/google": {
42
+ "optional": true
43
+ },
44
+ "@ai-sdk/mistral": {
45
+ "optional": true
46
+ },
47
+ "@ai-sdk/openai": {
48
+ "optional": true
49
+ }
50
+ },
34
51
  "devDependencies": {
52
+ "@ai-sdk/anthropic": "^2.0.0",
53
+ "@ai-sdk/google": "^2.0.0",
54
+ "@ai-sdk/mistral": "^2.0.0",
55
+ "@ai-sdk/openai": "^2.0.0",
35
56
  "@types/node": "^24.13.3",
36
57
  "typescript": "^5.9.3",
37
58
  "vitest": "^4.1.10",
38
- "skybridge": "0.0.0-dev.50c10e1"
59
+ "skybridge": "0.0.0-dev.874c2eb"
39
60
  },
40
61
  "engines": {
41
62
  "node": ">=22.23.1"
package/dist/setup.d.ts DELETED
@@ -1,2 +0,0 @@
1
- import "./matchers.js";
2
- //# sourceMappingURL=setup.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"setup.d.ts","sourceRoot":"","sources":["../src/setup.ts"],"names":[],"mappings":"AACA,OAAO,eAAe,CAAC"}
package/dist/setup.js DELETED
@@ -1,7 +0,0 @@
1
- import { afterEach } from "vitest";
2
- import "./matchers.js";
3
- import { closeOpenedSessions } from "./session-registry.js";
4
- afterEach(async () => {
5
- await closeOpenedSessions();
6
- });
7
- //# sourceMappingURL=setup.js.map
package/dist/setup.js.map DELETED
@@ -1 +0,0 @@
1
- {"version":3,"file":"setup.js","sourceRoot":"","sources":["../src/setup.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,eAAe,CAAC;AACvB,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAE5D,SAAS,CAAC,KAAK,IAAI,EAAE;IACnB,MAAM,mBAAmB,EAAE,CAAC;AAC9B,CAAC,CAAC,CAAC"}