@skybridge/test 0.0.0-dev.7aada9d → 0.0.0-dev.dde647a

package/README.md

@@ -0,0 +1,90 @@
+# @skybridge/test
+
+Evals for [Skybridge](https://skybridge.tech) MCP apps: they answer whether a
+prompt reaches the right tool with the right arguments, which no unit test can
+check.
+
+**Beta.** The API is still settling, so it publishes under the `beta`
+dist-tag and may change between releases.
+
+Add the `evals` option to the Skybridge Vite plugin, then write scenarios with
+vitest's own `it` and `expect`.
+
+```ts
+import { expect, it } from "vitest";
+import { start } from "@skybridge/test";
+import type { AppType } from "../src/server.js";
+
+it("finds products by category", async () => {
+  const chat = await start<AppType>();
+  await chat.send("I am looking for ski goggles");
+
+  expect.chat(chat).toHaveCalledToolWith("search-products", {
+    category: "goggles",
+  });
+});
+```
+
+## Choosing a model
+
+`model` is a `provider/model-id` string. Only the provider you name is
+imported, so install just that one as a dev dependency, and the key is read
+from its own environment variable.
+
+| prefix | package | key |
+| --- | --- | --- |
+| `anthropic` | `@ai-sdk/anthropic` | `ANTHROPIC_API_KEY` |
+| `openai` | `@ai-sdk/openai` | `OPENAI_API_KEY` |
+| `mistral` | `@ai-sdk/mistral` | `MISTRAL_API_KEY` |
+| `google` | `@ai-sdk/google` | `GOOGLE_GENERATIVE_AI_API_KEY` |
+
+```ts
+skybridge({
+  evals: {
+    model: "anthropic/claude-sonnet-4-5",
+    project: { cwd: ".", command: ["node", "dist/server.js"] },
+  },
+});
+```
+
+`command` has to start the server process directly, since wrappers such as
+`pnpm run` do not forward the channel the runner uses to learn the port.
+
+Any other prefix is passed to the AI SDK untouched, so a string such as
+`openai/gpt-5` also resolves through the Vercel AI Gateway if you have
+`AI_GATEWAY_API_KEY` set and no provider package installed.
+
+## Models the prefixes cannot reach
+
+For a local runtime, a company gateway, or a model that needs middleware,
+build the model yourself and register it from a vitest setup file. It takes
+precedence over `model`, which can then be left out.
+
+```ts
+// eval-model.ts
+import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
+import { defineEvalModel } from "@skybridge/test";
+
+defineEvalModel(
+  createOpenAICompatible({ baseURL: "http://localhost:11434/v1" })("llama3.1"),
+);
+```
+
+```ts
+// vitest.config.ts
+export default defineConfig({
+  plugins: [skybridge({ evals: { project: { ... } } })],
+  test: { setupFiles: ["./eval-model.ts"] },
+});
+```
+
+Tool calls the server rejected do not count towards a positive assertion, so a
+scenario cannot pass on a call that errored. To assert an error on purpose, use
+`toHaveFailedToolCall`.
+
+```ts
+expect.chat(chat).toHaveFailedToolCall("create-checkout");
+```
+
+See the [documentation](https://docs.skybridge.tech) for setup and the full
+matcher list.

package/dist/chat.d.ts

@@ -8,15 +8,10 @@ export type ToolCall<App> = {
     [Name in ToolNames<App>]: {
         name: Name;
         arguments: ToolInput<App, Name>;
-        invalid?: boolean;
+        failed?: string;
     };
 }[ToolNames<App>];
-/**
- * How the host is configured. `model` is an AI SDK model instance, so the
- * provider and its credentials are chosen by the project rather than by a
- * base URL the runner has to know about.
- */
-export interface HostConfig {
+interface HostConfig {
     model: LanguageModel;
     temperature?: number;
     systemPrompt?: string;
@@ -35,6 +30,7 @@ export declare class Chat<App = unknown> {
     /** Every call the model made, in order, as it crossed the wire. */
     readonly toolCalls: ToolCall<App>[];
     private readonly messages;
+    private readonly failures;
     private readonly client;
     private readonly transport;
     private readonly host;
@@ -46,6 +42,7 @@ export declare class Chat<App = unknown> {
     /** Takes a turn and returns the model's answer. */
     send(prompt: string): Promise<string>;
     close(): Promise<void>;
+    private failureFor;
     private toolSet;
 }
 export {};

package/dist/chat.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAEA,OAAO,EAIL,KAAK,aAAa,EAGnB,MAAM,IAAI,CAAC;AACZ,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7D;;;GAGG;AACH,MAAM,MAAM,QAAQ,CAAC,GAAG,IAAI;KACzB,IAAI,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG;QACxB,IAAI,EAAE,IAAI,CAAC;QACX,SAAS,EAAE,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAChC,OAAO,CAAC,EAAE,OAAO,CAAC;KACnB;CACF,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;AAElB;;;;GAIG;AACH,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,aAAa,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,UAAU,iBAAiB;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACtC;AAID;;;GAGG;AACH,qBAAa,IAAI,CAAC,GAAG,GAAG,OAAO;IAC7B,mEAAmE;IACnE,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAM;IAEzC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAa;IAClC,OAAO,CAAC,KAAK,CAA2B;IAExC,OAAO;WAUM,IAAI,CAAC,GAAG,EACnB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,UAAU,GACf,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAerB,0EAA0E;IAC1E,IAAI,eAAe,IAAI,iBAAiB,EAAE,CAEzC;IAED,mDAAmD;IAC7C,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IA4BrC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,OAAO,CAAC,OAAO;CAoBhB"}
+{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAEA,OAAO,EAIL,KAAK,aAAa,EAGnB,MAAM,IAAI,CAAC;AACZ,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7D;;;GAGG;AACH,MAAM,MAAM,QAAQ,CAAC,GAAG,IAAI;KACzB,IAAI,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG;QACxB,IAAI,EAAE,IAAI,CAAC;QACX,SAAS,EAAE,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAChC,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB;CACF,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;AAElB,UAAU,UAAU;IAClB,KAAK,EAAE,aAAa,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,UAAU,iBAAiB;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACtC;AAID;;;GAGG;AACH,qBAAa,IAAI,CAAC,GAAG,GAAG,OAAO;IAC7B,mEAAmE;IACnE,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAM;IAEzC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA6B;IACtD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAa;IAClC,OAAO,CAAC,KAAK,CAA2B;IAExC,OAAO;WAUM,IAAI,CAAC,GAAG,EACnB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,UAAU,GACf,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAerB,0EAA0E;IAC1E,IAAI,eAAe,IAAI,iBAAiB,EAAE,CAEzC;IAED,mDAAmD;IAC7C,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IA4BrC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,OAAO,CAAC,UAAU;IAalB,OAAO,CAAC,OAAO;CAmChB"}

package/dist/chat.js

@@ -10,6 +10,7 @@ export class Chat {
     /** Every call the model made, in order, as it crossed the wire. */
     toolCalls = [];
     messages = [];
+    failures = new Map();
     client;
     transport;
     host;
@@ -56,7 +57,7 @@ export class Chat {
                 this.toolCalls.push({
                     name: call.toolName,
                     arguments: (call.input ?? {}),
-                    ...(call.invalid === true ? { invalid: true } : {}),
+                    ...this.failureFor(call),
                 });
             }
         }
@@ -66,6 +67,16 @@ export class Chat {
     async close() {
         await this.transport.close();
     }
+    failureFor(call) {
+        const reported = this.failures.get(call.toolCallId);
+        if (reported !== undefined) {
+            return { failed: reported };
+        }
+        if (call.invalid === true) {
+            return { failed: "the model's arguments could not be parsed" };
+        }
+        return {};
+    }
     toolSet() {
         const entries = this.tools.map((definition) => {
             const { $schema: _ignored, ...parameters } = definition.inputSchema;
@@ -74,11 +85,21 @@ export class Chat {
                 dynamicTool({
                     description: definition.description ?? "",
                     inputSchema: jsonSchema(parameters),
-                    execute: async (input) => {
-                        const result = await this.client.callTool({
-                            name: definition.name,
-                            arguments: (input ?? {}),
-                        });
+                    execute: async (input, { toolCallId }) => {
+                        let result;
+                        try {
+                            result = await this.client.callTool({
+                                name: definition.name,
+                                arguments: (input ?? {}),
+                            });
+                        }
+                        catch (error) {
+                            this.failures.set(toolCallId, `the call never reached the server: ${error instanceof Error ? error.message : String(error)}`);
+                            throw error;
+                        }
+                        if (result.isError === true) {
+                            this.failures.set(toolCallId, `the tool returned an error: ${JSON.stringify(result.content)}`);
+                        }
                         return result.content ?? result;
                     },
                 }),

package/dist/chat.js.map

@@ -1 +1 @@
-{"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,6BAA6B,EAAE,MAAM,oDAAoD,CAAC;AACnG,OAAO,EACL,WAAW,EACX,YAAY,EACZ,UAAU,EAGV,WAAW,GACZ,MAAM,IAAI,CAAC;AAiCZ,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B;;;GAGG;AACH,MAAM,OAAO,IAAI;IACf,mEAAmE;IAC1D,SAAS,GAAoB,EAAE,CAAC;IAExB,QAAQ,GAAmB,EAAE,CAAC;IAC9B,MAAM,CAAS;IACf,SAAS,CAAgC;IACzC,IAAI,CAAa;IAC1B,KAAK,GAAwB,EAAE,CAAC;IAExC,YACE,MAAc,EACd,SAAwC,EACxC,IAAgB;QAEhB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,IAAI,CACf,SAAiB,EACjB,IAAgB;QAEhB,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,IAAI,6BAA6B,CAAC,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;QACxE,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAM,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QACpD,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,GAAG,KAAuC,CAAC;QACvD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YACxB,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,0EAA0E;IAC1E,IAAI,eAAe;QACjB,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED,mDAAmD;IACnD,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAEtD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;YACtB,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC;YACvC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,KAAK,SAAS;gBACtC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE;YACrB,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;SAC/D,CAAC,CAAC;QAEH,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAChC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAClC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;oBAClB,IAAI,EAAE,IAAI,CAAC,QAAQ;oBACnB,SAAS,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAA4B;oBACxD,GAAG,CAAC,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACnC,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAChD,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;IAEO,OAAO;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;YAC5C,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,UAAU,EAAE,GAAG,UAAU,CAAC,WAAW,CAAC;YACpE,OAAO;gBACL,UAAU,CAAC,IAAI;gBACf,WAAW,CAAC;oBACV,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,EAAE;oBACzC,WAAW,EAAE,UAAU,CAAC,UAAU,CAAC;oBACnC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;wBACvB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;4BACxC,IAAI,EAAE,UAAU,CAAC,IAAI;4BACrB,SAAS,EAAE,CAAC,KAAK,IAAI,EAAE,CAA4B;yBACpD,CAAC,CAAC;wBACH,OAAO,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC;oBAClC,CAAC;iBACF,CAAC;aACM,CAAC;QACb,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;CACF"}
+{"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,6BAA6B,EAAE,MAAM,oDAAoD,CAAC;AACnG,OAAO,EACL,WAAW,EACX,YAAY,EACZ,UAAU,EAGV,WAAW,GACZ,MAAM,IAAI,CAAC;AA4BZ,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B;;;GAGG;AACH,MAAM,OAAO,IAAI;IACf,mEAAmE;IAC1D,SAAS,GAAoB,EAAE,CAAC;IAExB,QAAQ,GAAmB,EAAE,CAAC;IAC9B,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrC,MAAM,CAAS;IACf,SAAS,CAAgC;IACzC,IAAI,CAAa;IAC1B,KAAK,GAAwB,EAAE,CAAC;IAExC,YACE,MAAc,EACd,SAAwC,EACxC,IAAgB;QAEhB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,IAAI,CACf,SAAiB,EACjB,IAAgB;QAEhB,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,IAAI,6BAA6B,CAAC,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;QACxE,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAM,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QACpD,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACrB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YACxB,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,0EAA0E;IAC1E,IAAI,eAAe;QACjB,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED,mDAAmD;IACnD,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAEtD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;YACtB,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC;YACvC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,KAAK,SAAS;gBACtC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE;YACrB,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;SAC/D,CAAC,CAAC;QAEH,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAChC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAClC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;oBAClB,IAAI,EAAE,IAAI,CAAC,QAAQ;oBACnB,SAAS,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAA4B;oBACxD,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;iBACR,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAChD,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;IAEO,UAAU,CAAC,IAA+C;QAGhE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QAC9B,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC1B,OAAO,EAAE,MAAM,EAAE,2CAA2C,EAAE,CAAC;QACjE,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAEO,OAAO;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;YAC5C,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,UAAU,EAAE,GAAG,UAAU,CAAC,WAAW,CAAC;YACpE,OAAO;gBACL,UAAU,CAAC,IAAI;gBACf,WAAW,CAAC;oBACV,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,EAAE;oBACzC,WAAW,EAAE,UAAU,CAAC,UAAU,CAAC;oBACnC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE;wBACvC,IAAI,MAA+C,CAAC;wBACpD,IAAI,CAAC;4BACH,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;gCAClC,IAAI,EAAE,UAAU,CAAC,IAAI;gCACrB,SAAS,EAAE,CAAC,KAAK,IAAI,EAAE,CAA4B;6BACpD,CAAC,CAAC;wBACL,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BACf,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,sCAAsC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC/F,CAAC;4BACF,MAAM,KAAK,CAAC;wBACd,CAAC;wBACD,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;4BAC5B,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,+BAA+B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAChE,CAAC;wBACJ,CAAC;wBACD,OAAO,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC;oBAClC,CAAC;iBACF,CAAC;aACM,CAAC;QACb,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;CACF"}

package/dist/config.d.ts

@@ -1,39 +1,8 @@
-/** A provider we can construct ourselves, so no extra file is needed. */
-export interface ModelDescriptor {
-    provider: "anthropic";
-    name: string;
-    /** Env var holding the key. Never the key itself. */
-    apiKeyEnv?: string;
-}
-/** Escape hatch: a module that calls `defineEvalModel` with its own instance. */
-export interface ModelFile {
-    file: string;
-}
-export interface EvalsOptions {
-    model: ModelDescriptor | ModelFile;
-    temperature?: number;
-    systemPrompt?: string;
-    maxSteps?: number;
-    server?: string;
-    /**
-     * How the runner starts the project's server. `command` must start the
-     * server process directly (`node`, `tsx`), not through a wrapper such as
-     * `pnpm run`: readiness and the bound port arrive over the spawned
-     * process's IPC channel, which wrappers do not forward.
-     */
-    project?: {
-        cwd: string;
-        command: string[];
-        env?: Record<string, string>;
-    };
-}
-/** The serializable half, the only part that crosses into the workers. */
-export type ProvidedEvalsOptions = Omit<EvalsOptions, "model"> & {
-    model?: ModelDescriptor;
-};
+import type { EvalsOptions } from "@skybridge/vite-plugin";
+export type { EvalsOptions } from "@skybridge/vite-plugin";
 declare module "vitest" {
     interface ProvidedContext {
-        skybridgeEvals: ProvidedEvalsOptions;
+        skybridgeEvals: EvalsOptions;
         skybridgeEvalsUrl: string;
     }
 }

package/dist/config.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,yEAAyE;AACzE,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,WAAW,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,qDAAqD;IACrD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,iFAAiF;AACjF,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,eAAe,GAAG,SAAS,CAAC;IACnC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;OAKG;IACH,OAAO,CAAC,EAAE;QACR,GAAG,EAAE,MAAM,CAAC;QACZ,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAC9B,CAAC;CACH;AAED,0EAA0E;AAC1E,MAAM,MAAM,oBAAoB,GAAG,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,GAAG;IAC/D,KAAK,CAAC,EAAE,eAAe,CAAC;CACzB,CAAC;AAEF,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,eAAe;QACvB,cAAc,EAAE,oBAAoB,CAAC;QACrC,iBAAiB,EAAE,MAAM,CAAC;KAC3B;CACF"}
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAE3D,YAAY,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAE3D,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,eAAe;QACvB,cAAc,EAAE,YAAY,CAAC;QAC7B,iBAAiB,EAAE,MAAM,CAAC;KAC3B;CACF"}

package/dist/global-setup.d.ts

@@ -1,3 +1,3 @@
 import type { TestProject } from "vitest/node";
-export default function setup(project: TestProject): Promise<() => Promise<void>>;
+export default function setup(project: TestProject): Promise<(() => Promise<void>) | undefined>;
 //# sourceMappingURL=global-setup.d.ts.map

package/dist/global-setup.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"global-setup.d.ts","sourceRoot":"","sources":["../src/global-setup.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAK/C,wBAA8B,KAAK,CAAC,OAAO,EAAE,WAAW,gCAwCvD"}
+{"version":3,"file":"global-setup.d.ts","sourceRoot":"","sources":["../src/global-setup.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAG/C,wBAA8B,KAAK,CAAC,OAAO,EAAE,WAAW,8CA0BvD"}

package/dist/global-setup.js

@@ -1,28 +1,17 @@
 import { startServer } from "./server-harness.js";
-import { captureEvalRun } from "./telemetry.js";
 export default async function setup(project) {
-    const options = project.config.provide
-        ?.skybridgeEvals;
-    const startedAt = Date.now();
+    const options = project.config.provide?.skybridgeEvals;
+    if (options === undefined) {
+        throw new Error("The evals plugin is not registered. Add `skybridge({ evals: {...} })` to the vitest config.");
+    }
     if (options.server !== undefined) {
         project.provide("skybridgeEvalsUrl", options.server);
-        return async () => {
-            await captureEvalRun({
-                durationMs: Date.now() - startedAt,
-                serverStarted: true,
-            });
-        };
+        return;
     }
     if (options.project === undefined) {
         throw new Error("The evals plugin needs either `server` or `project`");
     }
-    const server = await startServer(options.project).catch(async (error) => {
-        await captureEvalRun({
-            durationMs: Date.now() - startedAt,
-            serverStarted: false,
-        });
-        throw error;
-    });
+    const server = await startServer(options.project);
     project.provide("skybridgeEvalsUrl", server.url);
     return async () => {
         const output = server.output().trim();
@@ -30,10 +19,6 @@ export default async function setup(project) {
         if (output !== "") {
             console.log(`\n--- server output ---\n${output}\n---`);
         }
-        await captureEvalRun({
-            durationMs: Date.now() - startedAt,
-            serverStarted: true,
-        });
     };
 }
 //# sourceMappingURL=global-setup.js.map

package/dist/global-setup.js.map

@@ -1 +1 @@
-{"version":3,"file":"global-setup.js","sourceRoot":"","sources":["../src/global-setup.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAEhD,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,KAAK,CAAC,OAAoB;IACtD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO;QACpC,EAAE,cAAsC,CAAC;IAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QACjC,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACrD,OAAO,KAAK,IAAI,EAAE;YAChB,MAAM,cAAc,CAAC;gBACnB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAClC,aAAa,EAAE,IAAI;aACpB,CAAC,CAAC;QACL,CAAC,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,KAAK,CACrD,KAAK,EAAE,KAAc,EAAE,EAAE;QACvB,MAAM,cAAc,CAAC;YACnB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,aAAa,EAAE,KAAK;SACrB,CAAC,CAAC;QACH,MAAM,KAAK,CAAC;IACd,CAAC,CACF,CAAC;IACF,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;IAEjD,OAAO,KAAK,IAAI,EAAE;QAChB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;YAClB,OAAO,CAAC,GAAG,CAAC,4BAA4B,MAAM,OAAO,CAAC,CAAC;QACzD,CAAC;QACD,MAAM,cAAc,CAAC;YACnB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,aAAa,EAAE,IAAI;SACpB,CAAC,CAAC;IACL,CAAC,CAAC;AACJ,CAAC"}
+{"version":3,"file":"global-setup.js","sourceRoot":"","sources":["../src/global-setup.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAElD,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,KAAK,CAAC,OAAoB;IACtD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,cAAc,CAAC;IACvD,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,6FAA6F,CAC9F,CAAC;IACJ,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QACjC,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACrD,OAAO;IACT,CAAC;IACD,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAClD,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;IAEjD,OAAO,KAAK,IAAI,EAAE;QAChB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;YAClB,OAAO,CAAC,GAAG,CAAC,4BAA4B,MAAM,OAAO,CAAC,CAAC;QACzD,CAAC;IACH,CAAC,CAAC;AACJ,CAAC"}

package/dist/index.d.ts

@@ -1,5 +1,4 @@
-export type { Chat, HostConfig, ToolCall } from "./chat.js";
-export type { EvalsOptions, ModelDescriptor, ModelFile, ProvidedEvalsOptions, } from "./config.js";
+export type { Chat } from "./chat.js";
 export type { ChatAssertion, ChatMatchers } from "./matchers.js";
 export { defineEvalModel } from "./model.js";
 export { start } from "./session-registry.js";

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC5D,YAAY,EACV,YAAY,EACZ,eAAe,EACf,SAAS,EACT,oBAAoB,GACrB,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AACjE,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACtC,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AACjE,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}

package/dist/matchers.d.ts

@@ -9,6 +9,7 @@ export interface ChatMatchers<App> {
     toHaveCalledToolOnce<Name extends ToolNames<App>>(name: Name, args?: Partial<ToolInput<App, Name>>): void;
     toHaveCalledToolWith<Name extends ToolNames<App>>(name: Name, args: Partial<ToolInput<App, Name>>): void;
     toNeverHaveCalledTool<Name extends ToolNames<App>>(name: Name): void;
+    toHaveFailedToolCall<Name extends ToolNames<App>>(name: Name): void;
 }
 export interface ChatAssertion<App> extends ChatMatchers<App> {
     not: ChatMatchers<App>;

package/dist/matchers.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"matchers.d.ts","sourceRoot":"","sources":["../src/matchers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7D,OAAO,KAAK,EAAE,IAAI,EAAY,MAAM,WAAW,CAAC;AAEhD;;;;GAIG;AACH,MAAM,WAAW,YAAY,CAAC,GAAG;IAC/B,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAC9C,IAAI,EAAE,IAAI,EACV,IAAI,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GACnC,IAAI,CAAC;IACR,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAC9C,IAAI,EAAE,IAAI,EACV,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GAClC,IAAI,CAAC;IACR,qBAAqB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,CAAC;CACtE;AAED,MAAM,WAAW,aAAa,CAAC,GAAG,CAAE,SAAQ,YAAY,CAAC,GAAG,CAAC;IAC3D,GAAG,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC;CACxB;AAED,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,YAAY;QACpB,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;KAChD;CACF"}
+{"version":3,"file":"matchers.d.ts","sourceRoot":"","sources":["../src/matchers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7D,OAAO,KAAK,EAAE,IAAI,EAAY,MAAM,WAAW,CAAC;AAEhD;;;;GAIG;AACH,MAAM,WAAW,YAAY,CAAC,GAAG;IAC/B,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAC9C,IAAI,EAAE,IAAI,EACV,IAAI,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GACnC,IAAI,CAAC;IACR,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAC9C,IAAI,EAAE,IAAI,EACV,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GAClC,IAAI,CAAC;IACR,qBAAqB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,CAAC;IACrE,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,CAAC;CACrE;AAED,MAAM,WAAW,aAAa,CAAC,GAAG,CAAE,SAAQ,YAAY,CAAC,GAAG,CAAC;IAC3D,GAAG,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC;CACxB;AAED,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,YAAY;QACpB,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;KAChD;CACF"}

package/dist/matchers.js

@@ -1,26 +1,16 @@
 import { expect } from "vitest";
-/**
- * Every attempt at a tool, including calls the tool's schema rejected. Used by
- * the negative matcher: an attempt the model should not have made is a failure
- * whether or not its arguments parsed.
- */
-function attemptsAt(chat, name) {
+function attemptedCalls(chat, name) {
     return chat.toolCalls.filter((call) => String(call.name) === name);
 }
-/**
- * Only the calls that actually reached the server. Positive matchers use this,
- * so a rejected call never satisfies an assertion that the tool ran; the
- * rejected attempt still shows up in the failure report.
- */
-function callsTo(chat, name) {
-    return attemptsAt(chat, name).filter((call) => call.invalid !== true);
+function acceptedCalls(chat, name) {
+    return attemptedCalls(chat, name).filter((call) => call.failed === undefined);
 }
 function observed(chat) {
     if (chat.toolCalls.length === 0) {
         return "  (no tool was called)";
     }
     return chat.toolCalls
-        .map((call, index) => `  ${index + 1}. ${String(call.name)} ${JSON.stringify(call.arguments)}${call.invalid === true ? "  (arguments rejected by the tool schema)" : ""}`)
+        .map((call, index) => `  ${index + 1}. ${call.name} ${JSON.stringify(call.arguments)}${call.failed === undefined ? "" : `  (failed: ${call.failed})`}`)
         .join("\n");
 }
 function definitions(chat) {
@@ -39,7 +29,7 @@ ${definitions(chat)}`;
 }
 expect.extend({
     toHaveCalledToolOnce(received, name, args) {
-        const matching = callsTo(received, name);
+        const matching = acceptedCalls(received, name);
         const argsMatch = args === undefined ||
             (matching.length === 1 &&
                 this.equals(matching[0]?.arguments, expect.objectContaining(args)));
@@ -50,11 +40,11 @@ expect.extend({
                 ? `Expected "${name}" not to be called exactly once.`
                 : matching.length === 1
                     ? `Expected "${name}" to be called once with ${JSON.stringify(args)}, but it was called with ${JSON.stringify(matching[0]?.arguments)}.`
-                    : `Expected "${name}" to be called exactly once by a call the tool schema accepted, but ${matching.length} such calls were made.`),
+                    : `Expected "${name}" to be called exactly once by a call that succeeded, but ${matching.length} such calls were made.`),
         };
     },
     toHaveCalledToolWith(received, name, args) {
-        const pass = callsTo(received, name).some((call) => this.equals(call.arguments, expect.objectContaining(args)));
+        const pass = acceptedCalls(received, name).some((call) => this.equals(call.arguments, expect.objectContaining(args)));
         return {
             pass,
             message: () => report(received, pass
@@ -62,8 +52,20 @@ expect.extend({
                 : `Expected "${name}" to be called with ${JSON.stringify(args)}.`),
         };
     },
+    toHaveFailedToolCall(received, name) {
+        const attempted = attemptedCalls(received, name);
+        const failed = attempted.filter((call) => call.failed !== undefined);
+        return {
+            pass: failed.length > 0,
+            message: () => report(received, failed.length > 0
+                ? `Expected every call to "${name}" to succeed.`
+                : attempted.length === 0
+                    ? `Expected "${name}" to be called and fail, but it was never called.`
+                    : `Expected "${name}" to fail, but all ${attempted.length} calls succeeded.`),
+        };
+    },
     toNeverHaveCalledTool(received, name) {
-        const matching = attemptsAt(received, name);
+        const matching = attemptedCalls(received, name);
         return {
             pass: matching.length === 0,
             message: () => report(received, matching.length === 0
@@ -72,5 +74,5 @@ expect.extend({
         };
     },
 });
-expect.chat = ((chat) => expect(chat));
+expect.chat = (chat) => expect(chat);
 //# sourceMappingURL=matchers.js.map

package/dist/matchers.js.map

@@ -1 +1 @@
-{"version":3,"file":"matchers.js","sourceRoot":"","sources":["../src/matchers.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AA8BhC;;;;GAIG;AACH,SAAS,UAAU,CAAC,IAAmB,EAAE,IAAY;IACnD,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;AACrE,CAAC;AAED;;;;GAIG;AACH,SAAS,OAAO,CAAC,IAAmB,EAAE,IAAY;IAChD,OAAO,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC;AACxE,CAAC;AAED,SAAS,QAAQ,CAAC,IAAmB;IACnC,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,wBAAwB,CAAC;IAClC,CAAC;IACD,OAAO,IAAI,CAAC,SAAS;SAClB,GAAG,CACF,CAAC,IAAuB,EAAE,KAAa,EAAE,EAAE,CACzC,KAAK,KAAK,GAAG,CAAC,KAAK,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,2CAA2C,CAAC,CAAC,CAAC,EAAE,EAAE,CACtJ;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,WAAW,CAAC,IAAmB;IACtC,OAAO,IAAI,CAAC,eAAe;SACxB,GAAG,CACF,CAAC,IAAI,EAAE,EAAE,CACP,OAAO,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,WAAW,IAAI,EAAE,oBAAoB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CACpG;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,MAAM,CAAC,IAAmB,EAAE,OAAe;IAClD,OAAO,GAAG,OAAO;;;EAGjB,QAAQ,CAAC,IAAI,CAAC;;;EAGd,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;AACtB,CAAC;AAED,MAAM,CAAC,MAAM,CAAC;IACZ,oBAAoB,CAClB,QAAuB,EACvB,IAAY,EACZ,IAA8B;QAE9B,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QACzC,MAAM,SAAS,GACb,IAAI,KAAK,SAAS;YAClB,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACpB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxE,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC;QAEhD,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,aAAa,IAAI,kCAAkC;gBACrD,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;oBACrB,CAAC,CAAC,aAAa,IAAI,4BAA4B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,4BAA4B,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG;oBACxI,CAAC,CAAC,aAAa,IAAI,uEAAuE,QAAQ,CAAC,MAAM,wBAAwB,CACtI;SACJ,CAAC;IACJ,CAAC;IAED,oBAAoB,CAClB,QAAuB,EACvB,IAAY,EACZ,IAA6B;QAE7B,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CACjD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAC3D,CAAC;QAEF,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,wBAAwB,IAAI,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG;gBAC/D,CAAC,CAAC,aAAa,IAAI,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CACpE;SACJ,CAAC;IACJ,CAAC;IAED,qBAAqB,CAAC,QAAuB,EAAE,IAAY;QACzD,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAE5C,OAAO;YACL,IAAI,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC;YAC3B,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACnB,CAAC,CAAC,aAAa,IAAI,+BAA+B;gBAClD,CAAC,CAAC,aAAa,IAAI,2CAA2C,QAAQ,CAAC,MAAM,SAAS,CACzF;SACJ,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,IAAI,GAAG,CAAC,CAAM,IAAe,EAAE,EAAE,CACtC,MAAM,CAAC,IAAI,CAAC,CAAkC,CAAC"}
+{"version":3,"file":"matchers.js","sourceRoot":"","sources":["../src/matchers.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AA+BhC,SAAS,cAAc,CACrB,IAAmB,EACnB,IAAY;IAEZ,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;AACrE,CAAC;AAED,SAAS,aAAa,CAAC,IAAmB,EAAE,IAAY;IACtD,OAAO,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,QAAQ,CAAC,IAAmB;IACnC,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,wBAAwB,CAAC;IAClC,CAAC;IACD,OAAO,IAAI,CAAC,SAAS;SAClB,GAAG,CACF,CAAC,IAAuB,EAAE,KAAa,EAAE,EAAE,CACzC,KAAK,KAAK,GAAG,CAAC,KAAK,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,cAAc,IAAI,CAAC,MAAM,GAAG,EAAE,CACnI;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,WAAW,CAAC,IAAmB;IACtC,OAAO,IAAI,CAAC,eAAe;SACxB,GAAG,CACF,CAAC,IAAI,EAAE,EAAE,CACP,OAAO,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,WAAW,IAAI,EAAE,oBAAoB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CACpG;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,MAAM,CAAC,IAAmB,EAAE,OAAe;IAClD,OAAO,GAAG,OAAO;;;EAGjB,QAAQ,CAAC,IAAI,CAAC;;;EAGd,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;AACtB,CAAC;AAED,MAAM,CAAC,MAAM,CAAC;IACZ,oBAAoB,CAClB,QAAuB,EACvB,IAAY,EACZ,IAA8B;QAE9B,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC/C,MAAM,SAAS,GACb,IAAI,KAAK,SAAS;YAClB,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACpB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxE,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC;QAEhD,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,aAAa,IAAI,kCAAkC;gBACrD,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;oBACrB,CAAC,CAAC,aAAa,IAAI,4BAA4B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,4BAA4B,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG;oBACxI,CAAC,CAAC,aAAa,IAAI,6DAA6D,QAAQ,CAAC,MAAM,wBAAwB,CAC5H;SACJ,CAAC;IACJ,CAAC;IAED,oBAAoB,CAClB,QAAuB,EACvB,IAAY,EACZ,IAA6B;QAE7B,MAAM,IAAI,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CACvD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAC3D,CAAC;QAEF,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,wBAAwB,IAAI,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG;gBAC/D,CAAC,CAAC,aAAa,IAAI,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CACpE;SACJ,CAAC;IACJ,CAAC;IAED,oBAAoB,CAAC,QAAuB,EAAE,IAAY;QACxD,MAAM,SAAS,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;QAErE,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC;YACvB,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,MAAM,CAAC,MAAM,GAAG,CAAC;gBACf,CAAC,CAAC,2BAA2B,IAAI,eAAe;gBAChD,CAAC,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;oBACtB,CAAC,CAAC,aAAa,IAAI,mDAAmD;oBACtE,CAAC,CAAC,aAAa,IAAI,sBAAsB,SAAS,CAAC,MAAM,mBAAmB,CACjF;SACJ,CAAC;IACJ,CAAC;IAED,qBAAqB,CAAC,QAAuB,EAAE,IAAY;QACzD,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAEhD,OAAO;YACL,IAAI,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC;YAC3B,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACnB,CAAC,CAAC,aAAa,IAAI,+BAA+B;gBAClD,CAAC,CAAC,aAAa,IAAI,2CAA2C,QAAQ,CAAC,MAAM,SAAS,CACzF;SACJ,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,IAAI,GAAG,CAAM,IAAe,EAAsB,EAAE,CACzD,MAAM,CAAC,IAAI,CAAkC,CAAC"}

package/dist/model.d.ts

@@ -1,9 +1,15 @@
 import type { LanguageModel } from "ai";
-import type { ModelDescriptor } from "./config.js";
 /**
- * Registers a model instance from a setup file. The escape hatch for providers
- * the descriptor cannot construct, such as a company LLM gateway.
+ * Registers a model instance from a setup file, taking precedence over the
+ * configured `model` string. The escape hatch for anything the provider
+ * prefixes cannot express, such as a local runtime or a company LLM gateway.
  */
 export declare function defineEvalModel(model: LanguageModel): void;
-export declare function resolveModel(descriptor: ModelDescriptor | undefined): Promise<LanguageModel>;
+/**
+ * Turns a `provider/model-id` string into a model instance, importing only the
+ * provider named. Each provider package reads its own conventional API key
+ * variable, so no key configuration is needed. An unrecognised prefix is
+ * returned untouched for the AI SDK's default provider to resolve.
+ */
+export declare function resolveModel(spec: string | undefined): Promise<LanguageModel>;
 //# sourceMappingURL=model.d.ts.map

package/dist/model.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"model.d.ts","sourceRoot":"","sources":["../src/model.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACxC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAInD;;;GAGG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAE1D;AAaD,wBAAsB,YAAY,CAChC,UAAU,EAAE,eAAe,GAAG,SAAS,GACtC,OAAO,CAAC,aAAa,CAAC,CAWxB"}
+{"version":3,"file":"model.d.ts","sourceRoot":"","sources":["../src/model.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAaxC;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAE1D;AAED;;;;;GAKG;AACH,wBAAsB,YAAY,CAChC,IAAI,EAAE,MAAM,GAAG,SAAS,GACvB,OAAO,CAAC,aAAa,CAAC,CAwBxB"}

package/dist/model.js

@@ -1,28 +1,43 @@
+const PROVIDERS = {
+    anthropic: () => import("@ai-sdk/anthropic").then((m) => m.anthropic),
+    google: () => import("@ai-sdk/google").then((m) => m.google),
+    mistral: () => import("@ai-sdk/mistral").then((m) => m.mistral),
+    openai: () => import("@ai-sdk/openai").then((m) => m.openai),
+};
 let registered;
 /**
- * Registers a model instance from a setup file. The escape hatch for providers
- * the descriptor cannot construct, such as a company LLM gateway.
+ * Registers a model instance from a setup file, taking precedence over the
+ * configured `model` string. The escape hatch for anything the provider
+ * prefixes cannot express, such as a local runtime or a company LLM gateway.
  */
 export function defineEvalModel(model) {
     registered = model;
 }
-const DEFAULT_KEY_ENV = { anthropic: "ANTHROPIC_API_KEY" };
-function requireKey(descriptor) {
-    const name = descriptor.apiKeyEnv ?? DEFAULT_KEY_ENV[descriptor.provider];
-    const key = process.env[name];
-    if (key === undefined || key === "") {
-        throw new Error(`${name} is not set`);
-    }
-    return key;
-}
-export async function resolveModel(descriptor) {
+/**
+ * Turns a `provider/model-id` string into a model instance, importing only the
+ * provider named. Each provider package reads its own conventional API key
+ * variable, so no key configuration is needed. An unrecognised prefix is
+ * returned untouched for the AI SDK's default provider to resolve.
+ */
+export async function resolveModel(spec) {
     if (registered !== undefined) {
         return registered;
     }
-    if (descriptor === undefined) {
-        throw new Error("No model configured. Pass `model` to the evals plugin, or call defineEvalModel() from a setup file.");
+    if (spec === undefined) {
+        throw new Error("No model configured. Set `model` on the evals plugin, or call defineEvalModel() from a setup file.");
+    }
+    const separator = spec.indexOf("/");
+    if (separator === -1) {
+        return spec;
+    }
+    const name = spec.slice(0, separator);
+    const load = PROVIDERS[name];
+    if (load === undefined) {
+        return spec;
     }
-    const { createAnthropic } = await import("@ai-sdk/anthropic");
-    return createAnthropic({ apiKey: requireKey(descriptor) })(descriptor.name);
+    const provider = await load().catch(() => {
+        throw new Error(`The model "${spec}" needs @ai-sdk/${name}@^2. Install it as a dev dependency.`);
+    });
+    return provider(spec.slice(separator + 1));
 }
 //# sourceMappingURL=model.js.map

package/dist/model.js.map

@@ -1 +1 @@
-{"version":3,"file":"model.js","sourceRoot":"","sources":["../src/model.ts"],"names":[],"mappings":"AAGA,IAAI,UAAqC,CAAC;AAE1C;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,KAAoB;IAClD,UAAU,GAAG,KAAK,CAAC;AACrB,CAAC;AAED,MAAM,eAAe,GAAG,EAAE,SAAS,EAAE,mBAAmB,EAAW,CAAC;AAEpE,SAAS,UAAU,CAAC,UAA2B;IAC7C,MAAM,IAAI,GAAG,UAAU,CAAC,SAAS,IAAI,eAAe,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IAC1E,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC9B,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,EAAE,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,aAAa,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,UAAuC;IAEvC,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CACb,qGAAqG,CACtG,CAAC;IACJ,CAAC;IACD,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;IAC9D,OAAO,eAAe,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;AAC9E,CAAC"}
+{"version":3,"file":"model.js","sourceRoot":"","sources":["../src/model.ts"],"names":[],"mappings":"AAIA,MAAM,SAAS,GAAmD;IAChE,SAAS,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IACrE,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IAC5D,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;IAC/D,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;CAC7D,CAAC;AAEF,IAAI,UAAqC,CAAC;AAE1C;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,KAAoB;IAClD,UAAU,GAAG,KAAK,CAAC;AACrB,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAAwB;IAExB,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,oGAAoG,CACrG,CAAC;IACJ,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACpC,IAAI,SAAS,KAAK,CAAC,CAAC,EAAE,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC7B,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE;QACvC,MAAM,IAAI,KAAK,CACb,cAAc,IAAI,mBAAmB,IAAI,sCAAsC,CAChF,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC;AAC7C,CAAC"}

package/dist/server-harness.d.ts

@@ -10,10 +10,9 @@ export interface RunningServer {
  * Binding and reporting both happen in the child, so there is no window in
  * which another process can steal the port.
  */
-export declare function startServer({ cwd, command, env, timeoutMs, }: {
+export declare function startServer({ cwd, command, env, }: {
     cwd: string;
     command: string[];
     env?: Record<string, string>;
-    timeoutMs?: number;
 }): Promise<RunningServer>;
 //# sourceMappingURL=server-harness.d.ts.map

package/dist/server-harness.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"server-harness.d.ts","sourceRoot":"","sources":["../src/server-harness.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,aAAa;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,yEAAyE;IACzE,MAAM,EAAE,MAAM,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAiBD;;;;;GAKG;AACH,wBAAsB,WAAW,CAAC,EAChC,GAAG,EACH,OAAO,EACP,GAAQ,EACR,SAAkB,GACnB,EAAE;IACD,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,OAAO,CAAC,aAAa,CAAC,CA2EzB"}
+{"version":3,"file":"server-harness.d.ts","sourceRoot":"","sources":["../src/server-harness.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,aAAa;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,yEAAyE;IACzE,MAAM,EAAE,MAAM,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAiBD;;;;;GAKG;AACH,wBAAsB,WAAW,CAAC,EAChC,GAAG,EACH,OAAO,EACP,GAAQ,GACT,EAAE;IACD,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC9B,GAAG,OAAO,CAAC,aAAa,CAAC,CA4EzB"}

package/dist/server-harness.js

@@ -15,7 +15,8 @@ function isListeningMessage(message) {
  * Binding and reporting both happen in the child, so there is no window in
  * which another process can steal the port.
  */
-export async function startServer({ cwd, command, env = {}, timeoutMs = 60_000, }) {
+export async function startServer({ cwd, command, env = {}, }) {
+    const timeoutMs = 60_000;
     const [bin, ...args] = command;
     if (bin === undefined) {
         throw new Error("Empty server command");
@@ -53,7 +54,7 @@ export async function startServer({ cwd, command, env = {}, timeoutMs = 60_000,
     };
     const port = await new Promise((resolvePort, reject) => {
         const timer = setTimeout(() => {
-            reject(new Error(`Server did not report its port in ${timeoutMs}ms. It must call \`server.run()\` on a skybridge version that sends the skybridge:listening readiness message.\n${output()}`));
+            reject(new Error(`Server did not report a port in ${timeoutMs}ms. \`command\` must start the server process directly (node, tsx): wrappers such as \`pnpm run\` do not forward the IPC channel the readiness signal uses. It must also run a skybridge version that sends skybridge:listening.\n${output()}`));
         }, timeoutMs);
         child.on("message", (message) => {
             if (isListeningMessage(message)) {

package/dist/server-harness.js.map

@@ -1 +1 @@
-{"version":3,"file":"server-harness.js","sourceRoot":"","sources":["../src/server-harness.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqB,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AASpC,SAAS,SAAS,CAAC,KAAmB;IACpC,OAAO,KAAK,CAAC,QAAQ,KAAK,IAAI,IAAI,KAAK,CAAC,UAAU,KAAK,IAAI,CAAC;AAC9D,CAAC;AAED,SAAS,kBAAkB,CACzB,OAAgB;IAEhB,OAAO,CACL,OAAO,OAAO,KAAK,QAAQ;QAC3B,OAAO,KAAK,IAAI;QACf,OAA8B,CAAC,IAAI,KAAK,qBAAqB;QAC9D,OAAQ,OAA8B,CAAC,IAAI,KAAK,QAAQ,CACzD,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,EAChC,GAAG,EACH,OAAO,EACP,GAAG,GAAG,EAAE,EACR,SAAS,GAAG,MAAM,GAMnB;IACC,MAAM,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC;IAC/B,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAC1C,CAAC;IAED,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAEjE,MAAM,KAAK,GAAG,KAAK,CAAC,UAAU,EAAE,IAAI,EAAE;QACpC,GAAG;QACH,GAAG,EAAE;YACH,GAAG,OAAO,CAAC,GAAG;YACd,QAAQ,EAAE,YAAY;YACtB,GAAG,GAAG;YACN,MAAM,EAAE,GAAG;SACZ;QACD,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC;KACzC,CAAC,CAAC;IAEH,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,MAAM,MAAM,GAAG,GAAG,EAAE,CAAC,QAAQ,CAAC;IAC9B,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;QACjC,QAAQ,IAAI,KAAK,CAAC;IACpB,CAAC,CAAC,CAAC;IACH,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;QACjC,QAAQ,IAAI,KAAK,CAAC;IACpB,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,KAAK,IAAI,EAAE;QACtB,IAAI,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;YACrB,OAAO;QACT,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,OAAO,CAAO,CAAC,WAAW,EAAE,EAAE;YAC/C,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtB,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,CAAC,CAAC;QAC9D,MAAM,MAAM,CAAC;QACb,YAAY,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,MAAM,IAAI,OAAO,CAAS,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE;QAC7D,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,MAAM,CACJ,IAAI,KAAK,CACP,qCAAqC,SAAS,mHAAmH,MAAM,EAAE,EAAE,CAC5K,CACF,CAAC;QACJ,CAAC,EAAE,SAAS,CAAC,CAAC;QACd,KAAK,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE;YAC9B,IAAI,kBAAkB,CAAC,OAAO,CAAC,EAAE,CAAC;gBAChC,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YAC1B,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CACJ,IAAI,KAAK,CAAC,mBAAmB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CACpE,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;YACpB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CACJ,IAAI,KAAK,CACP,2BAA2B,KAAK,CAAC,QAAQ,gCAAgC,MAAM,EAAE,EAAE,CACpF,CACF,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,KAAc,EAAE,EAAE;QAChC,MAAM,IAAI,EAAE,CAAC;QACb,MAAM,KAAK,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,GAAG,EAAE,oBAAoB,IAAI,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;AAC/D,CAAC"}
+{"version":3,"file":"server-harness.js","sourceRoot":"","sources":["../src/server-harness.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqB,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AASpC,SAAS,SAAS,CAAC,KAAmB;IACpC,OAAO,KAAK,CAAC,QAAQ,KAAK,IAAI,IAAI,KAAK,CAAC,UAAU,KAAK,IAAI,CAAC;AAC9D,CAAC;AAED,SAAS,kBAAkB,CACzB,OAAgB;IAEhB,OAAO,CACL,OAAO,OAAO,KAAK,QAAQ;QAC3B,OAAO,KAAK,IAAI;QACf,OAA8B,CAAC,IAAI,KAAK,qBAAqB;QAC9D,OAAQ,OAA8B,CAAC,IAAI,KAAK,QAAQ,CACzD,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,EAChC,GAAG,EACH,OAAO,EACP,GAAG,GAAG,EAAE,GAKT;IACC,MAAM,SAAS,GAAG,MAAM,CAAC;IACzB,MAAM,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC;IAC/B,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAC1C,CAAC;IAED,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAEjE,MAAM,KAAK,GAAG,KAAK,CAAC,UAAU,EAAE,IAAI,EAAE;QACpC,GAAG;QACH,GAAG,EAAE;YACH,GAAG,OAAO,CAAC,GAAG;YACd,QAAQ,EAAE,YAAY;YACtB,GAAG,GAAG;YACN,MAAM,EAAE,GAAG;SACZ;QACD,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC;KACzC,CAAC,CAAC;IAEH,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,MAAM,MAAM,GAAG,GAAG,EAAE,CAAC,QAAQ,CAAC;IAC9B,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;QACjC,QAAQ,IAAI,KAAK,CAAC;IACpB,CAAC,CAAC,CAAC;IACH,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;QACjC,QAAQ,IAAI,KAAK,CAAC;IACpB,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,KAAK,IAAI,EAAE;QACtB,IAAI,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;YACrB,OAAO;QACT,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,OAAO,CAAO,CAAC,WAAW,EAAE,EAAE;YAC/C,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtB,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,CAAC,CAAC;QAC9D,MAAM,MAAM,CAAC;QACb,YAAY,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,MAAM,IAAI,OAAO,CAAS,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE;QAC7D,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,MAAM,CACJ,IAAI,KAAK,CACP,mCAAmC,SAAS,qOAAqO,MAAM,EAAE,EAAE,CAC5R,CACF,CAAC;QACJ,CAAC,EAAE,SAAS,CAAC,CAAC;QACd,KAAK,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE;YAC9B,IAAI,kBAAkB,CAAC,OAAO,CAAC,EAAE,CAAC;gBAChC,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YAC1B,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CACJ,IAAI,KAAK,CAAC,mBAAmB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CACpE,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;YACpB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CACJ,IAAI,KAAK,CACP,2BAA2B,KAAK,CAAC,QAAQ,gCAAgC,MAAM,EAAE,EAAE,CACpF,CACF,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,KAAc,EAAE,EAAE;QAChC,MAAM,IAAI,EAAE,CAAC;QACb,MAAM,KAAK,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,GAAG,EAAE,oBAAoB,IAAI,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;AAC/D,CAAC"}

package/dist/session-registry.d.ts

@@ -1,8 +1,9 @@
 import { Chat } from "./chat.js";
 /**
  * Opens a fresh MCP session and conversation against the server the plugin
- * started. Every chat opened during a test is closed by the plugin's
- * `afterEach`, so scenarios never leak sessions and never write teardown.
+ * started. The session is closed when the current test finishes, so scenarios
+ * never leak sessions and never write teardown, and tests that run
+ * concurrently cannot close each other's sessions.
  *
  * The type parameter pins the assertions to the project's registry:
  * `start<AppType>()` returns a `Chat<AppType>`, and `expect.chat` infers the
@@ -12,5 +13,4 @@ export declare function start<App>(overrides?: {
     systemPrompt?: string;
     temperature?: number;
 }): Promise<Chat<App>>;
-export declare function closeOpenedSessions(): Promise<void>;
 //# sourceMappingURL=session-registry.d.ts.map

package/dist/session-registry.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAKjC;;;;;;;;GAQG;AACH,wBAAsB,KAAK,CAAC,GAAG,EAAE,SAAS,CAAC,EAAE;IAC3C,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAUrB;AAED,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,IAAI,CAAC,CAEzD"}
+{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC;;;;;;;;;GASG;AACH,wBAAsB,KAAK,CAAC,GAAG,EAAE,SAAS,CAAC,EAAE;IAC3C,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAUrB"}

package/dist/session-registry.js

@@ -1,11 +1,11 @@
-import { inject } from "vitest";
+import { inject, onTestFinished } from "vitest";
 import { Chat } from "./chat.js";
 import { resolveModel } from "./model.js";
-const opened = [];
 /**
  * Opens a fresh MCP session and conversation against the server the plugin
- * started. Every chat opened during a test is closed by the plugin's
- * `afterEach`, so scenarios never leak sessions and never write teardown.
+ * started. The session is closed when the current test finishes, so scenarios
+ * never leak sessions and never write teardown, and tests that run
+ * concurrently cannot close each other's sessions.
  *
  * The type parameter pins the assertions to the project's registry:
  * `start<AppType>()` returns a `Chat<AppType>`, and `expect.chat` infers the
@@ -19,10 +19,7 @@ export async function start(overrides) {
         systemPrompt: overrides?.systemPrompt ?? config.systemPrompt,
         maxSteps: config.maxSteps,
     });
-    opened.push(chat);
+    onTestFinished(() => chat.close());
     return chat;
 }
-export async function closeOpenedSessions() {
-    await Promise.all(opened.splice(0).map((chat) => chat.close()));
-}
 //# sourceMappingURL=session-registry.js.map

package/dist/session-registry.js.map

@@ -1 +1 @@
-{"version":3,"file":"session-registry.js","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE1C,MAAM,MAAM,GAAoB,EAAE,CAAC;AAEnC;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CAAM,SAGhC;IACC,MAAM,MAAM,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAM,MAAM,CAAC,mBAAmB,CAAC,EAAE;QAC7D,KAAK,EAAE,MAAM,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC;QACvC,WAAW,EAAE,SAAS,EAAE,WAAW,IAAI,MAAM,CAAC,WAAW;QACzD,YAAY,EAAE,SAAS,EAAE,YAAY,IAAI,MAAM,CAAC,YAAY;QAC5D,QAAQ,EAAE,MAAM,CAAC,QAAQ;KAC1B,CAAC,CAAC;IACH,MAAM,CAAC,IAAI,CAAC,IAAqB,CAAC,CAAC;IACnC,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB;IACvC,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAClE,CAAC"}
+{"version":3,"file":"session-registry.js","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE1C;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CAAM,SAGhC;IACC,MAAM,MAAM,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAM,MAAM,CAAC,mBAAmB,CAAC,EAAE;QAC7D,KAAK,EAAE,MAAM,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC;QACvC,WAAW,EAAE,SAAS,EAAE,WAAW,IAAI,MAAM,CAAC,WAAW;QACzD,YAAY,EAAE,SAAS,EAAE,YAAY,IAAI,MAAM,CAAC,YAAY;QAC5D,QAAQ,EAAE,MAAM,CAAC,QAAQ;KAC1B,CAAC,CAAC;IACH,cAAc,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;IACnC,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/vitest-config.d.ts

@@ -0,0 +1,9 @@
+import type { ViteUserConfig } from "vitest/config";
+import type { EvalsOptions } from "./config.js";
+/**
+ * The vitest settings the eval runner needs, merged in by the `skybridge()`
+ * Vite plugin. Returned as plain config rather than a plugin so the paths stay
+ * resolved against this package, which owns the files they point at.
+ */
+export declare function evalsConfig(options: EvalsOptions): ViteUserConfig["test"];
+//# sourceMappingURL=vitest-config.d.ts.map

package/dist/vitest-config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vitest-config.d.ts","sourceRoot":"","sources":["../src/vitest-config.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAMhD;;;;GAIG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,YAAY,GAAG,cAAc,CAAC,MAAM,CAAC,CAOzE"}

package/dist/vitest-config.js

@@ -0,0 +1,18 @@
+import { fileURLToPath } from "node:url";
+function here(file) {
+    return fileURLToPath(new URL(file, import.meta.url));
+}
+/**
+ * The vitest settings the eval runner needs, merged in by the `skybridge()`
+ * Vite plugin. Returned as plain config rather than a plugin so the paths stay
+ * resolved against this package, which owns the files they point at.
+ */
+export function evalsConfig(options) {
+    return {
+        testTimeout: 120_000,
+        setupFiles: [here("./matchers.js")],
+        globalSetup: [here("./global-setup.js")],
+        provide: { skybridgeEvals: options },
+    };
+}
+//# sourceMappingURL=vitest-config.js.map

package/dist/vitest-config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vitest-config.js","sourceRoot":"","sources":["../src/vitest-config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAIzC,SAAS,IAAI,CAAC,IAAY;IACxB,OAAO,aAAa,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AACvD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAAC,OAAqB;IAC/C,OAAO;QACL,WAAW,EAAE,OAAO;QACpB,UAAU,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACnC,WAAW,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QACxC,OAAO,EAAE,EAAE,cAAc,EAAE,OAAO,EAAE;KACrC,CAAC;AACJ,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@skybridge/test",
-  "version": "0.0.0-dev.7aada9d",
+  "version": "0.0.0-dev.dde647a",
   "description": "Eval runner for Skybridge MCP apps: does a prompt reach the right tool with the right arguments",
   "license": "ISC",
   "type": "module",
@@ -13,34 +13,57 @@
       "types": "./dist/index.d.ts",
       "default": "./dist/index.js"
     },
-    "./plugin": {
-      "types": "./dist/plugin.d.ts",
-      "default": "./dist/plugin.js"
+    "./vitest-config": {
+      "types": "./dist/vitest-config.d.ts",
+      "default": "./dist/vitest-config.js"
     }
   },
   "files": [
     "dist"
   ],
   "dependencies": {
-    "@ai-sdk/anthropic": "^2.0.0",
     "@modelcontextprotocol/sdk": "^1.29.0",
     "ai": "^5.0.0",
-    "ci-info": "^4.4.0",
-    "posthog-node": "^5.45.2"
+    "@skybridge/vite-plugin": "0.0.0-dev.dde647a"
   },
   "peerDependencies": {
-    "vitest": "^4.1.0",
-    "skybridge": "^0.0.0-dev.7aada9d"
+    "@ai-sdk/anthropic": "^2.0.0",
+    "@ai-sdk/google": "^2.0.0",
+    "@ai-sdk/mistral": "^2.0.0",
+    "@ai-sdk/openai": "^2.0.0",
+    "skybridge": "^2.0.0",
+    "vitest": "^4.1.0"
+  },
+  "peerDependenciesMeta": {
+    "@ai-sdk/anthropic": {
+      "optional": true
+    },
+    "@ai-sdk/google": {
+      "optional": true
+    },
+    "@ai-sdk/mistral": {
+      "optional": true
+    },
+    "@ai-sdk/openai": {
+      "optional": true
+    }
   },
   "devDependencies": {
+    "@ai-sdk/anthropic": "^2.0.0",
+    "@ai-sdk/google": "^2.0.0",
+    "@ai-sdk/mistral": "^2.0.0",
+    "@ai-sdk/openai": "^2.0.0",
     "@types/node": "^24.13.3",
     "typescript": "^5.9.3",
     "vitest": "^4.1.10",
-    "skybridge": "0.0.0-dev.7aada9d"
+    "skybridge": "0.0.0-dev.dde647a"
   },
   "engines": {
     "node": ">=22.23.1"
   },
+  "publishConfig": {
+    "tag": "beta"
+  },
   "scripts": {
     "build": "tsc -p tsconfig.json",
     "format": "biome check --write --error-on-warnings",

package/dist/plugin.d.ts

@@ -1,10 +0,0 @@
-import type { Plugin } from "vitest/config";
-import type { EvalsOptions } from "./config.js";
-/**
- * Wires the eval runner into vitest. Usually reached through the `evals`
- * option of the `skybridge()` Vite plugin rather than registered directly.
- * Everything the runner needs is configured here, so a scenario file imports
- * nothing but `start` and its own `AppType`.
- */
-export declare function evals(options: EvalsOptions): Plugin;
-//# sourceMappingURL=plugin.d.ts.map

package/dist/plugin.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"plugin.d.ts","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,KAAK,EAAE,YAAY,EAAwB,MAAM,aAAa,CAAC;AAMtE;;;;;GAKG;AACH,wBAAgB,KAAK,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM,CAuBnD"}

package/dist/plugin.js

@@ -1,32 +0,0 @@
-import { fileURLToPath } from "node:url";
-function here(file) {
-    return fileURLToPath(new URL(file, import.meta.url));
-}
-/**
- * Wires the eval runner into vitest. Usually reached through the `evals`
- * option of the `skybridge()` Vite plugin rather than registered directly.
- * Everything the runner needs is configured here, so a scenario file imports
- * nothing but `start` and its own `AppType`.
- */
-export function evals(options) {
-    const { model, ...rest } = options;
-    const provided = "file" in model ? rest : { ...rest, model };
-    const setupFiles = [here("./setup.js")];
-    if ("file" in model) {
-        setupFiles.push(model.file);
-    }
-    return {
-        name: "skybridge-evals",
-        config() {
-            return {
-                test: {
-                    testTimeout: 120_000,
-                    setupFiles,
-                    globalSetup: [here("./global-setup.js")],
-                    provide: { skybridgeEvals: provided },
-                },
-            };
-        },
-    };
-}
-//# sourceMappingURL=plugin.js.map

package/dist/plugin.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"plugin.js","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAIzC,SAAS,IAAI,CAAC,IAAY;IACxB,OAAO,aAAa,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AACvD,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,KAAK,CAAC,OAAqB;IACzC,MAAM,EAAE,KAAK,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;IACnC,MAAM,QAAQ,GACZ,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,CAAC;IAE9C,MAAM,UAAU,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;IACxC,IAAI,MAAM,IAAI,KAAK,EAAE,CAAC;QACpB,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAED,OAAO;QACL,IAAI,EAAE,iBAAiB;QACvB,MAAM;YACJ,OAAO;gBACL,IAAI,EAAE;oBACJ,WAAW,EAAE,OAAO;oBACpB,UAAU;oBACV,WAAW,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;oBACxC,OAAO,EAAE,EAAE,cAAc,EAAE,QAAQ,EAAE;iBACtC;aACF,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/setup.d.ts

@@ -1,2 +0,0 @@
-import "./matchers.js";
-//# sourceMappingURL=setup.d.ts.map

package/dist/setup.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"setup.d.ts","sourceRoot":"","sources":["../src/setup.ts"],"names":[],"mappings":"AACA,OAAO,eAAe,CAAC"}

package/dist/setup.js

@@ -1,7 +0,0 @@
-import { afterEach } from "vitest";
-import "./matchers.js";
-import { closeOpenedSessions } from "./session-registry.js";
-afterEach(async () => {
-    await closeOpenedSessions();
-});
-//# sourceMappingURL=setup.js.map

package/dist/setup.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"setup.js","sourceRoot":"","sources":["../src/setup.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,eAAe,CAAC;AACvB,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAE5D,SAAS,CAAC,KAAK,IAAI,EAAE;IACnB,MAAM,mBAAmB,EAAE,CAAC;AAC9B,CAAC,CAAC,CAAC"}

package/dist/telemetry.d.ts

@@ -1,10 +0,0 @@
-/**
- * One event per eval run, honoring the same opt-outs, machine id file, and
- * env-var semantics as the skybridge CLI. Silently a no-op when telemetry is
- * disabled.
- */
-export declare function captureEvalRun(properties: {
-    durationMs: number;
-    serverStarted: boolean;
-}): Promise<void>;
-//# sourceMappingURL=telemetry.d.ts.map

package/dist/telemetry.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../src/telemetry.ts"],"names":[],"mappings":"AAyDA;;;;GAIG;AACH,wBAAsB,cAAc,CAAC,UAAU,EAAE;IAC/C,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,OAAO,CAAC;CACxB,GAAG,OAAO,CAAC,IAAI,CAAC,CA8BhB"}

package/dist/telemetry.js

@@ -1,77 +0,0 @@
-import { randomUUID } from "node:crypto";
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
-import { homedir } from "node:os";
-import { join } from "node:path";
-import ci from "ci-info";
-import { PostHog } from "posthog-node";
-const POSTHOG_API_KEY = "phc_rQdkCYr0DO4NcZBQXZnUwsHAbau9zuNwKIpil9FQP6v";
-const POSTHOG_HOST = "https://us.i.posthog.com";
-const GLOBAL_CONFIG_DIR = join(homedir(), ".skybridge");
-const GLOBAL_CONFIG_FILE = join(GLOBAL_CONFIG_DIR, "config.json");
-function isFlagSet(value) {
-    return value === "1" || value === "true";
-}
-function readGlobalConfig() {
-    try {
-        if (existsSync(GLOBAL_CONFIG_FILE)) {
-            return JSON.parse(readFileSync(GLOBAL_CONFIG_FILE, "utf-8"));
-        }
-    }
-    catch {
-        return null;
-    }
-    return null;
-}
-function getMachineId() {
-    const config = readGlobalConfig();
-    if (config?.telemetry?.enabled === false) {
-        return undefined;
-    }
-    if (config?.machineId !== undefined) {
-        return config.machineId;
-    }
-    try {
-        const machineId = randomUUID();
-        mkdirSync(GLOBAL_CONFIG_DIR, { recursive: true });
-        writeFileSync(GLOBAL_CONFIG_FILE, JSON.stringify({ ...config, machineId, telemetry: { enabled: true } }, null, 2));
-        return machineId;
-    }
-    catch {
-        return undefined;
-    }
-}
-/**
- * One event per eval run, honoring the same opt-outs, machine id file, and
- * env-var semantics as the skybridge CLI. Silently a no-op when telemetry is
- * disabled.
- */
-export async function captureEvalRun(properties) {
-    if (isFlagSet(process.env.SKYBRIDGE_TELEMETRY_DISABLED) ||
-        isFlagSet(process.env.DO_NOT_TRACK)) {
-        return;
-    }
-    const machineId = getMachineId();
-    if (machineId === undefined) {
-        return;
-    }
-    try {
-        const client = new PostHog(POSTHOG_API_KEY, {
-            host: POSTHOG_HOST,
-            flushAt: 1,
-            flushInterval: 0,
-        });
-        client.capture({
-            distinctId: machineId,
-            event: "evals run",
-            properties: { ...properties, isCI: ci.isCI },
-        });
-        await Promise.race([
-            client.shutdown(),
-            new Promise((resolve) => setTimeout(resolve, 2_000)),
-        ]);
-    }
-    catch {
-        return;
-    }
-}
-//# sourceMappingURL=telemetry.js.map

package/dist/telemetry.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"telemetry.js","sourceRoot":"","sources":["../src/telemetry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,MAAM,eAAe,GAAG,iDAAiD,CAAC;AAC1E,MAAM,YAAY,GAAG,0BAA0B,CAAC;AAChD,MAAM,iBAAiB,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,YAAY,CAAC,CAAC;AACxD,MAAM,kBAAkB,GAAG,IAAI,CAAC,iBAAiB,EAAE,aAAa,CAAC,CAAC;AAOlE,SAAS,SAAS,CAAC,KAAyB;IAC1C,OAAO,KAAK,KAAK,GAAG,IAAI,KAAK,KAAK,MAAM,CAAC;AAC3C,CAAC;AAED,SAAS,gBAAgB;IACvB,IAAI,CAAC;QACH,IAAI,UAAU,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,YAAY;IACnB,MAAM,MAAM,GAAG,gBAAgB,EAAE,CAAC;IAClC,IAAI,MAAM,EAAE,SAAS,EAAE,OAAO,KAAK,KAAK,EAAE,CAAC;QACzC,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,MAAM,EAAE,SAAS,KAAK,SAAS,EAAE,CAAC;QACpC,OAAO,MAAM,CAAC,SAAS,CAAC;IAC1B,CAAC;IACD,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,UAAU,EAAE,CAAC;QAC/B,SAAS,CAAC,iBAAiB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAClD,aAAa,CACX,kBAAkB,EAClB,IAAI,CAAC,SAAS,CACZ,EAAE,GAAG,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EACtD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;QACF,OAAO,SAAS,CAAC;IACnB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,UAGpC;IACC,IACE,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC;QACnD,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,EACnC,CAAC;QACD,OAAO;IACT,CAAC;IACD,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;IACjC,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QAC5B,OAAO;IACT,CAAC;IAED,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,OAAO,CAAC,eAAe,EAAE;YAC1C,IAAI,EAAE,YAAY;YAClB,OAAO,EAAE,CAAC;YACV,aAAa,EAAE,CAAC;SACjB,CAAC,CAAC;QACH,MAAM,CAAC,OAAO,CAAC;YACb,UAAU,EAAE,SAAS;YACrB,KAAK,EAAE,WAAW;YAClB,UAAU,EAAE,EAAE,GAAG,UAAU,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE;SAC7C,CAAC,CAAC;QACH,MAAM,OAAO,CAAC,IAAI,CAAC;YACjB,MAAM,CAAC,QAAQ,EAAE;YACjB,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;SACrD,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;IACT,CAAC;AACH,CAAC"}