@skybridge/test 0.0.0-dev.9a3a28d → 0.0.0-dev.b9466d5

package/README.md

@@ -4,19 +4,19 @@ Evals for [Skybridge](https://skybridge.tech) MCP apps: they answer whether a
 prompt reaches the right tool with the right arguments, which no unit test can
 check.
 
-**Beta.** The API is still settling, so it publishes under the `beta`
-dist-tag and may change between releases.
+**Beta.** The API is still settling and may change between releases.
 
 Add the `evals` option to the Skybridge Vite plugin, then write scenarios with
 vitest's own `it` and `expect`.
 
 ```ts
+import { anthropic } from "@ai-sdk/anthropic";
 import { expect, it } from "vitest";
 import { start } from "@skybridge/test";
 import type { AppType } from "../src/server.js";
 
 it("finds products by category", async () => {
-  const chat = await start<AppType>();
+  const chat = await start<AppType>({ model: anthropic("claude-sonnet-4-5") });
   await chat.send("I am looking for ski goggles");
 
   expect.chat(chat).toHaveCalledToolWith("search-products", {
@@ -27,64 +27,39 @@ it("finds products by category", async () => {
 
 ## Choosing a model
 
-`model` is a `provider/model-id` string. Only the provider you name is
-imported, so install just that one as a dev dependency, and the key is read
-from its own environment variable.
-
-| prefix | package | key |
-| --- | --- | --- |
-| `anthropic` | `@ai-sdk/anthropic` | `ANTHROPIC_API_KEY` |
-| `openai` | `@ai-sdk/openai` | `OPENAI_API_KEY` |
-| `mistral` | `@ai-sdk/mistral` | `MISTRAL_API_KEY` |
-| `google` | `@ai-sdk/google` | `GOOGLE_GENERATIVE_AI_API_KEY` |
+`start` takes any AI SDK model instance, so the provider and its credentials
+are the project's own. Install the provider package you want alongside `ai`.
 
 ```ts
-skybridge({
-  evals: {
-    model: "anthropic/claude-sonnet-4-5",
-    project: { cwd: ".", command: ["node", "dist/server.js"] },
-  },
-});
-```
-
-`command` has to start the server process directly, since wrappers such as
-`pnpm run` do not forward the channel the runner uses to learn the port.
-
-Any other prefix is passed to the AI SDK untouched, so a string such as
-`openai/gpt-5` also resolves through the Vercel AI Gateway if you have
-`AI_GATEWAY_API_KEY` set and no provider package installed.
+import { anthropic } from "@ai-sdk/anthropic";
+import { start } from "@skybridge/test";
 
-## Models the prefixes cannot reach
+const chat = await start<AppType>({ model: anthropic("claude-sonnet-4-5") });
+```
 
-For a local runtime, a company gateway, or a model that needs middleware,
-build the model yourself and register it from a vitest setup file. It takes
-precedence over `model`, which can then be left out.
+A local runtime or a company gateway works the same way, through the AI SDK's
+own `createOpenAICompatible` or `customProvider`.
 
 ```ts
-// eval-model.ts
 import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
-import { defineEvalModel } from "@skybridge/test";
-
-defineEvalModel(
-  createOpenAICompatible({ baseURL: "http://localhost:11434/v1" })("llama3.1"),
-);
-```
 
-```ts
-// vitest.config.ts
-export default defineConfig({
-  plugins: [skybridge({ evals: { project: { ... } } })],
-  test: { setupFiles: ["./eval-model.ts"] },
+const ollama = createOpenAICompatible({
+  name: "ollama",
+  baseURL: "http://localhost:11434/v1",
 });
+const chat = await start<AppType>({ model: ollama("llama3.1") });
 ```
 
-Tool calls the server rejected do not count towards a positive assertion, so a
-scenario cannot pass on a call that errored. To assert an error on purpose, use
-`toHaveFailedToolCall`.
+## Booting the server
 
 ```ts
-expect.chat(chat).toHaveFailedToolCall("create-checkout");
+skybridge({
+  evals: { project: { cwd: ".", command: ["node", "dist/server.js"] } },
+});
 ```
 
+`command` has to start the server process directly, since wrappers such as
+`pnpm run` do not forward the channel the runner uses to learn the port.
+
 See the [documentation](https://docs.skybridge.tech) for setup and the full
 matcher list.

package/dist/index.d.ts

@@ -1,5 +1,4 @@
 export type { ChatAssertion, ChatMatchers, } from "@skybridge/vite-plugin/evals";
 export type { Chat } from "./chat.js";
-export { defineEvalModel } from "./model.js";
 export { start } from "./session-registry.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,aAAa,EACb,YAAY,GACb,MAAM,8BAA8B,CAAC;AACtC,YAAY,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,aAAa,EACb,YAAY,GACb,MAAM,8BAA8B,CAAC;AACtC,YAAY,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}

package/dist/index.js

@@ -1,3 +1,2 @@
-export { defineEvalModel } from "./model.js";
 export { start } from "./session-registry.js";
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}

package/dist/session-registry.d.ts

@@ -1,3 +1,4 @@
+import type { LanguageModel } from "ai";
 import { Chat } from "./chat.js";
 /**
  * Opens a fresh MCP session and conversation against the server the plugin
@@ -9,8 +10,11 @@ import { Chat } from "./chat.js";
  * `start<AppType>()` returns a `Chat<AppType>`, and `expect.chat` infers the
  * tool names and argument shapes from it.
  */
-export declare function start<App>(overrides?: {
+export declare function start<App>(options: {
+    /** Any AI SDK model instance, built by the project. */
+    model: LanguageModel;
     systemPrompt?: string;
     temperature?: number;
+    maxSteps?: number;
 }): Promise<Chat<App>>;
 //# sourceMappingURL=session-registry.d.ts.map

package/dist/session-registry.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC;;;;;;;;;GASG;AACH,wBAAsB,KAAK,CAAC,GAAG,EAAE,SAAS,CAAC,EAAE;IAC3C,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAgBrB"}
+{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAExC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC;;;;;;;;;GASG;AACH,wBAAsB,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE;IACxC,uDAAuD;IACvD,KAAK,EAAE,aAAa,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAgBrB"}

package/dist/session-registry.js

@@ -1,6 +1,5 @@
 import { inject, onTestFinished } from "vitest";
 import { Chat } from "./chat.js";
-import { resolveModel } from "./model.js";
 /**
  * Opens a fresh MCP session and conversation against the server the plugin
  * started. The session is closed when the current test finishes, so scenarios
@@ -11,17 +10,17 @@ import { resolveModel } from "./model.js";
  * `start<AppType>()` returns a `Chat<AppType>`, and `expect.chat` infers the
  * tool names and argument shapes from it.
  */
-export async function start(overrides) {
-    const config = inject("skybridgeEvals");
+export async function start(options) {
     const url = inject("skybridgeEvalsUrl");
-    if (config === undefined || url === undefined) {
+    if (url === undefined) {
         throw new Error("No eval server is running. Add `skybridge({ evals: {...} })` to the vitest config.");
     }
+    const config = inject("skybridgeEvals");
     const chat = await Chat.open(url, {
-        model: await resolveModel(config.model),
-        temperature: overrides?.temperature ?? config.temperature,
-        systemPrompt: overrides?.systemPrompt ?? config.systemPrompt,
-        maxSteps: config.maxSteps,
+        model: options.model,
+        temperature: options.temperature ?? config?.temperature,
+        systemPrompt: options.systemPrompt ?? config?.systemPrompt,
+        maxSteps: options.maxSteps ?? config?.maxSteps,
     });
     onTestFinished(() => chat.close());
     return chat;

package/dist/session-registry.js.map

@@ -1 +1 @@
-{"version":3,"file":"session-registry.js","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE1C;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CAAM,SAGhC;IACC,MAAM,MAAM,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;IACxC,MAAM,GAAG,GAAG,MAAM,CAAC,mBAAmB,CAAC,CAAC;IACxC,IAAI,MAAM,KAAK,SAAS,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CACb,oFAAoF,CACrF,CAAC;IACJ,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAM,GAAG,EAAE;QACrC,KAAK,EAAE,MAAM,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC;QACvC,WAAW,EAAE,SAAS,EAAE,WAAW,IAAI,MAAM,CAAC,WAAW;QACzD,YAAY,EAAE,SAAS,EAAE,YAAY,IAAI,MAAM,CAAC,YAAY;QAC5D,QAAQ,EAAE,MAAM,CAAC,QAAQ;KAC1B,CAAC,CAAC;IACH,cAAc,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;IACnC,OAAO,IAAI,CAAC;AACd,CAAC"}
+{"version":3,"file":"session-registry.js","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CAAM,OAMhC;IACC,MAAM,GAAG,GAAG,MAAM,CAAC,mBAAmB,CAAC,CAAC;IACxC,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CACb,oFAAoF,CACrF,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAM,GAAG,EAAE;QACrC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,MAAM,EAAE,WAAW;QACvD,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,MAAM,EAAE,YAAY;QAC1D,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,MAAM,EAAE,QAAQ;KAC/C,CAAC,CAAC;IACH,cAAc,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;IACnC,OAAO,IAAI,CAAC;AACd,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@skybridge/test",
-  "version": "0.0.0-dev.9a3a28d",
+  "version": "0.0.0-dev.b9466d5",
   "description": "Eval runner for Skybridge MCP apps: does a prompt reach the right tool with the right arguments",
   "license": "ISC",
   "type": "module",
@@ -12,10 +12,6 @@
     ".": {
       "types": "./dist/index.d.ts",
       "default": "./dist/index.js"
-    },
-    "./vitest-config": {
-      "types": "./dist/vitest-config.d.ts",
-      "default": "./dist/vitest-config.js"
     }
   },
   "files": [
@@ -23,47 +19,23 @@
   ],
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.29.0",
-    "ai": "^5.0.0",
-    "@skybridge/vite-plugin": "0.0.0-dev.9a3a28d"
+    "@skybridge/vite-plugin": "0.0.0-dev.b9466d5"
   },
   "peerDependencies": {
-    "@ai-sdk/anthropic": "^2.0.0",
-    "@ai-sdk/google": "^2.0.0",
-    "@ai-sdk/mistral": "^2.0.0",
-    "@ai-sdk/openai": "^2.0.0",
+    "ai": "^5.0.0",
     "skybridge": "^2.0.0",
     "vitest": "^4.1.0"
   },
-  "peerDependenciesMeta": {
-    "@ai-sdk/anthropic": {
-      "optional": true
-    },
-    "@ai-sdk/google": {
-      "optional": true
-    },
-    "@ai-sdk/mistral": {
-      "optional": true
-    },
-    "@ai-sdk/openai": {
-      "optional": true
-    }
-  },
   "devDependencies": {
-    "@ai-sdk/anthropic": "^2.0.0",
-    "@ai-sdk/google": "^2.0.0",
-    "@ai-sdk/mistral": "^2.0.0",
-    "@ai-sdk/openai": "^2.0.0",
     "@types/node": "^24.13.3",
+    "ai": "^5.0.0",
     "typescript": "^5.9.3",
     "vitest": "^4.1.10",
-    "skybridge": "0.0.0-dev.9a3a28d"
+    "skybridge": "0.0.0-dev.b9466d5"
   },
   "engines": {
     "node": ">=22.23.1"
   },
-  "publishConfig": {
-    "tag": "beta"
-  },
   "scripts": {
     "build": "tsc -p tsconfig.json",
     "format": "biome check --write --error-on-warnings",

package/dist/model.d.ts

@@ -1,15 +0,0 @@
-import type { LanguageModel } from "ai";
-/**
- * Registers a model instance from a setup file, taking precedence over the
- * configured `model` string. The escape hatch for anything the provider
- * prefixes cannot express, such as a local runtime or a company LLM gateway.
- */
-export declare function defineEvalModel(model: LanguageModel): void;
-/**
- * Turns a `provider/model-id` string into a model instance, importing only the
- * provider named. Each provider package reads its own conventional API key
- * variable, so no key configuration is needed. An unrecognised prefix is
- * returned untouched for the AI SDK's default provider to resolve.
- */
-export declare function resolveModel(spec: string | undefined): Promise<LanguageModel>;
-//# sourceMappingURL=model.d.ts.map

package/dist/model.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"model.d.ts","sourceRoot":"","sources":["../src/model.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAaxC;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAE1D;AAED;;;;;GAKG;AACH,wBAAsB,YAAY,CAChC,IAAI,EAAE,MAAM,GAAG,SAAS,GACvB,OAAO,CAAC,aAAa,CAAC,CAwBxB"}

package/dist/model.js

@@ -1,43 +0,0 @@
-const PROVIDERS = {
-    anthropic: () => import("@ai-sdk/anthropic").then((m) => m.anthropic),
-    google: () => import("@ai-sdk/google").then((m) => m.google),
-    mistral: () => import("@ai-sdk/mistral").then((m) => m.mistral),
-    openai: () => import("@ai-sdk/openai").then((m) => m.openai),
-};
-let registered;
-/**
- * Registers a model instance from a setup file, taking precedence over the
- * configured `model` string. The escape hatch for anything the provider
- * prefixes cannot express, such as a local runtime or a company LLM gateway.
- */
-export function defineEvalModel(model) {
-    registered = model;
-}
-/**
- * Turns a `provider/model-id` string into a model instance, importing only the
- * provider named. Each provider package reads its own conventional API key
- * variable, so no key configuration is needed. An unrecognised prefix is
- * returned untouched for the AI SDK's default provider to resolve.
- */
-export async function resolveModel(spec) {
-    if (registered !== undefined) {
-        return registered;
-    }
-    if (spec === undefined) {
-        throw new Error("No model configured. Set `model` on the evals plugin, or call defineEvalModel() from a setup file.");
-    }
-    const separator = spec.indexOf("/");
-    if (separator === -1) {
-        return spec;
-    }
-    const name = spec.slice(0, separator);
-    const load = PROVIDERS[name];
-    if (load === undefined) {
-        return spec;
-    }
-    const provider = await load().catch(() => {
-        throw new Error(`The model "${spec}" needs @ai-sdk/${name}@^2. Install it as a dev dependency.`);
-    });
-    return provider(spec.slice(separator + 1));
-}
-//# sourceMappingURL=model.js.map

package/dist/model.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"model.js","sourceRoot":"","sources":["../src/model.ts"],"names":[],"mappings":"AAIA,MAAM,SAAS,GAAmD;IAChE,SAAS,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IACrE,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IAC5D,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;IAC/D,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;CAC7D,CAAC;AAEF,IAAI,UAAqC,CAAC;AAE1C;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,KAAoB;IAClD,UAAU,GAAG,KAAK,CAAC;AACrB,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAAwB;IAExB,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,oGAAoG,CACrG,CAAC;IACJ,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACpC,IAAI,SAAS,KAAK,CAAC,CAAC,EAAE,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC7B,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE;QACvC,MAAM,IAAI,KAAK,CACb,cAAc,IAAI,mBAAmB,IAAI,sCAAsC,CAChF,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC;AAC7C,CAAC"}