@skybridge/test 0.0.0-dev.874c2eb → 0.0.0-dev.88f1889

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/README.md +56 -42
  2. package/dist/chat.d.ts +26 -23
  3. package/dist/chat.d.ts.map +1 -1
  4. package/dist/chat.js +70 -19
  5. package/dist/chat.js.map +1 -1
  6. package/dist/index.d.ts +2 -2
  7. package/dist/index.d.ts.map +1 -1
  8. package/dist/index.js +0 -1
  9. package/dist/index.js.map +1 -1
  10. package/dist/matchers/index.d.ts +4 -0
  11. package/dist/matchers/index.d.ts.map +1 -0
  12. package/dist/matchers/index.js +2 -0
  13. package/dist/matchers/index.js.map +1 -0
  14. package/dist/matchers/matchers.d.ts +51 -0
  15. package/dist/matchers/matchers.d.ts.map +1 -0
  16. package/dist/matchers/matchers.js +217 -0
  17. package/dist/matchers/matchers.js.map +1 -0
  18. package/dist/matchers/types.d.ts +44 -0
  19. package/dist/matchers/types.d.ts.map +1 -0
  20. package/dist/matchers/types.js +2 -0
  21. package/dist/matchers/types.js.map +1 -0
  22. package/dist/session-registry.d.ts +64 -9
  23. package/dist/session-registry.d.ts.map +1 -1
  24. package/dist/session-registry.js +38 -16
  25. package/dist/session-registry.js.map +1 -1
  26. package/package.json +13 -34
  27. package/dist/config.d.ts +0 -9
  28. package/dist/config.d.ts.map +0 -1
  29. package/dist/config.js +0 -2
  30. package/dist/config.js.map +0 -1
  31. package/dist/global-setup.d.ts +0 -3
  32. package/dist/global-setup.d.ts.map +0 -1
  33. package/dist/global-setup.js +0 -24
  34. package/dist/global-setup.js.map +0 -1
  35. package/dist/matchers.d.ts +0 -21
  36. package/dist/matchers.d.ts.map +0 -1
  37. package/dist/matchers.js +0 -66
  38. package/dist/matchers.js.map +0 -1
  39. package/dist/model.d.ts +0 -15
  40. package/dist/model.d.ts.map +0 -1
  41. package/dist/model.js +0 -43
  42. package/dist/model.js.map +0 -1
  43. package/dist/plugin.d.ts +0 -10
  44. package/dist/plugin.d.ts.map +0 -1
  45. package/dist/plugin.js +0 -26
  46. package/dist/plugin.js.map +0 -1
  47. package/dist/server-harness.d.ts +0 -18
  48. package/dist/server-harness.d.ts.map +0 -1
  49. package/dist/server-harness.js +0 -79
  50. package/dist/server-harness.js.map +0 -1
package/README.md CHANGED
@@ -1,22 +1,23 @@
1
1
  # @skybridge/test
2
2
 
3
- Evals for [Skybridge](https://skybridge.tech) MCP apps: they answer whether a
3
+ Evals for [Skybridge](https://skybridge.tech) MCP Apps: they answer whether a
4
4
  prompt reaches the right tool with the right arguments, which no unit test can
5
5
  check.
6
6
 
7
- **Beta.** The API is still settling, so it publishes under the `beta`
8
- dist-tag and may change between releases.
9
-
10
- Add the `evals` option to the Skybridge Vite plugin, then write scenarios with
11
- vitest's own `it` and `expect`.
7
+ Add the `evals` option to the Skybridge Vite plugin so `expect.chat` exists,
8
+ then write scenarios with vitest's own `it` and `expect`. Pass the app itself
9
+ to `start`: it is served in-process, dialing an in-process handler built on `app.createServerInstance()`, and
10
+ the tool names and argument shapes come from the app value, so no type
11
+ parameter is needed.
12
12
 
13
13
  ```ts
14
+ import { anthropic } from "@ai-sdk/anthropic";
14
15
  import { expect, it } from "vitest";
15
16
  import { start } from "@skybridge/test";
16
- import type { AppType } from "../src/server.js";
17
+ import { app } from "../src/server.js";
17
18
 
18
19
  it("finds products by category", async () => {
19
- const chat = await start<AppType>();
20
+ const chat = await start({ app, model: anthropic("claude-sonnet-4-5") });
20
21
  await chat.send("I am looking for ski goggles");
21
22
 
22
23
  expect.chat(chat).toHaveCalledToolWith("search-products", {
@@ -25,57 +26,70 @@ it("finds products by category", async () => {
25
26
  });
26
27
  ```
27
28
 
28
- ## Choosing a model
29
-
30
- `model` is a `provider/model-id` string. Only the provider you name is
31
- imported, so install just that one as a dev dependency, and the key is read
32
- from its own environment variable.
29
+ There is no child process and no server to boot, so the evals run anywhere
30
+ vitest does. The caveat is that they bypass the Express layer, so nothing
31
+ verifies a bearer token; pass `authInfo` to claim an identity.
33
32
 
34
- | prefix | package | key |
35
- | --- | --- | --- |
36
- | `anthropic` | `@ai-sdk/anthropic` | `ANTHROPIC_API_KEY` |
37
- | `openai` | `@ai-sdk/openai` | `OPENAI_API_KEY` |
38
- | `mistral` | `@ai-sdk/mistral` | `MISTRAL_API_KEY` |
39
- | `google` | `@ai-sdk/google` | `GOOGLE_GENERATIVE_AI_API_KEY` |
33
+ ## Authenticated evals
40
34
 
41
35
  ```ts
42
- skybridge({
43
- evals: {
44
- model: "anthropic/claude-sonnet-4-5",
45
- project: { cwd: ".", command: ["node", "dist/server.js"] },
36
+ const chat = await start({
37
+ app,
38
+ model: anthropic("claude-sonnet-4-5"),
39
+ authInfo: {
40
+ token: "eval-token",
41
+ clientId: "eval-client",
42
+ scopes: ["checkout"],
43
+ extra: { sub: "user_123", email: "ada@example.com" },
46
44
  },
47
45
  });
48
46
  ```
49
47
 
50
- `command` has to start the server process directly, since wrappers such as
51
- `pnpm run` do not forward the channel the runner uses to learn the port.
48
+ `authInfo` injects a claimed identity: the app's own scheme and scope
49
+ enforcement runs for real, in-band challenges included, and only token
50
+ signature verification is skipped. Omit it to eval the anonymous path, which is
51
+ how you assert that a protected tool challenges instead of running.
52
+
53
+ ## Choosing a model
52
54
 
53
- Any other prefix is passed to the AI SDK untouched, so a string such as
54
- `openai/gpt-5` also resolves through the Vercel AI Gateway if you have
55
- `AI_GATEWAY_API_KEY` set and no provider package installed.
55
+ `start` takes any AI SDK model instance, so the provider and its credentials
56
+ are the project's own. Install the provider package you want alongside `ai`.
56
57
 
57
- ## Models the prefixes cannot reach
58
+ ```ts
59
+ import { anthropic } from "@ai-sdk/anthropic";
60
+ import { start } from "@skybridge/test";
61
+
62
+ const chat = await start({ app, model: anthropic("claude-sonnet-4-5") });
63
+ ```
58
64
 
59
- For a local runtime, a company gateway, or a model that needs middleware,
60
- build the model yourself and register it from a vitest setup file. It takes
61
- precedence over `model`, which can then be left out.
65
+ A local runtime or a company gateway works the same way, through the AI SDK's
66
+ own `createOpenAICompatible` or `customProvider`.
62
67
 
63
68
  ```ts
64
- // eval-model.ts
65
69
  import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
66
- import { defineEvalModel } from "@skybridge/test";
67
70
 
68
- defineEvalModel(
69
- createOpenAICompatible({ baseURL: "http://localhost:11434/v1" })("llama3.1"),
70
- );
71
+ const ollama = createOpenAICompatible({
72
+ name: "ollama",
73
+ baseURL: "http://localhost:11434/v1",
74
+ });
75
+ const chat = await start({ app, model: ollama("llama3.1") });
71
76
  ```
72
77
 
78
+ The plugin's `evals` option also carries the defaults every conversation starts
79
+ from, so a project can set them once instead of per scenario.
80
+
73
81
  ```ts
74
- // vitest.config.ts
75
- export default defineConfig({
76
- plugins: [skybridge({ evals: { project: { ... } } })],
77
- test: { setupFiles: ["./eval-model.ts"] },
78
- });
82
+ skybridge({ evals: { temperature: 0, maxSteps: 6 } });
83
+ ```
84
+
85
+ ## Matchers
86
+
87
+ Assertions on what the assistant said work the same way, though model prose
88
+ varies between runs, so keep them loose.
89
+
90
+ ```ts
91
+ expect.chat(chat).toHaveSaid("in stock");
92
+ expect.chat(chat).not.toHaveSaid("refund");
79
93
  ```
80
94
 
81
95
  See the [documentation](https://docs.skybridge.tech) for setup and the full
package/dist/chat.d.ts CHANGED
@@ -1,46 +1,49 @@
1
1
  import { type LanguageModel } from "ai";
2
- import type { ToolInput, ToolNames } from "skybridge/server";
3
- /**
4
- * One recorded call, typed against the project's own registry: `name` is a
5
- * union of the project's tool names and `arguments` is discriminated on it.
6
- */
7
- export type ToolCall<App> = {
8
- [Name in ToolNames<App>]: {
9
- name: Name;
10
- arguments: ToolInput<App, Name>;
11
- failed?: string;
12
- };
13
- }[ToolNames<App>];
2
+ import type { ToolCall, Turn } from "./matchers/types.js";
14
3
  interface HostConfig {
15
4
  model: LanguageModel;
16
5
  temperature?: number;
17
6
  systemPrompt?: string;
18
7
  maxSteps?: number;
19
8
  }
20
- interface McpToolDefinition {
21
- name: string;
22
- description?: string;
23
- inputSchema: Record<string, unknown>;
24
- }
9
+ /**
10
+ * Stubbed results keyed by tool name. The typed form scenarios write lives on
11
+ * `start`'s options.
12
+ *
13
+ * @internal
14
+ */
15
+ export type StubMap = Record<string, (args: Record<string, unknown>) => unknown>;
16
+ /**
17
+ * A `fetch` replacement the transport dials instead of the network. Mirrors the
18
+ * MCP client's own `fetch` option so an in-process app can serve the session.
19
+ */
20
+ export type TransportFetch = (url: string | URL, init?: RequestInit) => Promise<Response>;
25
21
  /**
26
22
  * One conversation against one MCP session. A scenario asserts on
27
23
  * {@link Chat.toolCalls}, the sequence of calls the model made.
28
24
  */
29
25
  export declare class Chat<App = unknown> {
26
+ /** Type-only anchor so `expect.chat` infers `App` from the conversation. */
27
+ readonly $app: App;
28
+ /** Everything that happened, in the order it happened. */
29
+ readonly turns: Turn<App>[];
30
30
  /** Every call the model made, in order, as it crossed the wire. */
31
- readonly toolCalls: ToolCall<App>[];
31
+ get toolCalls(): ToolCall<App>[];
32
+ /** What the assistant said, one entry per turn, intermediate ones included. */
33
+ get assistantTurns(): string[];
32
34
  private readonly messages;
33
35
  private readonly failures;
34
36
  private readonly client;
35
37
  private readonly transport;
36
38
  private readonly host;
39
+ private readonly stubs;
37
40
  private tools;
41
+ /** The model the conversation runs on, and the judge's default. */
42
+ get model(): LanguageModel;
38
43
  private constructor();
39
- static open<App>(serverUrl: string, host: HostConfig): Promise<Chat<App>>;
40
- /** The tool definitions the model was looking at, for failure reports. */
41
- get toolDefinitions(): McpToolDefinition[];
42
- /** Takes a turn and returns the model's answer. */
43
- send(prompt: string): Promise<string>;
44
+ static open<App>(host: HostConfig, fetchImpl: TransportFetch, stubs: StubMap): Promise<Chat<App>>;
45
+ /** Takes a turn. Assert on the result with `expect.chat`. */
46
+ send(prompt: string): Promise<void>;
44
47
  close(): Promise<void>;
45
48
  private failureFor;
46
49
  private toolSet;
@@ -1 +1 @@
1
- {"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAEA,OAAO,EAIL,KAAK,aAAa,EAGnB,MAAM,IAAI,CAAC;AACZ,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7D;;;GAGG;AACH,MAAM,MAAM,QAAQ,CAAC,GAAG,IAAI;KACzB,IAAI,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG;QACxB,IAAI,EAAE,IAAI,CAAC;QACX,SAAS,EAAE,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAChC,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB;CACF,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;AAElB,UAAU,UAAU;IAClB,KAAK,EAAE,aAAa,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,UAAU,iBAAiB;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACtC;AAID;;;GAGG;AACH,qBAAa,IAAI,CAAC,GAAG,GAAG,OAAO;IAC7B,mEAAmE;IACnE,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAM;IAEzC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA6B;IACtD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAa;IAClC,OAAO,CAAC,KAAK,CAA2B;IAExC,OAAO;WAUM,IAAI,CAAC,GAAG,EACnB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,UAAU,GACf,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAerB,0EAA0E;IAC1E,IAAI,eAAe,IAAI,iBAAiB,EAAE,CAEzC;IAED,mDAAmD;IAC7C,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IA4BrC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,OAAO,CAAC,UAAU;IAalB,OAAO,CAAC,OAAO;CAmChB"}
1
+ {"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAKA,OAAO,EAIL,KAAK,aAAa,EAGnB,MAAM,IAAI,CAAC;AACZ,OAAO,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AAE1D,UAAU,UAAU;IAClB,KAAK,EAAE,aAAa,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;GAKG;AACH,MAAM,MAAM,OAAO,GAAG,MAAM,CAC1B,MAAM,EACN,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,OAAO,CAC3C,CAAC;AA4BF;;;GAGG;AACH,MAAM,MAAM,cAAc,GAAG,CAC3B,GAAG,EAAE,MAAM,GAAG,GAAG,EACjB,IAAI,CAAC,EAAE,WAAW,KACf,OAAO,CAAC,QAAQ,CAAC,CAAC;AAEvB;;;GAGG;AACH,qBAAa,IAAI,CAAC,GAAG,GAAG,OAAO;IAC7B,4EAA4E;IAC5E,SAAiB,IAAI,EAAE,GAAG,CAAC;IAE3B,0DAA0D;IAC1D,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAM;IAEjC,mEAAmE;IACnE,IAAI,SAAS,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE,CAI/B;IAED,+EAA+E;IAC/E,IAAI,cAAc,IAAI,MAAM,EAAE,CAI7B;IAED,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA6B;IACtD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAa;IAClC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA+B;IACrD,OAAO,CAAC,KAAK,CAAc;IAE3B,mEAAmE;IACnE,IAAI,KAAK,IAAI,aAAa,CAEzB;IAED,OAAO;WAYM,IAAI,CAAC,GAAG,EACnB,IAAI,EAAE,UAAU,EAChB,SAAS,EAAE,cAAc,EACzB,KAAK,EAAE,OAAO,GACb,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IA0BrB,6DAA6D;IACvD,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAqCnC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,OAAO,CAAC,UAAU;IAalB,OAAO,CAAC,OAAO;CA8ChB"}
package/dist/chat.js CHANGED
@@ -1,33 +1,72 @@
1
- import { Client } from "@modelcontextprotocol/sdk/client/index.js";
2
- import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
1
+ import { Client, StreamableHTTPClientTransport, } from "@modelcontextprotocol/client";
3
2
  import { dynamicTool, generateText, jsonSchema, stepCountIs, } from "ai";
4
3
  const DEFAULT_MAX_STEPS = 8;
4
+ const IN_PROCESS_URL = "http://in-process.skybridge.test/mcp";
5
+ /**
6
+ * Dress a stubbed value the way the server dresses a real one, so the model
7
+ * reads the same envelope under a stub as it does live.
8
+ */
9
+ function asToolResult(value) {
10
+ return {
11
+ content: [
12
+ {
13
+ type: "text",
14
+ text: typeof value === "string" ? value : JSON.stringify(value),
15
+ },
16
+ ],
17
+ };
18
+ }
19
+ function outputOf(results, toolCallId) {
20
+ return results.find((result) => result.toolCallId === toolCallId)?.output;
21
+ }
5
22
  /**
6
23
  * One conversation against one MCP session. A scenario asserts on
7
24
  * {@link Chat.toolCalls}, the sequence of calls the model made.
8
25
  */
9
26
  export class Chat {
27
+ /** Everything that happened, in the order it happened. */
28
+ turns = [];
10
29
  /** Every call the model made, in order, as it crossed the wire. */
11
- toolCalls = [];
30
+ get toolCalls() {
31
+ return this.turns
32
+ .filter((turn) => turn.role === "tool")
33
+ .map((turn) => turn.call);
34
+ }
35
+ /** What the assistant said, one entry per turn, intermediate ones included. */
36
+ get assistantTurns() {
37
+ return this.turns
38
+ .filter((turn) => turn.role === "assistant")
39
+ .map((turn) => turn.text);
40
+ }
12
41
  messages = [];
13
42
  failures = new Map();
14
43
  client;
15
44
  transport;
16
45
  host;
46
+ stubs;
17
47
  tools = [];
18
- constructor(client, transport, host) {
48
+ /** The model the conversation runs on, and the judge's default. */
49
+ get model() {
50
+ return this.host.model;
51
+ }
52
+ constructor(client, transport, host, stubs) {
19
53
  this.client = client;
20
54
  this.transport = transport;
21
55
  this.host = host;
56
+ this.stubs = new Map(Object.entries(stubs));
22
57
  }
23
- static async open(serverUrl, host) {
58
+ static async open(host, fetchImpl, stubs) {
24
59
  const client = new Client({ name: "skybridge-eval", version: "0" });
25
- const transport = new StreamableHTTPClientTransport(new URL(serverUrl));
60
+ const transport = new StreamableHTTPClientTransport(new URL(IN_PROCESS_URL), { fetch: fetchImpl });
26
61
  await client.connect(transport);
27
- const chat = new Chat(client, transport, host);
62
+ const chat = new Chat(client, transport, host, stubs);
28
63
  try {
29
64
  const { tools } = await client.listTools();
30
65
  chat.tools = tools;
66
+ const unknownStubs = Object.keys(stubs).filter((name) => !tools.some((tool) => tool.name === name));
67
+ if (unknownStubs.length > 0) {
68
+ throw new Error(`the app exposes no tool named ${unknownStubs.map((name) => `"${name}"`).join(", ")}, so the stub would never run`);
69
+ }
31
70
  }
32
71
  catch (error) {
33
72
  await transport.close();
@@ -35,16 +74,15 @@ export class Chat {
35
74
  }
36
75
  return chat;
37
76
  }
38
- /** The tool definitions the model was looking at, for failure reports. */
39
- get toolDefinitions() {
40
- return this.tools;
41
- }
42
- /** Takes a turn and returns the model's answer. */
77
+ /** Takes a turn. Assert on the result with `expect.chat`. */
43
78
  async send(prompt) {
44
79
  this.messages.push({ role: "user", content: prompt });
80
+ this.turns.push({ role: "user", text: prompt });
45
81
  const result = await generateText({
46
82
  model: this.host.model,
47
- temperature: this.host.temperature ?? 0,
83
+ ...(this.host.temperature === undefined
84
+ ? {}
85
+ : { temperature: this.host.temperature }),
48
86
  ...(this.host.systemPrompt === undefined
49
87
  ? {}
50
88
  : { system: this.host.systemPrompt }),
@@ -53,16 +91,22 @@ export class Chat {
53
91
  stopWhen: stepCountIs(this.host.maxSteps ?? DEFAULT_MAX_STEPS),
54
92
  });
55
93
  for (const step of result.steps) {
94
+ if (step.text !== "") {
95
+ this.turns.push({ role: "assistant", text: step.text });
96
+ }
56
97
  for (const call of step.toolCalls) {
57
- this.toolCalls.push({
58
- name: call.toolName,
59
- arguments: (call.input ?? {}),
60
- ...this.failureFor(call),
98
+ this.turns.push({
99
+ role: "tool",
100
+ call: {
101
+ name: call.toolName,
102
+ arguments: (call.input ?? {}),
103
+ ...this.failureFor(call),
104
+ },
105
+ result: outputOf(step.toolResults, call.toolCallId),
61
106
  });
62
107
  }
63
108
  }
64
109
  this.messages.push(...result.response.messages);
65
- return result.text;
66
110
  }
67
111
  async close() {
68
112
  await this.transport.close();
@@ -86,6 +130,10 @@ export class Chat {
86
130
  description: definition.description ?? "",
87
131
  inputSchema: jsonSchema(parameters),
88
132
  execute: async (input, { toolCallId }) => {
133
+ const stubbed = await this.stubs.get(definition.name)?.((input ?? {}));
134
+ if (stubbed !== undefined) {
135
+ return asToolResult(stubbed);
136
+ }
89
137
  let result;
90
138
  try {
91
139
  result = await this.client.callTool({
@@ -100,7 +148,10 @@ export class Chat {
100
148
  if (result.isError === true) {
101
149
  this.failures.set(toolCallId, `the tool returned an error: ${JSON.stringify(result.content)}`);
102
150
  }
103
- return result.content ?? result;
151
+ return {
152
+ content: result.content,
153
+ structuredContent: result.structuredContent,
154
+ };
104
155
  },
105
156
  }),
106
157
  ];
package/dist/chat.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,6BAA6B,EAAE,MAAM,oDAAoD,CAAC;AACnG,OAAO,EACL,WAAW,EACX,YAAY,EACZ,UAAU,EAGV,WAAW,GACZ,MAAM,IAAI,CAAC;AA4BZ,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B;;;GAGG;AACH,MAAM,OAAO,IAAI;IACf,mEAAmE;IAC1D,SAAS,GAAoB,EAAE,CAAC;IAExB,QAAQ,GAAmB,EAAE,CAAC;IAC9B,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrC,MAAM,CAAS;IACf,SAAS,CAAgC;IACzC,IAAI,CAAa;IAC1B,KAAK,GAAwB,EAAE,CAAC;IAExC,YACE,MAAc,EACd,SAAwC,EACxC,IAAgB;QAEhB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,IAAI,CACf,SAAiB,EACjB,IAAgB;QAEhB,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,IAAI,6BAA6B,CAAC,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;QACxE,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAM,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QACpD,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACrB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YACxB,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,0EAA0E;IAC1E,IAAI,eAAe;QACjB,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED,mDAAmD;IACnD,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAEtD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;YACtB,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC;YACvC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,KAAK,SAAS;gBACtC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE;YACrB,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;SAC/D,CAAC,CAAC;QAEH,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAChC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAClC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;oBAClB,IAAI,EAAE,IAAI,CAAC,QAAQ;oBACnB,SAAS,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAA4B;oBACxD,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;iBACR,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAChD,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;IAEO,UAAU,CAAC,IAA+C;QAGhE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QAC9B,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC1B,OAAO,EAAE,MAAM,EAAE,2CAA2C,EAAE,CAAC;QACjE,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAEO,OAAO;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;YAC5C,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,UAAU,EAAE,GAAG,UAAU,CAAC,WAAW,CAAC;YACpE,OAAO;gBACL,UAAU,CAAC,IAAI;gBACf,WAAW,CAAC;oBACV,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,EAAE;oBACzC,WAAW,EAAE,UAAU,CAAC,UAAU,CAAC;oBACnC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE;wBACvC,IAAI,MAA+C,CAAC;wBACpD,IAAI,CAAC;4BACH,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;gCAClC,IAAI,EAAE,UAAU,CAAC,IAAI;gCACrB,SAAS,EAAE,CAAC,KAAK,IAAI,EAAE,CAA4B;6BACpD,CAAC,CAAC;wBACL,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BACf,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,sCAAsC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC/F,CAAC;4BACF,MAAM,KAAK,CAAC;wBACd,CAAC;wBACD,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;4BAC5B,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,+BAA+B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAChE,CAAC;wBACJ,CAAC;wBACD,OAAO,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC;oBAClC,CAAC;iBACF,CAAC;aACM,CAAC;QACb,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;CACF"}
1
+ {"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,MAAM,EACN,6BAA6B,GAE9B,MAAM,8BAA8B,CAAC;AACtC,OAAO,EACL,WAAW,EACX,YAAY,EACZ,UAAU,EAGV,WAAW,GACZ,MAAM,IAAI,CAAC;AAqBZ,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B,MAAM,cAAc,GAAG,sCAAsC,CAAC;AAE9D;;;GAGG;AACH,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO;QACL,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;aAChE;SACF;KACF,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CACf,OAAmD,EACnD,UAAkB;IAElB,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,KAAK,UAAU,CAAC,EAAE,MAAM,CAAC;AAC5E,CAAC;AAWD;;;GAGG;AACH,MAAM,OAAO,IAAI;IAIf,0DAA0D;IACjD,KAAK,GAAgB,EAAE,CAAC;IAEjC,mEAAmE;IACnE,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,KAAK;aACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC;aACtC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAED,+EAA+E;IAC/E,IAAI,cAAc;QAChB,OAAO,IAAI,CAAC,KAAK;aACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,WAAW,CAAC;aAC3C,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAEgB,QAAQ,GAAmB,EAAE,CAAC;IAC9B,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrC,MAAM,CAAS;IACf,SAAS,CAAgC;IACzC,IAAI,CAAa;IACjB,KAAK,CAA+B;IAC7C,KAAK,GAAW,EAAE,CAAC;IAE3B,mEAAmE;IACnE,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;IACzB,CAAC;IAED,YACE,MAAc,EACd,SAAwC,EACxC,IAAgB,EAChB,KAAc;QAEd,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,IAAI,CACf,IAAgB,EAChB,SAAyB,EACzB,KAAc;QAEd,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,IAAI,6BAA6B,CACjD,IAAI,GAAG,CAAC,cAAc,CAAC,EACvB,EAAE,KAAK,EAAE,SAAS,EAAE,CACrB,CAAC;QACF,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAM,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;QAC3D,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;YACnB,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAC5C,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,CACpD,CAAC;YACF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,IAAI,KAAK,CACb,iCAAiC,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,+BAA+B,CACnH,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YACxB,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,6DAA6D;IAC7D,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QACtD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QAEhD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;YACtB,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,KAAK,SAAS;gBACrC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YAC3C,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,KAAK,SAAS;gBACtC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE;YACrB,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;SAC/D,CAAC,CAAC;QAEH,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,IAAI,CAAC,IAAI,KAAK,EAAE,EAAE,CAAC;gBACrB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YAC1D,CAAC;YACD,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAClC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;oBACd,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE;wBACJ,IAAI,EAAE,IAAI,CAAC,QAAQ;wBACnB,SAAS,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAA4B;wBACxD,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;qBACR;oBAClB,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,UAAU,CAAC;iBACpD,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;IAEO,UAAU,CAAC,IAA+C;QAGhE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QAC9B,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC1B,OAAO,EAAE,MAAM,EAAE,2CAA2C,EAAE,CAAC;QACjE,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAEO,OAAO;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;YAC5C,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,UAAU,EAAE,GAAG,UAAU,CAAC,WAAW,CAAC;YACpE,OAAO;gBACL,UAAU,CAAC,IAAI;gBACf,WAAW,CAAC;oBACV,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,EAAE;oBACzC,WAAW,EAAE,UAAU,CACrB,UAA8C,CAC/C;oBACD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE;wBACvC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CACrD,CAAC,KAAK,IAAI,EAAE,CAA4B,CACzC,CAAC;wBACF,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;4BAC1B,OAAO,YAAY,CAAC,OAAO,CAAC,CAAC;wBAC/B,CAAC;wBACD,IAAI,MAA+C,CAAC;wBACpD,IAAI,CAAC;4BACH,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;gCAClC,IAAI,EAAE,UAAU,CAAC,IAAI;gCACrB,SAAS,EAAE,CAAC,KAAK,IAAI,EAAE,CAA4B;6BACpD,CAAC,CAAC;wBACL,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BACf,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,sCAAsC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC/F,CAAC;4BACF,MAAM,KAAK,CAAC;wBACd,CAAC;wBACD,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;4BAC5B,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,+BAA+B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAChE,CAAC;wBACJ,CAAC;wBACD,OAAO;4BACL,OAAO,EAAE,MAAM,CAAC,OAAO;4BACvB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;yBAC5C,CAAC;oBACJ,CAAC;iBACF,CAAC;aACM,CAAC;QACb,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;CACF"}
package/dist/index.d.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  export type { Chat } from "./chat.js";
2
- export type { ChatAssertion, ChatMatchers } from "./matchers.js";
3
- export { defineEvalModel } from "./model.js";
2
+ export type { ChatAssertion, ChatMatchers, Judgment, JudgmentOptions, Verdict, } from "./matchers/index.js";
3
+ export type { EvalApp, EvalIdentity, ToolStubs, } from "./session-registry.js";
4
4
  export { start } from "./session-registry.js";
5
5
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACtC,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AACjE,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACtC,YAAY,EACV,aAAa,EACb,YAAY,EACZ,QAAQ,EACR,eAAe,EACf,OAAO,GACR,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EACV,OAAO,EACP,YAAY,EACZ,SAAS,GACV,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
package/dist/index.js CHANGED
@@ -1,3 +1,2 @@
1
- export { defineEvalModel } from "./model.js";
2
1
  export { start } from "./session-registry.js";
3
2
  //# sourceMappingURL=index.js.map
package/dist/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAaA,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
@@ -0,0 +1,4 @@
1
+ import "./matchers.js";
2
+ export type { ChatAssertion, ChatMatchers, Judgment, JudgmentOptions, Verdict, } from "./matchers.js";
3
+ export type { ChatLike, ToolCall, Turn } from "./types.js";
4
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/matchers/index.ts"],"names":[],"mappings":"AAAA,OAAO,eAAe,CAAC;AAEvB,YAAY,EACV,aAAa,EACb,YAAY,EACZ,QAAQ,EACR,eAAe,EACf,OAAO,GACR,MAAM,eAAe,CAAC;AACvB,YAAY,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC"}
@@ -0,0 +1,2 @@
1
+ import "./matchers.js";
2
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/matchers/index.ts"],"names":[],"mappings":"AAAA,OAAO,eAAe,CAAC"}
@@ -0,0 +1,51 @@
1
+ import "vitest";
2
+ import type { LanguageModel } from "ai";
3
+ import type { ToolInput, ToolNames } from "skybridge/server";
4
+ import type { ChatLike } from "./types.js";
5
+ /** What a judge decided about a conversation. */
6
+ export interface Verdict {
7
+ pass: boolean;
8
+ /** Why, quoted in the failure message. Omit it when the judge gives no prose. */
9
+ reasoning?: string;
10
+ }
11
+ /** The conversation handed to a {@link JudgmentOptions.judge}. */
12
+ export interface Judgment {
13
+ criteria: string;
14
+ /** The rendered conversation, one turn per line, tool results included. */
15
+ transcript: string;
16
+ }
17
+ /** Options for {@link ChatMatchers.toPassJudgment}. */
18
+ export interface JudgmentOptions {
19
+ /** Overrides the judge for this assertion. Defaults to the chat's model. */
20
+ model?: LanguageModel;
21
+ /**
22
+ * Grades the conversation instead of the default model call, so an
23
+ * evaluation model or a scoring service can stand in for the judge. Takes
24
+ * precedence over {@link JudgmentOptions.model}.
25
+ */
26
+ judge?: (judgment: Judgment) => Verdict | Promise<Verdict>;
27
+ }
28
+ /**
29
+ * Assertions about the tool calls a conversation produced. Kept off vitest's
30
+ * global `Matchers` so they never appear on unrelated `expect()` calls, and so
31
+ * this stays the one place later tool-call helpers are added.
32
+ */
33
+ export interface ChatMatchers<App> {
34
+ toHaveCalledToolOnce<Name extends ToolNames<App>>(name: Name, args?: Partial<ToolInput<App, Name>>): void;
35
+ toHaveCalledToolWith<Name extends ToolNames<App>>(name: Name, args: Partial<ToolInput<App, Name>>): void;
36
+ toNeverHaveCalledTool<Name extends ToolNames<App>>(name: Name): void;
37
+ toHaveFailedToolCall<Name extends ToolNames<App>>(name: Name): void;
38
+ toHaveSaid(text: string | RegExp): void;
39
+ toHaveCalledToolsInOrder(...names: [ToolNames<App>, ...ToolNames<App>[]]): void;
40
+ toHaveCalledNoTools(): void;
41
+ toPassJudgment(criteria: string, options?: JudgmentOptions): Promise<void>;
42
+ }
43
+ export interface ChatAssertion<App> extends ChatMatchers<App> {
44
+ not: ChatMatchers<App>;
45
+ }
46
+ declare module "vitest" {
47
+ interface ExpectStatic {
48
+ chat<App>(chat: ChatLike<App>): ChatAssertion<App>;
49
+ }
50
+ }
51
+ //# sourceMappingURL=matchers.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"matchers.d.ts","sourceRoot":"","sources":["../../src/matchers/matchers.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,CAAC;AAChB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACxC,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7D,OAAO,KAAK,EAAE,QAAQ,EAAkB,MAAM,YAAY,CAAC;AAE3D,iDAAiD;AACjD,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,OAAO,CAAC;IACd,iFAAiF;IACjF,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,kEAAkE;AAClE,MAAM,WAAW,QAAQ;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,2EAA2E;IAC3E,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,uDAAuD;AACvD,MAAM,WAAW,eAAe;IAC9B,4EAA4E;IAC5E,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB;;;;OAIG;IACH,KAAK,CAAC,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAC5D;AAED;;;;GAIG;AACH,MAAM,WAAW,YAAY,CAAC,GAAG;IAC/B,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAC9C,IAAI,EAAE,IAAI,EACV,IAAI,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GACnC,IAAI,CAAC;IACR,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAC9C,IAAI,EAAE,IAAI,EACV,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GAClC,IAAI,CAAC;IACR,qBAAqB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,CAAC;IACrE,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,CAAC;IACpE,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IACxC,wBAAwB,CACtB,GAAG,KAAK,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,GAAG,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,GAC9C,IAAI,CAAC;IACR,mBAAmB,IAAI,IAAI,CAAC;IAC5B,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5E;AAED,MAAM,WAAW,aAAa,CAAC,GAAG,CAAE,SAAQ,YAAY,CAAC,GAAG,CAAC;IAC3D,GAAG,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC;CACxB;AAED,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,YAAY;QACpB,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;KACpD;CACF"}