@skybridge/test 0.0.0-dev.41ba66d → 0.0.0-dev.44b577c

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/README.md +105 -8
  2. package/dist/chat.d.ts +23 -24
  3. package/dist/chat.d.ts.map +1 -1
  4. package/dist/chat.js +100 -26
  5. package/dist/chat.js.map +1 -1
  6. package/dist/eval-middlewares.d.ts +12 -0
  7. package/dist/eval-middlewares.d.ts.map +1 -0
  8. package/dist/eval-middlewares.js +18 -0
  9. package/dist/eval-middlewares.js.map +1 -0
  10. package/dist/index.d.ts +2 -2
  11. package/dist/index.d.ts.map +1 -1
  12. package/dist/index.js +0 -1
  13. package/dist/index.js.map +1 -1
  14. package/dist/input-files.d.ts +14 -0
  15. package/dist/input-files.d.ts.map +1 -0
  16. package/dist/input-files.js +126 -0
  17. package/dist/input-files.js.map +1 -0
  18. package/dist/matchers/index.d.ts +4 -0
  19. package/dist/matchers/index.d.ts.map +1 -0
  20. package/dist/matchers/index.js +2 -0
  21. package/dist/matchers/index.js.map +1 -0
  22. package/dist/matchers/matchers.d.ts +54 -0
  23. package/dist/matchers/matchers.d.ts.map +1 -0
  24. package/dist/matchers/matchers.js +258 -0
  25. package/dist/matchers/matchers.js.map +1 -0
  26. package/dist/matchers/types.d.ts +44 -0
  27. package/dist/matchers/types.d.ts.map +1 -0
  28. package/dist/matchers/types.js +2 -0
  29. package/dist/matchers/types.js.map +1 -0
  30. package/dist/middleware/list-tools.d.ts +4 -0
  31. package/dist/middleware/list-tools.d.ts.map +1 -0
  32. package/dist/middleware/list-tools.js +67 -0
  33. package/dist/middleware/list-tools.js.map +1 -0
  34. package/dist/middleware/resolve-files.d.ts +4 -0
  35. package/dist/middleware/resolve-files.d.ts.map +1 -0
  36. package/dist/middleware/resolve-files.js +13 -0
  37. package/dist/middleware/resolve-files.js.map +1 -0
  38. package/dist/middleware/stub-tool.d.ts +4 -0
  39. package/dist/middleware/stub-tool.d.ts.map +1 -0
  40. package/dist/middleware/stub-tool.js +16 -0
  41. package/dist/middleware/stub-tool.js.map +1 -0
  42. package/dist/session-registry.d.ts +53 -9
  43. package/dist/session-registry.d.ts.map +1 -1
  44. package/dist/session-registry.js +50 -16
  45. package/dist/session-registry.js.map +1 -1
  46. package/package.json +13 -13
  47. package/dist/config.d.ts +0 -13
  48. package/dist/config.d.ts.map +0 -1
  49. package/dist/config.js +0 -2
  50. package/dist/config.js.map +0 -1
  51. package/dist/global-setup.d.ts +0 -3
  52. package/dist/global-setup.d.ts.map +0 -1
  53. package/dist/global-setup.js +0 -24
  54. package/dist/global-setup.js.map +0 -1
  55. package/dist/matchers.d.ts +0 -21
  56. package/dist/matchers.d.ts.map +0 -1
  57. package/dist/matchers.js +0 -66
  58. package/dist/matchers.js.map +0 -1
  59. package/dist/model.d.ts +0 -9
  60. package/dist/model.d.ts.map +0 -1
  61. package/dist/model.js +0 -24
  62. package/dist/model.js.map +0 -1
  63. package/dist/plugin.d.ts +0 -10
  64. package/dist/plugin.d.ts.map +0 -1
  65. package/dist/plugin.js +0 -32
  66. package/dist/plugin.js.map +0 -1
  67. package/dist/server-harness.d.ts +0 -18
  68. package/dist/server-harness.d.ts.map +0 -1
  69. package/dist/server-harness.js +0 -79
  70. package/dist/server-harness.js.map +0 -1
package/README.md CHANGED
@@ -1,22 +1,23 @@
1
1
  # @skybridge/test
2
2
 
3
- Evals for [Skybridge](https://skybridge.tech) MCP apps: they answer whether a
3
+ Evals for [Skybridge](https://skybridge.tech) MCP Apps: they answer whether a
4
4
  prompt reaches the right tool with the right arguments, which no unit test can
5
5
  check.
6
6
 
7
- **Beta.** The API is still settling, so it publishes under the `beta`
8
- dist-tag and may change between releases.
9
-
10
- Add the `evals` option to the Skybridge Vite plugin, then write scenarios with
11
- vitest's own `it` and `expect`.
7
+ Add the `evals` option to the Skybridge Vite plugin so `expect.chat` exists,
8
+ then write scenarios with vitest's own `it` and `expect`. Pass the app itself
9
+ to `start`: it is served in-process, using your exported app, and
10
+ the tool names and argument shapes come from the app value, so no type
11
+ parameter is needed.
12
12
 
13
13
  ```ts
14
+ import { anthropic } from "@ai-sdk/anthropic";
14
15
  import { expect, it } from "vitest";
15
16
  import { start } from "@skybridge/test";
16
- import type { AppType } from "../src/server.js";
17
+ import { app } from "../src/server.js";
17
18
 
18
19
  it("finds products by category", async () => {
19
- const chat = await start<AppType>();
20
+ const chat = await start({ app, model: anthropic("claude-sonnet-4-5") });
20
21
  await chat.send("I am looking for ski goggles");
21
22
 
22
23
  expect.chat(chat).toHaveCalledToolWith("search-products", {
@@ -25,5 +26,101 @@ it("finds products by category", async () => {
25
26
  });
26
27
  ```
27
28
 
29
+ There is no child process and no server to boot, so the evals run anywhere
30
+ vitest does. The caveat is that they bypass the Express layer, so nothing
31
+ verifies a bearer token; pass `authInfo` to claim an identity.
32
+
33
+ ## Authenticated evals
34
+
35
+ ```ts
36
+ const chat = await start({
37
+ app,
38
+ model: anthropic("claude-sonnet-4-5"),
39
+ authInfo: {
40
+ token: "eval-token",
41
+ clientId: "eval-client",
42
+ scopes: ["checkout"],
43
+ extra: { sub: "user_123", email: "ada@example.com" },
44
+ },
45
+ });
46
+ ```
47
+
48
+ `authInfo` injects a claimed identity: the app's own scheme and scope
49
+ enforcement runs for real, in-band challenges included, and only token
50
+ signature verification is skipped. Omit it to eval the anonymous path, which is
51
+ how you assert that a protected tool challenges instead of running.
52
+
53
+ ## Choosing a model
54
+
55
+ `start` takes any AI SDK model instance, so the provider and its credentials
56
+ are the project's own. Install the provider package you want alongside `ai`.
57
+
58
+ ```ts
59
+ import { anthropic } from "@ai-sdk/anthropic";
60
+ import { start } from "@skybridge/test";
61
+
62
+ const chat = await start({ app, model: anthropic("claude-sonnet-4-5") });
63
+ ```
64
+
65
+ A local runtime or a company gateway works the same way, through the AI SDK's
66
+ own `createOpenAICompatible` or `customProvider`.
67
+
68
+ ```ts
69
+ import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
70
+
71
+ const ollama = createOpenAICompatible({
72
+ name: "ollama",
73
+ baseURL: "http://localhost:11434/v1",
74
+ });
75
+ const chat = await start({ app, model: ollama("llama3.1") });
76
+ ```
77
+
78
+ The plugin's `evals` option also carries the defaults every conversation starts
79
+ from, so a project can set them once instead of per scenario.
80
+
81
+ ```ts
82
+ skybridge({ evals: { temperature: 0, maxSteps: 6 } });
83
+ ```
84
+
85
+ ## Matchers
86
+
87
+ Assertions on what the assistant said work the same way, though model prose
88
+ varies between runs, so keep them loose.
89
+
90
+ ```ts
91
+ expect.chat(chat).toHaveSaid("in stock");
92
+ expect.chat(chat).not.toHaveSaid("refund");
93
+ ```
94
+
28
95
  See the [documentation](https://docs.skybridge.tech) for setup and the full
29
96
  matcher list.
97
+
98
+ ## Input files
99
+
100
+ `chat.send` also accepts AI SDK `UserContent`. Attach a `FilePart` and use the same object in `expect.inputFile` to check which file a tool received:
101
+
102
+ ```ts
103
+ import { readFile } from "node:fs/promises";
104
+ import type { FilePart } from "ai";
105
+
106
+ const portrait: FilePart = {
107
+ type: "file",
108
+ mediaType: "image/png",
109
+ filename: "photo.png",
110
+ data: await readFile(new URL("./fixtures/photo.png", import.meta.url)),
111
+ };
112
+
113
+ await chat.send([{ type: "text", text: "Resize this photo" }, portrait]);
114
+
115
+ expect.chat(chat).toHaveCalledToolWith("image_crop_and_resize", {
116
+ file: expect.inputFile(portrait),
117
+ });
118
+ expect.chat(chat).toHaveCalledToolOnce("asset_openai_file_upload", {
119
+ files: [expect.inputFile(portrait)],
120
+ });
121
+ ```
122
+
123
+ `file` and `files` must be listed in their tools' `_meta["openai/fileParams"]`.
124
+ Misplaced file matchers throw immediately, including nested matchers and `.not`.
125
+ Declare file parameters with `FileRef` and list their names in `_meta["openai/fileParams"]`. Optional, nullable, and array forms are supported. Your handler or stub receives the attached file; downloads remain available until the conversation closes.
126
+ See [Evals](https://docs.skybridge.tech/test/evals) for how files reach the model and the tool.
package/dist/chat.d.ts CHANGED
@@ -1,46 +1,45 @@
1
- import { type LanguageModel } from "ai";
2
- import type { ToolInput, ToolNames } from "skybridge/server";
3
- /**
4
- * One recorded call, typed against the project's own registry: `name` is a
5
- * union of the project's tool names and `arguments` is discriminated on it.
6
- */
7
- export type ToolCall<App> = {
8
- [Name in ToolNames<App>]: {
9
- name: Name;
10
- arguments: ToolInput<App, Name>;
11
- failed?: string;
12
- };
13
- }[ToolNames<App>];
1
+ import { type LanguageModel, type UserContent } from "ai";
2
+ import type { EvalMiddlewares } from "./eval-middlewares.js";
3
+ import type { ToolCall, Turn } from "./matchers/types.js";
14
4
  interface HostConfig {
15
5
  model: LanguageModel;
16
6
  temperature?: number;
17
7
  systemPrompt?: string;
18
8
  maxSteps?: number;
19
9
  }
20
- interface McpToolDefinition {
21
- name: string;
22
- description?: string;
23
- inputSchema: Record<string, unknown>;
24
- }
10
+ /**
11
+ * A `fetch` replacement the transport dials instead of the network. Mirrors the
12
+ * MCP client's own `fetch` option so an in-process app can serve the session.
13
+ */
14
+ export type TransportFetch = (url: string | URL, init?: RequestInit) => Promise<Response>;
25
15
  /**
26
16
  * One conversation against one MCP session. A scenario asserts on
27
17
  * {@link Chat.toolCalls}, the sequence of calls the model made.
28
18
  */
29
19
  export declare class Chat<App = unknown> {
20
+ private readonly middleware;
21
+ private readonly dispose;
22
+ /** Type-only anchor so `expect.chat` infers `App` from the conversation. */
23
+ readonly $app: App;
24
+ /** Everything that happened, in the order it happened. */
25
+ readonly turns: Turn<App>[];
30
26
  /** Every call the model made, in order, as it crossed the wire. */
31
- readonly toolCalls: ToolCall<App>[];
27
+ get toolCalls(): ToolCall<App>[];
28
+ /** What the assistant said, one entry per turn, intermediate ones included. */
29
+ get assistantTurns(): string[];
32
30
  private readonly messages;
33
31
  private readonly failures;
34
32
  private readonly client;
35
33
  private readonly transport;
36
34
  private readonly host;
37
35
  private tools;
36
+ private closePromise?;
37
+ /** The model the conversation runs on, and the judge's default. */
38
+ get model(): LanguageModel;
38
39
  private constructor();
39
- static open<App>(serverUrl: string, host: HostConfig): Promise<Chat<App>>;
40
- /** The tool definitions the model was looking at, for failure reports. */
41
- get toolDefinitions(): McpToolDefinition[];
42
- /** Takes a turn and returns the model's answer. */
43
- send(prompt: string): Promise<string>;
40
+ static open<App>(host: HostConfig, fetchImpl: TransportFetch, middleware: EvalMiddlewares, dispose?: () => Promise<void>): Promise<Chat<App>>;
41
+ /** Takes a turn. Assert on the result with `expect.chat`. */
42
+ send(content: UserContent): Promise<void>;
44
43
  close(): Promise<void>;
45
44
  private failureFor;
46
45
  private toolSet;
@@ -1 +1 @@
1
- {"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAEA,OAAO,EAIL,KAAK,aAAa,EAGnB,MAAM,IAAI,CAAC;AACZ,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7D;;;GAGG;AACH,MAAM,MAAM,QAAQ,CAAC,GAAG,IAAI;KACzB,IAAI,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG;QACxB,IAAI,EAAE,IAAI,CAAC;QACX,SAAS,EAAE,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAChC,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB;CACF,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;AAElB,UAAU,UAAU;IAClB,KAAK,EAAE,aAAa,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,UAAU,iBAAiB;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACtC;AAID;;;GAGG;AACH,qBAAa,IAAI,CAAC,GAAG,GAAG,OAAO;IAC7B,mEAAmE;IACnE,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAM;IAEzC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA6B;IACtD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAa;IAClC,OAAO,CAAC,KAAK,CAA2B;IAExC,OAAO;WAUM,IAAI,CAAC,GAAG,EACnB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,UAAU,GACf,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAerB,0EAA0E;IAC1E,IAAI,eAAe,IAAI,iBAAiB,EAAE,CAEzC;IAED,mDAAmD;IAC7C,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IA4BrC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,OAAO,CAAC,UAAU;IAalB,OAAO,CAAC,OAAO;CAmChB"}
1
+ {"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAOA,OAAO,EAKL,KAAK,aAAa,EAGlB,KAAK,WAAW,EACjB,MAAM,IAAI,CAAC;AACZ,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAE7D,OAAO,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AAE1D,UAAU,UAAU;IAClB,KAAK,EAAE,aAAa,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAaD;;;GAGG;AACH,MAAM,MAAM,cAAc,GAAG,CAC3B,GAAG,EAAE,MAAM,GAAG,GAAG,EACjB,IAAI,CAAC,EAAE,WAAW,KACf,OAAO,CAAC,QAAQ,CAAC,CAAC;AAEvB;;;GAGG;AACH,qBAAa,IAAI,CAAC,GAAG,GAAG,OAAO;IAsC3B,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO;IAtC1B,4EAA4E;IAC5E,SAAiB,IAAI,EAAE,GAAG,CAAC;IAE3B,0DAA0D;IAC1D,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAM;IAEjC,mEAAmE;IACnE,IAAI,SAAS,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE,CAI/B;IAED,+EAA+E;IAC/E,IAAI,cAAc,IAAI,MAAM,EAAE,CAI7B;IAED,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA6B;IACtD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAa;IAClC,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,YAAY,CAAC,CAAgB;IAErC,mEAAmE;IACnE,IAAI,KAAK,IAAI,aAAa,CAEzB;IAED,OAAO;WAYM,IAAI,CAAC,GAAG,EACnB,IAAI,EAAE,UAAU,EAChB,SAAS,EAAE,cAAc,EACzB,UAAU,EAAE,eAAe,EAC3B,OAAO,GAAE,MAAM,OAAO,CAAC,IAAI,CAAkB,GAC5C,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAmBrB,6DAA6D;IACvD,IAAI,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAsD/C,KAAK;IAeL,OAAO,CAAC,UAAU;IAalB,OAAO,CAAC,OAAO;CAkDhB"}
package/dist/chat.js CHANGED
@@ -1,33 +1,59 @@
1
- import { Client } from "@modelcontextprotocol/sdk/client/index.js";
2
- import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
1
+ import { Client, fromJsonSchema, StreamableHTTPClientTransport, } from "@modelcontextprotocol/client";
3
2
  import { dynamicTool, generateText, jsonSchema, stepCountIs, } from "ai";
3
+ import { registerFileParameters } from "./matchers/matchers.js";
4
4
  const DEFAULT_MAX_STEPS = 8;
5
+ const IN_PROCESS_URL = "http://in-process.skybridge.test/mcp";
6
+ function outputOf(results, toolCallId) {
7
+ return results.find((result) => result.toolCallId === toolCallId)?.output;
8
+ }
5
9
  /**
6
10
  * One conversation against one MCP session. A scenario asserts on
7
11
  * {@link Chat.toolCalls}, the sequence of calls the model made.
8
12
  */
9
13
  export class Chat {
14
+ middleware;
15
+ dispose;
16
+ /** Everything that happened, in the order it happened. */
17
+ turns = [];
10
18
  /** Every call the model made, in order, as it crossed the wire. */
11
- toolCalls = [];
19
+ get toolCalls() {
20
+ return this.turns
21
+ .filter((turn) => turn.role === "tool")
22
+ .map((turn) => turn.call);
23
+ }
24
+ /** What the assistant said, one entry per turn, intermediate ones included. */
25
+ get assistantTurns() {
26
+ return this.turns
27
+ .filter((turn) => turn.role === "assistant")
28
+ .map((turn) => turn.text);
29
+ }
12
30
  messages = [];
13
31
  failures = new Map();
14
32
  client;
15
33
  transport;
16
34
  host;
17
35
  tools = [];
18
- constructor(client, transport, host) {
36
+ closePromise;
37
+ /** The model the conversation runs on, and the judge's default. */
38
+ get model() {
39
+ return this.host.model;
40
+ }
41
+ constructor(client, transport, host, middleware, dispose) {
42
+ this.middleware = middleware;
43
+ this.dispose = dispose;
19
44
  this.client = client;
20
45
  this.transport = transport;
21
46
  this.host = host;
22
47
  }
23
- static async open(serverUrl, host) {
48
+ static async open(host, fetchImpl, middleware, dispose = async () => { }) {
24
49
  const client = new Client({ name: "skybridge-eval", version: "0" });
25
- const transport = new StreamableHTTPClientTransport(new URL(serverUrl));
26
- await client.connect(transport);
27
- const chat = new Chat(client, transport, host);
50
+ const transport = new StreamableHTTPClientTransport(new URL(IN_PROCESS_URL), { fetch: fetchImpl });
51
+ const chat = new Chat(client, transport, host, middleware, dispose);
28
52
  try {
53
+ await client.connect(transport);
29
54
  const { tools } = await client.listTools();
30
55
  chat.tools = tools;
56
+ registerFileParameters(chat, middleware.fileParams);
31
57
  }
32
58
  catch (error) {
33
59
  await transport.close();
@@ -35,16 +61,29 @@ export class Chat {
35
61
  }
36
62
  return chat;
37
63
  }
38
- /** The tool definitions the model was looking at, for failure reports. */
39
- get toolDefinitions() {
40
- return this.tools;
41
- }
42
- /** Takes a turn and returns the model's answer. */
43
- async send(prompt) {
44
- this.messages.push({ role: "user", content: prompt });
64
+ /** Takes a turn. Assert on the result with `expect.chat`. */
65
+ async send(content) {
66
+ if (this.closePromise) {
67
+ throw new Error("The conversation is closed");
68
+ }
69
+ const prepared = await this.middleware.files.prepare(content);
70
+ this.middleware.inputs.clear();
71
+ this.failures.clear();
72
+ this.messages.push({ role: "user", content: prepared });
73
+ this.turns.push({
74
+ role: "user",
75
+ text: typeof prepared === "string"
76
+ ? prepared
77
+ : prepared
78
+ .filter((part) => part.type === "text")
79
+ .map((part) => part.text)
80
+ .join("\n"),
81
+ });
45
82
  const result = await generateText({
46
83
  model: this.host.model,
47
- temperature: this.host.temperature ?? 0,
84
+ ...(this.host.temperature === undefined
85
+ ? {}
86
+ : { temperature: this.host.temperature }),
48
87
  ...(this.host.systemPrompt === undefined
49
88
  ? {}
50
89
  : { system: this.host.systemPrompt }),
@@ -53,19 +92,39 @@ export class Chat {
53
92
  stopWhen: stepCountIs(this.host.maxSteps ?? DEFAULT_MAX_STEPS),
54
93
  });
55
94
  for (const step of result.steps) {
95
+ if (step.text !== "") {
96
+ this.turns.push({ role: "assistant", text: step.text });
97
+ }
56
98
  for (const call of step.toolCalls) {
57
- this.toolCalls.push({
58
- name: call.toolName,
59
- arguments: (call.input ?? {}),
60
- ...this.failureFor(call),
99
+ this.turns.push({
100
+ role: "tool",
101
+ call: {
102
+ name: call.toolName,
103
+ arguments: this.middleware.inputs.get(call.toolCallId) ??
104
+ (call.input ?? {}),
105
+ ...this.failureFor(call),
106
+ },
107
+ result: outputOf(step.toolResults, call.toolCallId),
61
108
  });
62
109
  }
63
110
  }
64
111
  this.messages.push(...result.response.messages);
65
- return result.text;
66
112
  }
67
- async close() {
68
- await this.transport.close();
113
+ close() {
114
+ this.closePromise ??= (async () => {
115
+ try {
116
+ await this.transport.close();
117
+ }
118
+ finally {
119
+ try {
120
+ await this.middleware.files.close();
121
+ }
122
+ finally {
123
+ await this.dispose();
124
+ }
125
+ }
126
+ })();
127
+ return this.closePromise;
69
128
  }
70
129
  failureFor(call) {
71
130
  const reported = this.failures.get(call.toolCallId);
@@ -80,27 +139,42 @@ export class Chat {
80
139
  toolSet() {
81
140
  const entries = this.tools.map((definition) => {
82
141
  const { $schema: _ignored, ...parameters } = definition.inputSchema;
142
+ const schema = fromJsonSchema(parameters);
83
143
  return [
84
144
  definition.name,
85
145
  dynamicTool({
86
146
  description: definition.description ?? "",
87
- inputSchema: jsonSchema(parameters),
147
+ inputSchema: jsonSchema(parameters, {
148
+ validate: async (input) => {
149
+ const result = await schema["~standard"].validate(input);
150
+ return result.issues
151
+ ? {
152
+ success: false,
153
+ error: new Error(JSON.stringify(result.issues)),
154
+ }
155
+ : { success: true, value: result.value };
156
+ },
157
+ }),
88
158
  execute: async (input, { toolCallId }) => {
89
159
  let result;
90
160
  try {
91
161
  result = await this.client.callTool({
92
162
  name: definition.name,
93
163
  arguments: (input ?? {}),
164
+ _meta: { "skybridge/evalToolCallId": toolCallId },
94
165
  });
95
166
  }
96
167
  catch (error) {
97
- this.failures.set(toolCallId, `the call never reached the server: ${error instanceof Error ? error.message : String(error)}`);
168
+ this.failures.set(toolCallId, `the tool call failed: ${error instanceof Error ? error.message : String(error)}`);
98
169
  throw error;
99
170
  }
100
171
  if (result.isError === true) {
101
172
  this.failures.set(toolCallId, `the tool returned an error: ${JSON.stringify(result.content)}`);
102
173
  }
103
- return result.content ?? result;
174
+ return {
175
+ content: result.content,
176
+ structuredContent: result.structuredContent,
177
+ };
104
178
  },
105
179
  }),
106
180
  ];
package/dist/chat.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,6BAA6B,EAAE,MAAM,oDAAoD,CAAC;AACnG,OAAO,EACL,WAAW,EACX,YAAY,EACZ,UAAU,EAGV,WAAW,GACZ,MAAM,IAAI,CAAC;AA4BZ,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B;;;GAGG;AACH,MAAM,OAAO,IAAI;IACf,mEAAmE;IAC1D,SAAS,GAAoB,EAAE,CAAC;IAExB,QAAQ,GAAmB,EAAE,CAAC;IAC9B,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrC,MAAM,CAAS;IACf,SAAS,CAAgC;IACzC,IAAI,CAAa;IAC1B,KAAK,GAAwB,EAAE,CAAC;IAExC,YACE,MAAc,EACd,SAAwC,EACxC,IAAgB;QAEhB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,IAAI,CACf,SAAiB,EACjB,IAAgB;QAEhB,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,IAAI,6BAA6B,CAAC,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;QACxE,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAM,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QACpD,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,GAAG,KAAuC,CAAC;QACvD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YACxB,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,0EAA0E;IAC1E,IAAI,eAAe;QACjB,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED,mDAAmD;IACnD,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAEtD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;YACtB,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC;YACvC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,KAAK,SAAS;gBACtC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE;YACrB,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;SAC/D,CAAC,CAAC;QAEH,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAChC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAClC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;oBAClB,IAAI,EAAE,IAAI,CAAC,QAAQ;oBACnB,SAAS,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAA4B;oBACxD,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;iBACR,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAChD,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;IAEO,UAAU,CAAC,IAA+C;QAGhE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QAC9B,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC1B,OAAO,EAAE,MAAM,EAAE,2CAA2C,EAAE,CAAC;QACjE,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAEO,OAAO;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;YAC5C,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,UAAU,EAAE,GAAG,UAAU,CAAC,WAAW,CAAC;YACpE,OAAO;gBACL,UAAU,CAAC,IAAI;gBACf,WAAW,CAAC;oBACV,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,EAAE;oBACzC,WAAW,EAAE,UAAU,CAAC,UAAU,CAAC;oBACnC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE;wBACvC,IAAI,MAA+C,CAAC;wBACpD,IAAI,CAAC;4BACH,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;gCAClC,IAAI,EAAE,UAAU,CAAC,IAAI;gCACrB,SAAS,EAAE,CAAC,KAAK,IAAI,EAAE,CAA4B;6BACpD,CAAC,CAAC;wBACL,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BACf,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,sCAAsC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC/F,CAAC;4BACF,MAAM,KAAK,CAAC;wBACd,CAAC;wBACD,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;4BAC5B,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,+BAA+B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAChE,CAAC;wBACJ,CAAC;wBACD,OAAO,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC;oBAClC,CAAC;iBACF,CAAC;aACM,CAAC;QACb,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;CACF"}
1
+ {"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,MAAM,EACN,cAAc,EAEd,6BAA6B,GAE9B,MAAM,8BAA8B,CAAC;AACtC,OAAO,EACL,WAAW,EACX,YAAY,EAEZ,UAAU,EAGV,WAAW,GAEZ,MAAM,IAAI,CAAC;AAEZ,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAUhE,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B,MAAM,cAAc,GAAG,sCAAsC,CAAC;AAE9D,SAAS,QAAQ,CACf,OAAmD,EACnD,UAAkB;IAElB,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,KAAK,UAAU,CAAC,EAAE,MAAM,CAAC;AAC5E,CAAC;AAWD;;;GAGG;AACH,MAAM,OAAO,IAAI;IAsCI;IACA;IAnCnB,0DAA0D;IACjD,KAAK,GAAgB,EAAE,CAAC;IAEjC,mEAAmE;IACnE,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,KAAK;aACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC;aACtC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAED,+EAA+E;IAC/E,IAAI,cAAc;QAChB,OAAO,IAAI,CAAC,KAAK;aACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,WAAW,CAAC;aAC3C,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAEgB,QAAQ,GAAmB,EAAE,CAAC;IAC9B,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrC,MAAM,CAAS;IACf,SAAS,CAAgC;IACzC,IAAI,CAAa;IAC1B,KAAK,GAAW,EAAE,CAAC;IACnB,YAAY,CAAiB;IAErC,mEAAmE;IACnE,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;IACzB,CAAC;IAED,YACE,MAAc,EACd,SAAwC,EACxC,IAAgB,EACC,UAA2B,EAC3B,OAA4B;QAD5B,eAAU,GAAV,UAAU,CAAiB;QAC3B,YAAO,GAAP,OAAO,CAAqB;QAE7C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,IAAI,CACf,IAAgB,EAChB,SAAyB,EACzB,UAA2B,EAC3B,UAA+B,KAAK,IAAI,EAAE,GAAE,CAAC;QAE7C,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,IAAI,6BAA6B,CACjD,IAAI,GAAG,CAAC,cAAc,CAAC,EACvB,EAAE,KAAK,EAAE,SAAS,EAAE,CACrB,CAAC;QACF,MAAM,IAAI,GAAG,IAAI,IAAI,CAAM,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;QACzE,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YAChC,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;YACnB,sBAAsB,CAAC,IAAI,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;QACtD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YACxB,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,6DAA6D;IAC7D,KAAK,CAAC,IAAI,CAAC,OAAoB;QAC7B,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAChD,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC9D,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;QACxD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;YACd,IAAI,EAAE,MAAM;YACZ,IAAI,EACF,OAAO,QAAQ,KAAK,QAAQ;gBAC1B,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,QAAQ;qBACL,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC;qBACtC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;qBACxB,IAAI,CAAC,IAAI,CAAC;SACpB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;YACtB,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,KAAK,SAAS;gBACrC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YAC3C,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,KAAK,SAAS;gBACtC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE;YACrB,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;SAC/D,CAAC,CAAC;QAEH,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,IAAI,CAAC,IAAI,KAAK,EAAE,EAAE,CAAC;gBACrB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YAC1D,CAAC;YACD,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAClC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;oBACd,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE;wBACJ,IAAI,EAAE,IAAI,CAAC,QAAQ;wBACnB,SAAS,EACP,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC;4BAC1C,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAA6B;wBACjD,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;qBACR;oBAClB,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,UAAU,CAAC;iBACpD,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClD,CAAC;IAED,KAAK;QACH,IAAI,CAAC,YAAY,KAAK,CAAC,KAAK,IAAI,EAAE;YAChC,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;YAC/B,CAAC;oBAAS,CAAC;gBACT,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;gBACtC,CAAC;wBAAS,CAAC;oBACT,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;gBACvB,CAAC;YACH,CAAC;QACH,CAAC,CAAC,EAAE,CAAC;QACL,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAEO,UAAU,CAAC,IAA+C;QAGhE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QAC9B,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC1B,OAAO,EAAE,MAAM,EAAE,2CAA2C,EAAE,CAAC;QACjE,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAEO,OAAO;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;YAC5C,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,UAAU,EAAE,GAAG,UAAU,CAAC,WAAW,CAAC;YACpE,MAAM,MAAM,GAAG,cAAc,CAAC,UAA4B,CAAC,CAAC;YAC5D,OAAO;gBACL,UAAU,CAAC,IAAI;gBACf,WAAW,CAAC;oBACV,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,EAAE;oBACzC,WAAW,EAAE,UAAU,CAAC,UAAyB,EAAE;wBACjD,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;4BACxB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;4BACzD,OAAO,MAAM,CAAC,MAAM;gCAClB,CAAC,CAAC;oCACE,OAAO,EAAE,KAAK;oCACd,KAAK,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;iCAChD;gCACH,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;wBAC7C,CAAC;qBACF,CAAC;oBACF,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE;wBACvC,IAAI,MAA+C,CAAC;wBACpD,IAAI,CAAC;4BACH,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;gCAClC,IAAI,EAAE,UAAU,CAAC,IAAI;gCACrB,SAAS,EAAE,CAAC,KAAK,IAAI,EAAE,CAA4B;gCACnD,KAAK,EAAE,EAAE,0BAA0B,EAAE,UAAU,EAAE;6BAClD,CAAC,CAAC;wBACL,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BACf,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,yBAAyB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAClF,CAAC;4BACF,MAAM,KAAK,CAAC;wBACd,CAAC;wBACD,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;4BAC5B,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,+BAA+B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAChE,CAAC;wBACJ,CAAC;wBACD,OAAO;4BACL,OAAO,EAAE,MAAM,CAAC,OAAO;4BACvB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;yBAC5C,CAAC;oBACJ,CAAC;iBACF,CAAC;aACM,CAAC;QACb,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;CACF"}
@@ -0,0 +1,12 @@
1
+ import { InputFiles } from "./input-files.js";
2
+ import type { ToolStubs } from "./session-registry.js";
3
+ export declare function createEvalMiddlewares(stubs?: ToolStubs): {
4
+ files: InputFiles;
5
+ inputs: Map<string, Record<string, unknown>>;
6
+ fileParams: Map<string, string[]>;
7
+ listTools: import("skybridge/server").McpMiddlewareFn;
8
+ resolveFiles: import("skybridge/server").McpMiddlewareFn;
9
+ stubTool: import("skybridge/server").McpMiddlewareFn;
10
+ };
11
+ export type EvalMiddlewares = ReturnType<typeof createEvalMiddlewares>;
12
+ //# sourceMappingURL=eval-middlewares.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"eval-middlewares.d.ts","sourceRoot":"","sources":["../src/eval-middlewares.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAI9C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAEvD,wBAAgB,qBAAqB,CAAC,KAAK,GAAE,SAAc;;;;;;;EAa1D;AAED,MAAM,MAAM,eAAe,GAAG,UAAU,CAAC,OAAO,qBAAqB,CAAC,CAAC"}
@@ -0,0 +1,18 @@
1
+ import { InputFiles } from "./input-files.js";
2
+ import { createListToolsMiddleware } from "./middleware/list-tools.js";
3
+ import { createResolveFilesMiddleware } from "./middleware/resolve-files.js";
4
+ import { createStubToolMiddleware } from "./middleware/stub-tool.js";
5
+ export function createEvalMiddlewares(stubs = {}) {
6
+ const files = new InputFiles();
7
+ const inputs = new Map();
8
+ const fileParams = new Map();
9
+ return {
10
+ files,
11
+ inputs,
12
+ fileParams,
13
+ listTools: createListToolsMiddleware(fileParams, stubs),
14
+ resolveFiles: createResolveFilesMiddleware(files, fileParams, inputs),
15
+ stubTool: createStubToolMiddleware(stubs),
16
+ };
17
+ }
18
+ //# sourceMappingURL=eval-middlewares.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"eval-middlewares.js","sourceRoot":"","sources":["../src/eval-middlewares.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AACvE,OAAO,EAAE,4BAA4B,EAAE,MAAM,+BAA+B,CAAC;AAC7E,OAAO,EAAE,wBAAwB,EAAE,MAAM,2BAA2B,CAAC;AAGrE,MAAM,UAAU,qBAAqB,CAAC,QAAmB,EAAE;IACzD,MAAM,KAAK,GAAG,IAAI,UAAU,EAAE,CAAC;IAC/B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAmC,CAAC;IAC1D,MAAM,UAAU,GAAG,IAAI,GAAG,EAAoB,CAAC;IAE/C,OAAO;QACL,KAAK;QACL,MAAM;QACN,UAAU;QACV,SAAS,EAAE,yBAAyB,CAAC,UAAU,EAAE,KAAK,CAAC;QACvD,YAAY,EAAE,4BAA4B,CAAC,KAAK,EAAE,UAAU,EAAE,MAAM,CAAC;QACrE,QAAQ,EAAE,wBAAwB,CAAC,KAAK,CAAC;KAC1C,CAAC;AACJ,CAAC"}
package/dist/index.d.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  export type { Chat } from "./chat.js";
2
- export type { ChatAssertion, ChatMatchers } from "./matchers.js";
3
- export { defineEvalModel } from "./model.js";
2
+ export type { ChatAssertion, ChatMatchers, Judgment, JudgmentOptions, Verdict, } from "./matchers/index.js";
3
+ export type { EvalApp, EvalIdentity, ToolStubs, } from "./session-registry.js";
4
4
  export { start } from "./session-registry.js";
5
5
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACtC,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AACjE,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACtC,YAAY,EACV,aAAa,EACb,YAAY,EACZ,QAAQ,EACR,eAAe,EACf,OAAO,GACR,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EACV,OAAO,EACP,YAAY,EACZ,SAAS,GACV,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
package/dist/index.js CHANGED
@@ -1,3 +1,2 @@
1
- export { defineEvalModel } from "./model.js";
2
1
  export { start } from "./session-registry.js";
3
2
  //# sourceMappingURL=index.js.map
package/dist/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAaA,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
@@ -0,0 +1,14 @@
1
+ import type { UserContent } from "ai";
2
+ export declare class InputFiles {
3
+ private closed;
4
+ private listening?;
5
+ private readonly files;
6
+ private readonly paths;
7
+ private readonly server;
8
+ prepare(content: UserContent): Promise<UserContent>;
9
+ rewrite(input: Record<string, unknown>, fields: string[]): Promise<{
10
+ [x: string]: unknown;
11
+ }>;
12
+ close(): Promise<void>;
13
+ }
14
+ //# sourceMappingURL=input-files.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"input-files.d.ts","sourceRoot":"","sources":["../src/input-files.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAY,WAAW,EAAE,MAAM,IAAI,CAAC;AAkChD,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAC,CAAgB;IAClC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAGlB;IACJ,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA8B;IACpD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAQpB;IAEG,OAAO,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAyCnD,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE;;;IAkCxD,KAAK;CASZ"}