@skybridge/test 0.0.0-dev.8ceb3ae → 0.0.0-dev.8e34da3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -13
- package/dist/chat.d.ts +21 -6
- package/dist/chat.d.ts.map +1 -1
- package/dist/chat.js +98 -25
- package/dist/chat.js.map +1 -1
- package/dist/eval-middlewares.d.ts +12 -0
- package/dist/eval-middlewares.d.ts.map +1 -0
- package/dist/eval-middlewares.js +18 -0
- package/dist/eval-middlewares.js.map +1 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/input-files.d.ts +14 -0
- package/dist/input-files.d.ts.map +1 -0
- package/dist/input-files.js +126 -0
- package/dist/input-files.js.map +1 -0
- package/dist/matchers/index.d.ts +4 -0
- package/dist/matchers/index.d.ts.map +1 -0
- package/dist/matchers/index.js +2 -0
- package/dist/matchers/index.js.map +1 -0
- package/dist/matchers/matchers.d.ts +54 -0
- package/dist/matchers/matchers.d.ts.map +1 -0
- package/dist/matchers/matchers.js +258 -0
- package/dist/matchers/matchers.js.map +1 -0
- package/dist/matchers/types.d.ts +44 -0
- package/dist/matchers/types.d.ts.map +1 -0
- package/dist/matchers/types.js +2 -0
- package/dist/matchers/types.js.map +1 -0
- package/dist/middleware/list-tools.d.ts +4 -0
- package/dist/middleware/list-tools.d.ts.map +1 -0
- package/dist/middleware/list-tools.js +67 -0
- package/dist/middleware/list-tools.js.map +1 -0
- package/dist/middleware/resolve-files.d.ts +4 -0
- package/dist/middleware/resolve-files.d.ts.map +1 -0
- package/dist/middleware/resolve-files.js +13 -0
- package/dist/middleware/resolve-files.js.map +1 -0
- package/dist/middleware/stub-tool.d.ts +4 -0
- package/dist/middleware/stub-tool.d.ts.map +1 -0
- package/dist/middleware/stub-tool.js +16 -0
- package/dist/middleware/stub-tool.js.map +1 -0
- package/dist/session-registry.d.ts +49 -9
- package/dist/session-registry.d.ts.map +1 -1
- package/dist/session-registry.js +48 -17
- package/dist/session-registry.js.map +1 -1
- package/package.json +13 -6
package/README.md
CHANGED
|
@@ -1,20 +1,23 @@
|
|
|
1
1
|
# @skybridge/test
|
|
2
2
|
|
|
3
|
-
Evals for [Skybridge](https://skybridge.tech) MCP
|
|
3
|
+
Evals for [Skybridge](https://skybridge.tech) MCP Apps: they answer whether a
|
|
4
4
|
prompt reaches the right tool with the right arguments, which no unit test can
|
|
5
5
|
check.
|
|
6
6
|
|
|
7
|
-
Add the `evals` option to the Skybridge Vite plugin
|
|
8
|
-
vitest's own `it` and `expect`.
|
|
7
|
+
Add the `evals` option to the Skybridge Vite plugin so `expect.chat` exists,
|
|
8
|
+
then write scenarios with vitest's own `it` and `expect`. Pass the app itself
|
|
9
|
+
to `start`: it is served in-process, using your exported app, and
|
|
10
|
+
the tool names and argument shapes come from the app value, so no type
|
|
11
|
+
parameter is needed.
|
|
9
12
|
|
|
10
13
|
```ts
|
|
11
14
|
import { anthropic } from "@ai-sdk/anthropic";
|
|
12
15
|
import { expect, it } from "vitest";
|
|
13
16
|
import { start } from "@skybridge/test";
|
|
14
|
-
import
|
|
17
|
+
import { app } from "../src/server.js";
|
|
15
18
|
|
|
16
19
|
it("finds products by category", async () => {
|
|
17
|
-
const chat = await start
|
|
20
|
+
const chat = await start({ app, model: anthropic("claude-sonnet-4-5") });
|
|
18
21
|
await chat.send("I am looking for ski goggles");
|
|
19
22
|
|
|
20
23
|
expect.chat(chat).toHaveCalledToolWith("search-products", {
|
|
@@ -23,6 +26,30 @@ it("finds products by category", async () => {
|
|
|
23
26
|
});
|
|
24
27
|
```
|
|
25
28
|
|
|
29
|
+
There is no child process and no server to boot, so the evals run anywhere
|
|
30
|
+
vitest does. The caveat is that they bypass the Express layer, so nothing
|
|
31
|
+
verifies a bearer token; pass `authInfo` to claim an identity.
|
|
32
|
+
|
|
33
|
+
## Authenticated evals
|
|
34
|
+
|
|
35
|
+
```ts
|
|
36
|
+
const chat = await start({
|
|
37
|
+
app,
|
|
38
|
+
model: anthropic("claude-sonnet-4-5"),
|
|
39
|
+
authInfo: {
|
|
40
|
+
token: "eval-token",
|
|
41
|
+
clientId: "eval-client",
|
|
42
|
+
scopes: ["checkout"],
|
|
43
|
+
extra: { sub: "user_123", email: "ada@example.com" },
|
|
44
|
+
},
|
|
45
|
+
});
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
`authInfo` injects a claimed identity: the app's own scheme and scope
|
|
49
|
+
enforcement runs for real, in-band challenges included, and only token
|
|
50
|
+
signature verification is skipped. Omit it to eval the anonymous path, which is
|
|
51
|
+
how you assert that a protected tool challenges instead of running.
|
|
52
|
+
|
|
26
53
|
## Choosing a model
|
|
27
54
|
|
|
28
55
|
`start` takes any AI SDK model instance, so the provider and its credentials
|
|
@@ -32,7 +59,7 @@ are the project's own. Install the provider package you want alongside `ai`.
|
|
|
32
59
|
import { anthropic } from "@ai-sdk/anthropic";
|
|
33
60
|
import { start } from "@skybridge/test";
|
|
34
61
|
|
|
35
|
-
const chat = await start
|
|
62
|
+
const chat = await start({ app, model: anthropic("claude-sonnet-4-5") });
|
|
36
63
|
```
|
|
37
64
|
|
|
38
65
|
A local runtime or a company gateway works the same way, through the AI SDK's
|
|
@@ -45,19 +72,17 @@ const ollama = createOpenAICompatible({
|
|
|
45
72
|
name: "ollama",
|
|
46
73
|
baseURL: "http://localhost:11434/v1",
|
|
47
74
|
});
|
|
48
|
-
const chat = await start
|
|
75
|
+
const chat = await start({ app, model: ollama("llama3.1") });
|
|
49
76
|
```
|
|
50
77
|
|
|
51
|
-
|
|
78
|
+
The plugin's `evals` option also carries the defaults every conversation starts
|
|
79
|
+
from, so a project can set them once instead of per scenario.
|
|
52
80
|
|
|
53
81
|
```ts
|
|
54
|
-
skybridge({
|
|
55
|
-
evals: { project: { cwd: ".", command: ["node", "dist/server.js"] } },
|
|
56
|
-
});
|
|
82
|
+
skybridge({ evals: { temperature: 0, maxSteps: 6 } });
|
|
57
83
|
```
|
|
58
84
|
|
|
59
|
-
|
|
60
|
-
`pnpm run` do not forward the channel the runner uses to learn the port.
|
|
85
|
+
## Matchers
|
|
61
86
|
|
|
62
87
|
Assertions on what the assistant said work the same way, though model prose
|
|
63
88
|
varies between runs, so keep them loose.
|
|
@@ -69,3 +94,33 @@ expect.chat(chat).not.toHaveSaid("refund");
|
|
|
69
94
|
|
|
70
95
|
See the [documentation](https://docs.skybridge.tech) for setup and the full
|
|
71
96
|
matcher list.
|
|
97
|
+
|
|
98
|
+
## Input files
|
|
99
|
+
|
|
100
|
+
`chat.send` also accepts AI SDK `UserContent`. Attach a `FilePart` and use the same object in `expect.inputFile` to check which file a tool received:
|
|
101
|
+
|
|
102
|
+
```ts
|
|
103
|
+
import { readFile } from "node:fs/promises";
|
|
104
|
+
import type { FilePart } from "ai";
|
|
105
|
+
|
|
106
|
+
const portrait: FilePart = {
|
|
107
|
+
type: "file",
|
|
108
|
+
mediaType: "image/png",
|
|
109
|
+
filename: "photo.png",
|
|
110
|
+
data: await readFile(new URL("./fixtures/photo.png", import.meta.url)),
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
await chat.send([{ type: "text", text: "Resize this photo" }, portrait]);
|
|
114
|
+
|
|
115
|
+
expect.chat(chat).toHaveCalledToolWith("image_crop_and_resize", {
|
|
116
|
+
file: expect.inputFile(portrait),
|
|
117
|
+
});
|
|
118
|
+
expect.chat(chat).toHaveCalledToolOnce("asset_openai_file_upload", {
|
|
119
|
+
files: [expect.inputFile(portrait)],
|
|
120
|
+
});
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
`file` and `files` must be listed in their tools' `_meta["openai/fileParams"]`.
|
|
124
|
+
Misplaced file matchers throw immediately, including nested matchers and `.not`.
|
|
125
|
+
Declare file parameters with `FileRef` and list their names in `_meta["openai/fileParams"]`. Optional, nullable, and array forms are supported. Your handler or stub receives the attached file; downloads remain available until the conversation closes.
|
|
126
|
+
See [Evals](https://docs.skybridge.tech/test/evals) for how files reach the model and the tool.
|
package/dist/chat.d.ts
CHANGED
|
@@ -1,30 +1,45 @@
|
|
|
1
|
-
import type
|
|
2
|
-
import {
|
|
1
|
+
import { type LanguageModel, type UserContent } from "ai";
|
|
2
|
+
import type { EvalMiddlewares } from "./eval-middlewares.js";
|
|
3
|
+
import type { ToolCall, Turn } from "./matchers/types.js";
|
|
3
4
|
interface HostConfig {
|
|
4
5
|
model: LanguageModel;
|
|
5
6
|
temperature?: number;
|
|
6
7
|
systemPrompt?: string;
|
|
7
8
|
maxSteps?: number;
|
|
8
9
|
}
|
|
10
|
+
/**
|
|
11
|
+
* A `fetch` replacement the transport dials instead of the network. Mirrors the
|
|
12
|
+
* MCP client's own `fetch` option so an in-process app can serve the session.
|
|
13
|
+
*/
|
|
14
|
+
export type TransportFetch = (url: string | URL, init?: RequestInit) => Promise<Response>;
|
|
9
15
|
/**
|
|
10
16
|
* One conversation against one MCP session. A scenario asserts on
|
|
11
17
|
* {@link Chat.toolCalls}, the sequence of calls the model made.
|
|
12
18
|
*/
|
|
13
19
|
export declare class Chat<App = unknown> {
|
|
20
|
+
private readonly middleware;
|
|
21
|
+
private readonly dispose;
|
|
22
|
+
/** Type-only anchor so `expect.chat` infers `App` from the conversation. */
|
|
23
|
+
readonly $app: App;
|
|
24
|
+
/** Everything that happened, in the order it happened. */
|
|
25
|
+
readonly turns: Turn<App>[];
|
|
14
26
|
/** Every call the model made, in order, as it crossed the wire. */
|
|
15
|
-
|
|
27
|
+
get toolCalls(): ToolCall<App>[];
|
|
16
28
|
/** What the assistant said, one entry per turn, intermediate ones included. */
|
|
17
|
-
|
|
29
|
+
get assistantTurns(): string[];
|
|
18
30
|
private readonly messages;
|
|
19
31
|
private readonly failures;
|
|
20
32
|
private readonly client;
|
|
21
33
|
private readonly transport;
|
|
22
34
|
private readonly host;
|
|
23
35
|
private tools;
|
|
36
|
+
private closePromise?;
|
|
37
|
+
/** The model the conversation runs on, and the judge's default. */
|
|
38
|
+
get model(): LanguageModel;
|
|
24
39
|
private constructor();
|
|
25
|
-
static open<App>(
|
|
40
|
+
static open<App>(host: HostConfig, fetchImpl: TransportFetch, middleware: EvalMiddlewares, dispose?: () => Promise<void>): Promise<Chat<App>>;
|
|
26
41
|
/** Takes a turn. Assert on the result with `expect.chat`. */
|
|
27
|
-
send(
|
|
42
|
+
send(content: UserContent): Promise<void>;
|
|
28
43
|
close(): Promise<void>;
|
|
29
44
|
private failureFor;
|
|
30
45
|
private toolSet;
|
package/dist/chat.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAOA,OAAO,EAKL,KAAK,aAAa,EAGlB,KAAK,WAAW,EACjB,MAAM,IAAI,CAAC;AACZ,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAE7D,OAAO,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AAE1D,UAAU,UAAU;IAClB,KAAK,EAAE,aAAa,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAaD;;;GAGG;AACH,MAAM,MAAM,cAAc,GAAG,CAC3B,GAAG,EAAE,MAAM,GAAG,GAAG,EACjB,IAAI,CAAC,EAAE,WAAW,KACf,OAAO,CAAC,QAAQ,CAAC,CAAC;AAEvB;;;GAGG;AACH,qBAAa,IAAI,CAAC,GAAG,GAAG,OAAO;IAsC3B,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO;IAtC1B,4EAA4E;IAC5E,SAAiB,IAAI,EAAE,GAAG,CAAC;IAE3B,0DAA0D;IAC1D,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAM;IAEjC,mEAAmE;IACnE,IAAI,SAAS,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE,CAI/B;IAED,+EAA+E;IAC/E,IAAI,cAAc,IAAI,MAAM,EAAE,CAI7B;IAED,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA6B;IACtD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAa;IAClC,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,YAAY,CAAC,CAAgB;IAErC,mEAAmE;IACnE,IAAI,KAAK,IAAI,aAAa,CAEzB;IAED,OAAO;WAYM,IAAI,CAAC,GAAG,EACnB,IAAI,EAAE,UAAU,EAChB,SAAS,EAAE,cAAc,EACzB,UAAU,EAAE,eAAe,EAC3B,OAAO,GAAE,MAAM,OAAO,CAAC,IAAI,CAAkB,GAC5C,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAmBrB,6DAA6D;IACvD,IAAI,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAsD/C,KAAK;IAeL,OAAO,CAAC,UAAU;IAalB,OAAO,CAAC,OAAO;CAkDhB"}
|
package/dist/chat.js
CHANGED
|
@@ -1,34 +1,59 @@
|
|
|
1
|
-
import { Client, StreamableHTTPClientTransport, } from "@modelcontextprotocol/client";
|
|
1
|
+
import { Client, fromJsonSchema, StreamableHTTPClientTransport, } from "@modelcontextprotocol/client";
|
|
2
2
|
import { dynamicTool, generateText, jsonSchema, stepCountIs, } from "ai";
|
|
3
|
+
import { registerFileParameters } from "./matchers/matchers.js";
|
|
3
4
|
const DEFAULT_MAX_STEPS = 8;
|
|
5
|
+
const IN_PROCESS_URL = "http://in-process.skybridge.test/mcp";
|
|
6
|
+
function outputOf(results, toolCallId) {
|
|
7
|
+
return results.find((result) => result.toolCallId === toolCallId)?.output;
|
|
8
|
+
}
|
|
4
9
|
/**
|
|
5
10
|
* One conversation against one MCP session. A scenario asserts on
|
|
6
11
|
* {@link Chat.toolCalls}, the sequence of calls the model made.
|
|
7
12
|
*/
|
|
8
13
|
export class Chat {
|
|
14
|
+
middleware;
|
|
15
|
+
dispose;
|
|
16
|
+
/** Everything that happened, in the order it happened. */
|
|
17
|
+
turns = [];
|
|
9
18
|
/** Every call the model made, in order, as it crossed the wire. */
|
|
10
|
-
toolCalls
|
|
19
|
+
get toolCalls() {
|
|
20
|
+
return this.turns
|
|
21
|
+
.filter((turn) => turn.role === "tool")
|
|
22
|
+
.map((turn) => turn.call);
|
|
23
|
+
}
|
|
11
24
|
/** What the assistant said, one entry per turn, intermediate ones included. */
|
|
12
|
-
assistantTurns
|
|
25
|
+
get assistantTurns() {
|
|
26
|
+
return this.turns
|
|
27
|
+
.filter((turn) => turn.role === "assistant")
|
|
28
|
+
.map((turn) => turn.text);
|
|
29
|
+
}
|
|
13
30
|
messages = [];
|
|
14
31
|
failures = new Map();
|
|
15
32
|
client;
|
|
16
33
|
transport;
|
|
17
34
|
host;
|
|
18
35
|
tools = [];
|
|
19
|
-
|
|
36
|
+
closePromise;
|
|
37
|
+
/** The model the conversation runs on, and the judge's default. */
|
|
38
|
+
get model() {
|
|
39
|
+
return this.host.model;
|
|
40
|
+
}
|
|
41
|
+
constructor(client, transport, host, middleware, dispose) {
|
|
42
|
+
this.middleware = middleware;
|
|
43
|
+
this.dispose = dispose;
|
|
20
44
|
this.client = client;
|
|
21
45
|
this.transport = transport;
|
|
22
46
|
this.host = host;
|
|
23
47
|
}
|
|
24
|
-
static async open(
|
|
48
|
+
static async open(host, fetchImpl, middleware, dispose = async () => { }) {
|
|
25
49
|
const client = new Client({ name: "skybridge-eval", version: "0" });
|
|
26
|
-
const transport = new StreamableHTTPClientTransport(new URL(
|
|
27
|
-
|
|
28
|
-
const chat = new Chat(client, transport, host);
|
|
50
|
+
const transport = new StreamableHTTPClientTransport(new URL(IN_PROCESS_URL), { fetch: fetchImpl });
|
|
51
|
+
const chat = new Chat(client, transport, host, middleware, dispose);
|
|
29
52
|
try {
|
|
53
|
+
await client.connect(transport);
|
|
30
54
|
const { tools } = await client.listTools();
|
|
31
55
|
chat.tools = tools;
|
|
56
|
+
registerFileParameters(chat, middleware.fileParams);
|
|
32
57
|
}
|
|
33
58
|
catch (error) {
|
|
34
59
|
await transport.close();
|
|
@@ -37,11 +62,28 @@ export class Chat {
|
|
|
37
62
|
return chat;
|
|
38
63
|
}
|
|
39
64
|
/** Takes a turn. Assert on the result with `expect.chat`. */
|
|
40
|
-
async send(
|
|
41
|
-
this.
|
|
65
|
+
async send(content) {
|
|
66
|
+
if (this.closePromise) {
|
|
67
|
+
throw new Error("The conversation is closed");
|
|
68
|
+
}
|
|
69
|
+
const prepared = await this.middleware.files.prepare(content);
|
|
70
|
+
this.middleware.inputs.clear();
|
|
71
|
+
this.failures.clear();
|
|
72
|
+
this.messages.push({ role: "user", content: prepared });
|
|
73
|
+
this.turns.push({
|
|
74
|
+
role: "user",
|
|
75
|
+
text: typeof prepared === "string"
|
|
76
|
+
? prepared
|
|
77
|
+
: prepared
|
|
78
|
+
.filter((part) => part.type === "text")
|
|
79
|
+
.map((part) => part.text)
|
|
80
|
+
.join("\n"),
|
|
81
|
+
});
|
|
42
82
|
const result = await generateText({
|
|
43
83
|
model: this.host.model,
|
|
44
|
-
|
|
84
|
+
...(this.host.temperature === undefined
|
|
85
|
+
? {}
|
|
86
|
+
: { temperature: this.host.temperature }),
|
|
45
87
|
...(this.host.systemPrompt === undefined
|
|
46
88
|
? {}
|
|
47
89
|
: { system: this.host.systemPrompt }),
|
|
@@ -50,23 +92,39 @@ export class Chat {
|
|
|
50
92
|
stopWhen: stepCountIs(this.host.maxSteps ?? DEFAULT_MAX_STEPS),
|
|
51
93
|
});
|
|
52
94
|
for (const step of result.steps) {
|
|
95
|
+
if (step.text !== "") {
|
|
96
|
+
this.turns.push({ role: "assistant", text: step.text });
|
|
97
|
+
}
|
|
53
98
|
for (const call of step.toolCalls) {
|
|
54
|
-
this.
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
99
|
+
this.turns.push({
|
|
100
|
+
role: "tool",
|
|
101
|
+
call: {
|
|
102
|
+
name: call.toolName,
|
|
103
|
+
arguments: this.middleware.inputs.get(call.toolCallId) ??
|
|
104
|
+
(call.input ?? {}),
|
|
105
|
+
...this.failureFor(call),
|
|
106
|
+
},
|
|
107
|
+
result: outputOf(step.toolResults, call.toolCallId),
|
|
58
108
|
});
|
|
59
109
|
}
|
|
60
110
|
}
|
|
61
|
-
for (const step of result.steps) {
|
|
62
|
-
if (step.text !== "") {
|
|
63
|
-
this.assistantTurns.push(step.text);
|
|
64
|
-
}
|
|
65
|
-
}
|
|
66
111
|
this.messages.push(...result.response.messages);
|
|
67
112
|
}
|
|
68
|
-
|
|
69
|
-
|
|
113
|
+
close() {
|
|
114
|
+
this.closePromise ??= (async () => {
|
|
115
|
+
try {
|
|
116
|
+
await this.transport.close();
|
|
117
|
+
}
|
|
118
|
+
finally {
|
|
119
|
+
try {
|
|
120
|
+
await this.middleware.files.close();
|
|
121
|
+
}
|
|
122
|
+
finally {
|
|
123
|
+
await this.dispose();
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
})();
|
|
127
|
+
return this.closePromise;
|
|
70
128
|
}
|
|
71
129
|
failureFor(call) {
|
|
72
130
|
const reported = this.failures.get(call.toolCallId);
|
|
@@ -81,27 +139,42 @@ export class Chat {
|
|
|
81
139
|
toolSet() {
|
|
82
140
|
const entries = this.tools.map((definition) => {
|
|
83
141
|
const { $schema: _ignored, ...parameters } = definition.inputSchema;
|
|
142
|
+
const schema = fromJsonSchema(parameters);
|
|
84
143
|
return [
|
|
85
144
|
definition.name,
|
|
86
145
|
dynamicTool({
|
|
87
146
|
description: definition.description ?? "",
|
|
88
|
-
inputSchema: jsonSchema(parameters
|
|
147
|
+
inputSchema: jsonSchema(parameters, {
|
|
148
|
+
validate: async (input) => {
|
|
149
|
+
const result = await schema["~standard"].validate(input);
|
|
150
|
+
return result.issues
|
|
151
|
+
? {
|
|
152
|
+
success: false,
|
|
153
|
+
error: new Error(JSON.stringify(result.issues)),
|
|
154
|
+
}
|
|
155
|
+
: { success: true, value: result.value };
|
|
156
|
+
},
|
|
157
|
+
}),
|
|
89
158
|
execute: async (input, { toolCallId }) => {
|
|
90
159
|
let result;
|
|
91
160
|
try {
|
|
92
161
|
result = await this.client.callTool({
|
|
93
162
|
name: definition.name,
|
|
94
163
|
arguments: (input ?? {}),
|
|
164
|
+
_meta: { "skybridge/evalToolCallId": toolCallId },
|
|
95
165
|
});
|
|
96
166
|
}
|
|
97
167
|
catch (error) {
|
|
98
|
-
this.failures.set(toolCallId, `the call
|
|
168
|
+
this.failures.set(toolCallId, `the tool call failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
99
169
|
throw error;
|
|
100
170
|
}
|
|
101
171
|
if (result.isError === true) {
|
|
102
172
|
this.failures.set(toolCallId, `the tool returned an error: ${JSON.stringify(result.content)}`);
|
|
103
173
|
}
|
|
104
|
-
return
|
|
174
|
+
return {
|
|
175
|
+
content: result.content,
|
|
176
|
+
structuredContent: result.structuredContent,
|
|
177
|
+
};
|
|
105
178
|
},
|
|
106
179
|
}),
|
|
107
180
|
];
|
package/dist/chat.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,MAAM,EACN,6BAA6B,GAE9B,MAAM,8BAA8B,CAAC;
|
|
1
|
+
{"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,MAAM,EACN,cAAc,EAEd,6BAA6B,GAE9B,MAAM,8BAA8B,CAAC;AACtC,OAAO,EACL,WAAW,EACX,YAAY,EAEZ,UAAU,EAGV,WAAW,GAEZ,MAAM,IAAI,CAAC;AAEZ,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAUhE,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B,MAAM,cAAc,GAAG,sCAAsC,CAAC;AAE9D,SAAS,QAAQ,CACf,OAAmD,EACnD,UAAkB;IAElB,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,KAAK,UAAU,CAAC,EAAE,MAAM,CAAC;AAC5E,CAAC;AAWD;;;GAGG;AACH,MAAM,OAAO,IAAI;IAsCI;IACA;IAnCnB,0DAA0D;IACjD,KAAK,GAAgB,EAAE,CAAC;IAEjC,mEAAmE;IACnE,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,KAAK;aACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC;aACtC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAED,+EAA+E;IAC/E,IAAI,cAAc;QAChB,OAAO,IAAI,CAAC,KAAK;aACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,WAAW,CAAC;aAC3C,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAEgB,QAAQ,GAAmB,EAAE,CAAC;IAC9B,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrC,MAAM,CAAS;IACf,SAAS,CAAgC;IACzC,IAAI,CAAa;IAC1B,KAAK,GAAW,EAAE,CAAC;IACnB,YAAY,CAAiB;IAErC,mEAAmE;IACnE,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;IACzB,CAAC;IAED,YACE,MAAc,EACd,SAAwC,EACxC,IAAgB,EACC,UAA2B,EAC3B,OAA4B;QAD5B,eAAU,GAAV,UAAU,CAAiB;QAC3B,YAAO,GAAP,OAAO,CAAqB;QAE7C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,IAAI,CACf,IAAgB,EAChB,SAAyB,EACzB,UAA2B,EAC3B,UAA+B,KAAK,IAAI,EAAE,GAAE,CAAC;QAE7C,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,IAAI,6BAA6B,CACjD,IAAI,GAAG,CAAC,cAAc,CAAC,EACvB,EAAE,KAAK,EAAE,SAAS,EAAE,CACrB,CAAC;QACF,MAAM,IAAI,GAAG,IAAI,IAAI,CAAM,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;QACzE,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YAChC,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;YACnB,sBAAsB,CAAC,IAAI,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;QACtD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YACxB,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,6DAA6D;IAC7D,KAAK,CAAC,IAAI,CAAC,OAAoB;QAC7B,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAChD,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC9D,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;QACxD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;YACd,IAAI,EAAE,MAAM;YACZ,IAAI,EACF,OAAO,QAAQ,KAAK,QAAQ;gBAC1B,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,QAAQ;qBACL,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC;qBACtC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;qBACxB,IAAI,CAAC,IAAI,CAAC;SACpB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;YACtB,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,KAAK,SAAS;gBACrC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YAC3C,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,KAAK,SAAS;gBACtC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE;YACrB,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;SAC/D,CAAC,CAAC;QAEH,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,IAAI,CAAC,IAAI,KAAK,EAAE,EAAE,CAAC;gBACrB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YAC1D,CAAC;YACD,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAClC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;oBACd,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE;wBACJ,IAAI,EAAE,IAAI,CAAC,QAAQ;wBACnB,SAAS,EACP,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC;4BAC1C,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAA6B;wBACjD,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;qBACR;oBAClB,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,UAAU,CAAC;iBACpD,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClD,CAAC;IAED,KAAK;QACH,IAAI,CAAC,YAAY,KAAK,CAAC,KAAK,IAAI,EAAE;YAChC,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;YAC/B,CAAC;oBAAS,CAAC;gBACT,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;gBACtC,CAAC;wBAAS,CAAC;oBACT,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;gBACvB,CAAC;YACH,CAAC;QACH,CAAC,CAAC,EAAE,CAAC;QACL,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAEO,UAAU,CAAC,IAA+C;QAGhE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QAC9B,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC1B,OAAO,EAAE,MAAM,EAAE,2CAA2C,EAAE,CAAC;QACjE,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAEO,OAAO;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;YAC5C,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,UAAU,EAAE,GAAG,UAAU,CAAC,WAAW,CAAC;YACpE,MAAM,MAAM,GAAG,cAAc,CAAC,UAA4B,CAAC,CAAC;YAC5D,OAAO;gBACL,UAAU,CAAC,IAAI;gBACf,WAAW,CAAC;oBACV,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,EAAE;oBACzC,WAAW,EAAE,UAAU,CAAC,UAAyB,EAAE;wBACjD,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;4BACxB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;4BACzD,OAAO,MAAM,CAAC,MAAM;gCAClB,CAAC,CAAC;oCACE,OAAO,EAAE,KAAK;oCACd,KAAK,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;iCAChD;gCACH,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;wBAC7C,CAAC;qBACF,CAAC;oBACF,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE;wBACvC,IAAI,MAA+C,CAAC;wBACpD,IAAI,CAAC;4BACH,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;gCAClC,IAAI,EAAE,UAAU,CAAC,IAAI;gCACrB,SAAS,EAAE,CAAC,KAAK,IAAI,EAAE,CAA4B;gCACnD,KAAK,EAAE,EAAE,0BAA0B,EAAE,UAAU,EAAE;6BAClD,CAAC,CAAC;wBACL,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BACf,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,yBAAyB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAClF,CAAC;4BACF,MAAM,KAAK,CAAC;wBACd,CAAC;wBACD,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;4BAC5B,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,+BAA+B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAChE,CAAC;wBACJ,CAAC;wBACD,OAAO;4BACL,OAAO,EAAE,MAAM,CAAC,OAAO;4BACvB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;yBAC5C,CAAC;oBACJ,CAAC;iBACF,CAAC;aACM,CAAC;QACb,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;CACF"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { InputFiles } from "./input-files.js";
|
|
2
|
+
import type { ToolStubs } from "./session-registry.js";
|
|
3
|
+
export declare function createEvalMiddlewares(stubs?: ToolStubs): {
|
|
4
|
+
files: InputFiles;
|
|
5
|
+
inputs: Map<string, Record<string, unknown>>;
|
|
6
|
+
fileParams: Map<string, string[]>;
|
|
7
|
+
listTools: import("skybridge/server").McpMiddlewareFn;
|
|
8
|
+
resolveFiles: import("skybridge/server").McpMiddlewareFn;
|
|
9
|
+
stubTool: import("skybridge/server").McpMiddlewareFn;
|
|
10
|
+
};
|
|
11
|
+
export type EvalMiddlewares = ReturnType<typeof createEvalMiddlewares>;
|
|
12
|
+
//# sourceMappingURL=eval-middlewares.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval-middlewares.d.ts","sourceRoot":"","sources":["../src/eval-middlewares.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAI9C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAEvD,wBAAgB,qBAAqB,CAAC,KAAK,GAAE,SAAc;;;;;;;EAa1D;AAED,MAAM,MAAM,eAAe,GAAG,UAAU,CAAC,OAAO,qBAAqB,CAAC,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { InputFiles } from "./input-files.js";
|
|
2
|
+
import { createListToolsMiddleware } from "./middleware/list-tools.js";
|
|
3
|
+
import { createResolveFilesMiddleware } from "./middleware/resolve-files.js";
|
|
4
|
+
import { createStubToolMiddleware } from "./middleware/stub-tool.js";
|
|
5
|
+
export function createEvalMiddlewares(stubs = {}) {
|
|
6
|
+
const files = new InputFiles();
|
|
7
|
+
const inputs = new Map();
|
|
8
|
+
const fileParams = new Map();
|
|
9
|
+
return {
|
|
10
|
+
files,
|
|
11
|
+
inputs,
|
|
12
|
+
fileParams,
|
|
13
|
+
listTools: createListToolsMiddleware(fileParams, stubs),
|
|
14
|
+
resolveFiles: createResolveFilesMiddleware(files, fileParams, inputs),
|
|
15
|
+
stubTool: createStubToolMiddleware(stubs),
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
//# sourceMappingURL=eval-middlewares.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval-middlewares.js","sourceRoot":"","sources":["../src/eval-middlewares.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AACvE,OAAO,EAAE,4BAA4B,EAAE,MAAM,+BAA+B,CAAC;AAC7E,OAAO,EAAE,wBAAwB,EAAE,MAAM,2BAA2B,CAAC;AAGrE,MAAM,UAAU,qBAAqB,CAAC,QAAmB,EAAE;IACzD,MAAM,KAAK,GAAG,IAAI,UAAU,EAAE,CAAC;IAC/B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAmC,CAAC;IAC1D,MAAM,UAAU,GAAG,IAAI,GAAG,EAAoB,CAAC;IAE/C,OAAO;QACL,KAAK;QACL,MAAM;QACN,UAAU;QACV,SAAS,EAAE,yBAAyB,CAAC,UAAU,EAAE,KAAK,CAAC;QACvD,YAAY,EAAE,4BAA4B,CAAC,KAAK,EAAE,UAAU,EAAE,MAAM,CAAC;QACrE,QAAQ,EAAE,wBAAwB,CAAC,KAAK,CAAC;KAC1C,CAAC;AACJ,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
export type { ChatAssertion, ChatMatchers, } from "@skybridge/vite-plugin/evals";
|
|
2
1
|
export type { Chat } from "./chat.js";
|
|
2
|
+
export type { ChatAssertion, ChatMatchers, Judgment, JudgmentOptions, Verdict, } from "./matchers/index.js";
|
|
3
|
+
export type { EvalApp, EvalIdentity, ToolStubs, } from "./session-registry.js";
|
|
3
4
|
export { start } from "./session-registry.js";
|
|
4
5
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,aAAa,EACb,YAAY,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACtC,YAAY,EACV,aAAa,EACb,YAAY,EACZ,QAAQ,EACR,eAAe,EACf,OAAO,GACR,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EACV,OAAO,EACP,YAAY,EACZ,SAAS,GACV,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAaA,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { UserContent } from "ai";
|
|
2
|
+
export declare class InputFiles {
|
|
3
|
+
private closed;
|
|
4
|
+
private listening?;
|
|
5
|
+
private readonly files;
|
|
6
|
+
private readonly paths;
|
|
7
|
+
private readonly server;
|
|
8
|
+
prepare(content: UserContent): Promise<UserContent>;
|
|
9
|
+
rewrite(input: Record<string, unknown>, fields: string[]): Promise<{
|
|
10
|
+
[x: string]: unknown;
|
|
11
|
+
}>;
|
|
12
|
+
close(): Promise<void>;
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=input-files.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"input-files.d.ts","sourceRoot":"","sources":["../src/input-files.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAY,WAAW,EAAE,MAAM,IAAI,CAAC;AAkChD,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAC,CAAgB;IAClC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAGlB;IACJ,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA8B;IACpD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAQpB;IAEG,OAAO,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAyCnD,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE;;;IAkCxD,KAAK;CASZ"}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { once } from "node:events";
|
|
3
|
+
import { createServer } from "node:http";
|
|
4
|
+
import { registerInputFile } from "./matchers/matchers.js";
|
|
5
|
+
async function fileBytes(data) {
|
|
6
|
+
if (data instanceof URL) {
|
|
7
|
+
const response = await fetch(data);
|
|
8
|
+
if (!response.ok) {
|
|
9
|
+
throw new Error(`Input file download failed (HTTP ${response.status})`);
|
|
10
|
+
}
|
|
11
|
+
return Buffer.from(await response.arrayBuffer());
|
|
12
|
+
}
|
|
13
|
+
if (typeof data === "string") {
|
|
14
|
+
return data.startsWith("data:")
|
|
15
|
+
? fileBytes(new URL(data))
|
|
16
|
+
: Buffer.from(data, "base64");
|
|
17
|
+
}
|
|
18
|
+
if (data instanceof Uint8Array || data instanceof ArrayBuffer) {
|
|
19
|
+
return Buffer.from(new Uint8Array(data));
|
|
20
|
+
}
|
|
21
|
+
if (data.type === "data") {
|
|
22
|
+
return fileBytes(data.data);
|
|
23
|
+
}
|
|
24
|
+
if (data.type === "url") {
|
|
25
|
+
return fileBytes(new URL(data.url));
|
|
26
|
+
}
|
|
27
|
+
if (data.type === "text") {
|
|
28
|
+
return Buffer.from(data.text);
|
|
29
|
+
}
|
|
30
|
+
throw new Error("Input files need bytes, text, or a URL; provider-only file references cannot be downloaded by the eval app");
|
|
31
|
+
}
|
|
32
|
+
export class InputFiles {
|
|
33
|
+
closed = false;
|
|
34
|
+
listening;
|
|
35
|
+
files = new Map();
|
|
36
|
+
paths = new Map();
|
|
37
|
+
server = createServer((request, response) => {
|
|
38
|
+
const file = this.files.get(request.url?.slice(1) ?? "");
|
|
39
|
+
if (!file) {
|
|
40
|
+
response.writeHead(404).end();
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
response.writeHead(200, { "Content-Type": file.reference.mime_type });
|
|
44
|
+
response.end(file.bytes);
|
|
45
|
+
});
|
|
46
|
+
async prepare(content) {
|
|
47
|
+
if (this.closed) {
|
|
48
|
+
throw new Error("The conversation is closed");
|
|
49
|
+
}
|
|
50
|
+
if (typeof content === "string") {
|
|
51
|
+
return content;
|
|
52
|
+
}
|
|
53
|
+
const prepared = [];
|
|
54
|
+
for (const part of content) {
|
|
55
|
+
if (part.type !== "file") {
|
|
56
|
+
prepared.push(part);
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
const bytes = await fileBytes(part.data);
|
|
60
|
+
if (this.closed) {
|
|
61
|
+
throw new Error("The conversation is closed");
|
|
62
|
+
}
|
|
63
|
+
const id = randomUUID();
|
|
64
|
+
const file_id = `file_${id}`;
|
|
65
|
+
const reference = {
|
|
66
|
+
file_id,
|
|
67
|
+
download_url: "",
|
|
68
|
+
mime_type: part.mediaType,
|
|
69
|
+
...(part.filename === undefined ? {} : { file_name: part.filename }),
|
|
70
|
+
};
|
|
71
|
+
const uploaded_name = part.filename ?? "input";
|
|
72
|
+
const workspace_path = `/workspace/scratch/${id}/upload/${uploaded_name}`;
|
|
73
|
+
this.files.set(file_id, { reference, bytes });
|
|
74
|
+
this.paths.set(workspace_path, reference);
|
|
75
|
+
registerInputFile(reference, part);
|
|
76
|
+
prepared.push({
|
|
77
|
+
type: "text",
|
|
78
|
+
text: `Attached file: ${JSON.stringify({ uploaded_name, workspace_path })}`,
|
|
79
|
+
}, part);
|
|
80
|
+
}
|
|
81
|
+
return prepared;
|
|
82
|
+
}
|
|
83
|
+
async rewrite(input, fields) {
|
|
84
|
+
const rewritten = { ...input };
|
|
85
|
+
const resolve = (value) => {
|
|
86
|
+
if (typeof value !== "string") {
|
|
87
|
+
return value;
|
|
88
|
+
}
|
|
89
|
+
const reference = this.paths.get(value);
|
|
90
|
+
if (!reference) {
|
|
91
|
+
throw new Error(`Unknown input file path: ${value}`);
|
|
92
|
+
}
|
|
93
|
+
return reference;
|
|
94
|
+
};
|
|
95
|
+
for (const field of fields) {
|
|
96
|
+
if (field in rewritten) {
|
|
97
|
+
const value = rewritten[field];
|
|
98
|
+
rewritten[field] = Array.isArray(value)
|
|
99
|
+
? value.map(resolve)
|
|
100
|
+
: resolve(value);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
if (fields.some((field) => input[field] != null) && this.files.size) {
|
|
104
|
+
this.listening ??= (async () => {
|
|
105
|
+
this.server.listen(0, "127.0.0.1").unref();
|
|
106
|
+
await once(this.server, "listening");
|
|
107
|
+
})();
|
|
108
|
+
await this.listening;
|
|
109
|
+
const { port } = this.server.address();
|
|
110
|
+
for (const { reference } of this.files.values()) {
|
|
111
|
+
reference.download_url = `http://127.0.0.1:${port}/${reference.file_id}`;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
return rewritten;
|
|
115
|
+
}
|
|
116
|
+
async close() {
|
|
117
|
+
this.closed = true;
|
|
118
|
+
this.files.clear();
|
|
119
|
+
this.paths.clear();
|
|
120
|
+
if (this.server.listening) {
|
|
121
|
+
this.server.close();
|
|
122
|
+
await once(this.server, "close");
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
//# sourceMappingURL=input-files.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"input-files.js","sourceRoot":"","sources":["../src/input-files.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AACnC,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAIzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAE3D,KAAK,UAAU,SAAS,CAAC,IAAsB;IAC7C,IAAI,IAAI,YAAY,GAAG,EAAE,CAAC;QACxB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,oCAAoC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;QAC1E,CAAC;QACD,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;IACnD,CAAC;IACD,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;YAC7B,CAAC,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;YAC1B,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;IACD,IAAI,IAAI,YAAY,UAAU,IAAI,IAAI,YAAY,WAAW,EAAE,CAAC;QAC9D,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3C,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QACzB,OAAO,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;QACxB,OAAO,SAAS,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QACzB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IACD,MAAM,IAAI,KAAK,CACb,4GAA4G,CAC7G,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,UAAU;IACb,MAAM,GAAG,KAAK,CAAC;IACf,SAAS,CAAiB;IACjB,KAAK,GAAG,IAAI,GAAG,EAG7B,CAAC;IACa,KAAK,GAAG,IAAI,GAAG,EAAmB,CAAC;IACnC,MAAM,GAAG,YAAY,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE;QAC3D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACzD,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;YAC9B,OAAO;QACT,CAAC;QACD,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC,CAAC;QACtE,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,KAAK,CAAC,OAAO,CAAC,OAAoB;QAChC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAChD,CAAC;QACD,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YAChC,OAAO,OAAO,CAAC;QACjB,CAAC;QACD,MAAM,QAAQ,GAAiC,EAAE,CAAC;QAClD,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YAC3B,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACzB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACpB,SAAS;YACX,CAAC;YACD,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAChB,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;YAChD,CAAC;YACD,MAAM,EAAE,GAAG,UAAU,EAAE,CAAC;YACxB,MAAM,OAAO,GAAG,QAAQ,EAAE,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG;gBAChB,OAAO;gBACP,YAAY,EAAE,EAAE;gBAChB,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,GAAG,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;aACrE,CAAC;YACF,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC;YAC/C,MAAM,cAAc,GAAG,sBAAsB,EAAE,WAAW,aAAa,EAAE,CAAC;YAC1E,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;YAC9C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;YAC1C,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;YACnC,QAAQ,CAAC,IAAI,CACX;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,kBAAkB,IAAI,CAAC,SAAS,CAAC,EAAE,aAAa,EAAE,cAAc,EAAE,CAAC,EAAE;aAC5E,EACD,IAAI,CACL,CAAC;QACJ,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,KAA8B,EAAE,MAAgB;QAC5D,MAAM,SAAS,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;QAC/B,MAAM,OAAO,GAAG,CAAC,KAAc,EAAW,EAAE;YAC1C,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,OAAO,KAAK,CAAC;YACf,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACxC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,4BAA4B,KAAK,EAAE,CAAC,CAAC;YACvD,CAAC;YACD,OAAO,SAAS,CAAC;QACnB,CAAC,CAAC;QACF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,KAAK,IAAI,SAAS,EAAE,CAAC;gBACvB,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;gBAC/B,SAAS,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;oBACrC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC;oBACpB,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YACpE,IAAI,CAAC,SAAS,KAAK,CAAC,KAAK,IAAI,EAAE;gBAC7B,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,KAAK,EAAE,CAAC;gBAC3C,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;YACvC,CAAC,CAAC,EAAE,CAAC;YACL,MAAM,IAAI,CAAC,SAAS,CAAC;YACrB,MAAM,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,EAAiB,CAAC;YACtD,KAAK,MAAM,EAAE,SAAS,EAAE,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;gBAChD,SAAS,CAAC,YAAY,GAAG,oBAAoB,IAAI,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;YAC3E,CAAC;QACH,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YACpB,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/matchers/index.ts"],"names":[],"mappings":"AAAA,OAAO,eAAe,CAAC;AAEvB,YAAY,EACV,aAAa,EACb,YAAY,EACZ,QAAQ,EACR,eAAe,EACf,OAAO,GACR,MAAM,eAAe,CAAC;AACvB,YAAY,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/matchers/index.ts"],"names":[],"mappings":"AAAA,OAAO,eAAe,CAAC"}
|