@skybridge/test 0.0.0-dev.1ac91c5 → 0.0.0-dev.1b18ac7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -2
- package/dist/chat.d.ts +7 -3
- package/dist/chat.d.ts.map +1 -1
- package/dist/chat.js +68 -16
- package/dist/chat.js.map +1 -1
- package/dist/eval-middlewares.d.ts +12 -0
- package/dist/eval-middlewares.d.ts.map +1 -0
- package/dist/eval-middlewares.js +18 -0
- package/dist/eval-middlewares.js.map +1 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/input-files.d.ts +14 -0
- package/dist/input-files.d.ts.map +1 -0
- package/dist/input-files.js +126 -0
- package/dist/input-files.js.map +1 -0
- package/dist/matchers/index.d.ts +1 -1
- package/dist/matchers/index.d.ts.map +1 -1
- package/dist/matchers/matchers.d.ts +23 -2
- package/dist/matchers/matchers.d.ts.map +1 -1
- package/dist/matchers/matchers.js +69 -19
- package/dist/matchers/matchers.js.map +1 -1
- package/dist/middleware/list-tools.d.ts +4 -0
- package/dist/middleware/list-tools.d.ts.map +1 -0
- package/dist/middleware/list-tools.js +67 -0
- package/dist/middleware/list-tools.js.map +1 -0
- package/dist/middleware/resolve-files.d.ts +4 -0
- package/dist/middleware/resolve-files.d.ts.map +1 -0
- package/dist/middleware/resolve-files.js +13 -0
- package/dist/middleware/resolve-files.js.map +1 -0
- package/dist/middleware/stub-tool.d.ts +4 -0
- package/dist/middleware/stub-tool.d.ts.map +1 -0
- package/dist/middleware/stub-tool.js +16 -0
- package/dist/middleware/stub-tool.js.map +1 -0
- package/dist/session-registry.d.ts +18 -13
- package/dist/session-registry.d.ts.map +1 -1
- package/dist/session-registry.js +25 -8
- package/dist/session-registry.js.map +1 -1
- package/package.json +4 -4
package/README.md
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
# @skybridge/test
|
|
2
2
|
|
|
3
|
-
Evals for [Skybridge](https://skybridge.tech) MCP
|
|
3
|
+
Evals for [Skybridge](https://skybridge.tech) MCP Apps: they answer whether a
|
|
4
4
|
prompt reaches the right tool with the right arguments, which no unit test can
|
|
5
5
|
check.
|
|
6
6
|
|
|
7
7
|
Add the `evals` option to the Skybridge Vite plugin so `expect.chat` exists,
|
|
8
8
|
then write scenarios with vitest's own `it` and `expect`. Pass the app itself
|
|
9
|
-
to `start`: it is served in-process,
|
|
9
|
+
to `start`: it is served in-process, using your exported app, and
|
|
10
10
|
the tool names and argument shapes come from the app value, so no type
|
|
11
11
|
parameter is needed.
|
|
12
12
|
|
|
@@ -94,3 +94,33 @@ expect.chat(chat).not.toHaveSaid("refund");
|
|
|
94
94
|
|
|
95
95
|
See the [documentation](https://docs.skybridge.tech) for setup and the full
|
|
96
96
|
matcher list.
|
|
97
|
+
|
|
98
|
+
## Input files
|
|
99
|
+
|
|
100
|
+
`chat.send` also accepts AI SDK `UserContent`. Attach a `FilePart` and use the same object in `expect.inputFile` to check which file a tool received:
|
|
101
|
+
|
|
102
|
+
```ts
|
|
103
|
+
import { readFile } from "node:fs/promises";
|
|
104
|
+
import type { FilePart } from "ai";
|
|
105
|
+
|
|
106
|
+
const portrait: FilePart = {
|
|
107
|
+
type: "file",
|
|
108
|
+
mediaType: "image/png",
|
|
109
|
+
filename: "photo.png",
|
|
110
|
+
data: await readFile(new URL("./fixtures/photo.png", import.meta.url)),
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
await chat.send([{ type: "text", text: "Resize this photo" }, portrait]);
|
|
114
|
+
|
|
115
|
+
expect.chat(chat).toHaveCalledToolWith("image_crop_and_resize", {
|
|
116
|
+
file: expect.inputFile(portrait),
|
|
117
|
+
});
|
|
118
|
+
expect.chat(chat).toHaveCalledToolOnce("asset_openai_file_upload", {
|
|
119
|
+
files: [expect.inputFile(portrait)],
|
|
120
|
+
});
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
`file` and `files` must be listed in their tools' `_meta["openai/fileParams"]`.
|
|
124
|
+
Misplaced file matchers throw immediately, including nested matchers and `.not`.
|
|
125
|
+
Declare file parameters with `FileRef` and list their names in `_meta["openai/fileParams"]`. Optional, nullable, and array forms are supported. Your handler or stub receives the attached file; downloads remain available until the conversation closes.
|
|
126
|
+
See [Evals](https://docs.skybridge.tech/test/evals) for how files reach the model and the tool.
|
package/dist/chat.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { type LanguageModel } from "ai";
|
|
1
|
+
import { type LanguageModel, type UserContent } from "ai";
|
|
2
|
+
import type { EvalMiddlewares } from "./eval-middlewares.js";
|
|
2
3
|
import type { ToolCall, Turn } from "./matchers/types.js";
|
|
3
4
|
interface HostConfig {
|
|
4
5
|
model: LanguageModel;
|
|
@@ -16,6 +17,8 @@ export type TransportFetch = (url: string | URL, init?: RequestInit) => Promise<
|
|
|
16
17
|
* {@link Chat.toolCalls}, the sequence of calls the model made.
|
|
17
18
|
*/
|
|
18
19
|
export declare class Chat<App = unknown> {
|
|
20
|
+
private readonly middleware;
|
|
21
|
+
private readonly dispose;
|
|
19
22
|
/** Type-only anchor so `expect.chat` infers `App` from the conversation. */
|
|
20
23
|
readonly $app: App;
|
|
21
24
|
/** Everything that happened, in the order it happened. */
|
|
@@ -30,12 +33,13 @@ export declare class Chat<App = unknown> {
|
|
|
30
33
|
private readonly transport;
|
|
31
34
|
private readonly host;
|
|
32
35
|
private tools;
|
|
36
|
+
private closePromise?;
|
|
33
37
|
/** The model the conversation runs on, and the judge's default. */
|
|
34
38
|
get model(): LanguageModel;
|
|
35
39
|
private constructor();
|
|
36
|
-
static open<App>(host: HostConfig, fetchImpl: TransportFetch): Promise<Chat<App>>;
|
|
40
|
+
static open<App>(host: HostConfig, fetchImpl: TransportFetch, middleware: EvalMiddlewares, dispose?: () => Promise<void>): Promise<Chat<App>>;
|
|
37
41
|
/** Takes a turn. Assert on the result with `expect.chat`. */
|
|
38
|
-
send(
|
|
42
|
+
send(content: UserContent): Promise<void>;
|
|
39
43
|
close(): Promise<void>;
|
|
40
44
|
private failureFor;
|
|
41
45
|
private toolSet;
|
package/dist/chat.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAOA,OAAO,EAKL,KAAK,aAAa,EAGlB,KAAK,WAAW,EACjB,MAAM,IAAI,CAAC;AACZ,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAE7D,OAAO,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AAE1D,UAAU,UAAU;IAClB,KAAK,EAAE,aAAa,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAaD;;;GAGG;AACH,MAAM,MAAM,cAAc,GAAG,CAC3B,GAAG,EAAE,MAAM,GAAG,GAAG,EACjB,IAAI,CAAC,EAAE,WAAW,KACf,OAAO,CAAC,QAAQ,CAAC,CAAC;AAEvB;;;GAGG;AACH,qBAAa,IAAI,CAAC,GAAG,GAAG,OAAO;IAsC3B,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO;IAtC1B,4EAA4E;IAC5E,SAAiB,IAAI,EAAE,GAAG,CAAC;IAE3B,0DAA0D;IAC1D,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAM;IAEjC,mEAAmE;IACnE,IAAI,SAAS,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE,CAI/B;IAED,+EAA+E;IAC/E,IAAI,cAAc,IAAI,MAAM,EAAE,CAI7B;IAED,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA6B;IACtD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAa;IAClC,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,YAAY,CAAC,CAAgB;IAErC,mEAAmE;IACnE,IAAI,KAAK,IAAI,aAAa,CAEzB;IAED,OAAO;WAYM,IAAI,CAAC,GAAG,EACnB,IAAI,EAAE,UAAU,EAChB,SAAS,EAAE,cAAc,EACzB,UAAU,EAAE,eAAe,EAC3B,OAAO,GAAE,MAAM,OAAO,CAAC,IAAI,CAAkB,GAC5C,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAmBrB,6DAA6D;IACvD,IAAI,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAsD/C,KAAK;IAeL,OAAO,CAAC,UAAU;IAalB,OAAO,CAAC,OAAO;CAkDhB"}
|
package/dist/chat.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import { Client, StreamableHTTPClientTransport, } from "@modelcontextprotocol/client";
|
|
1
|
+
import { Client, fromJsonSchema, StreamableHTTPClientTransport, } from "@modelcontextprotocol/client";
|
|
2
2
|
import { dynamicTool, generateText, jsonSchema, stepCountIs, } from "ai";
|
|
3
|
+
import { registerFileParameters } from "./matchers/matchers.js";
|
|
3
4
|
const DEFAULT_MAX_STEPS = 8;
|
|
4
5
|
const IN_PROCESS_URL = "http://in-process.skybridge.test/mcp";
|
|
5
6
|
function outputOf(results, toolCallId) {
|
|
@@ -10,6 +11,8 @@ function outputOf(results, toolCallId) {
|
|
|
10
11
|
* {@link Chat.toolCalls}, the sequence of calls the model made.
|
|
11
12
|
*/
|
|
12
13
|
export class Chat {
|
|
14
|
+
middleware;
|
|
15
|
+
dispose;
|
|
13
16
|
/** Everything that happened, in the order it happened. */
|
|
14
17
|
turns = [];
|
|
15
18
|
/** Every call the model made, in order, as it crossed the wire. */
|
|
@@ -30,23 +33,27 @@ export class Chat {
|
|
|
30
33
|
transport;
|
|
31
34
|
host;
|
|
32
35
|
tools = [];
|
|
36
|
+
closePromise;
|
|
33
37
|
/** The model the conversation runs on, and the judge's default. */
|
|
34
38
|
get model() {
|
|
35
39
|
return this.host.model;
|
|
36
40
|
}
|
|
37
|
-
constructor(client, transport, host) {
|
|
41
|
+
constructor(client, transport, host, middleware, dispose) {
|
|
42
|
+
this.middleware = middleware;
|
|
43
|
+
this.dispose = dispose;
|
|
38
44
|
this.client = client;
|
|
39
45
|
this.transport = transport;
|
|
40
46
|
this.host = host;
|
|
41
47
|
}
|
|
42
|
-
static async open(host, fetchImpl) {
|
|
48
|
+
static async open(host, fetchImpl, middleware, dispose = async () => { }) {
|
|
43
49
|
const client = new Client({ name: "skybridge-eval", version: "0" });
|
|
44
50
|
const transport = new StreamableHTTPClientTransport(new URL(IN_PROCESS_URL), { fetch: fetchImpl });
|
|
45
|
-
|
|
46
|
-
const chat = new Chat(client, transport, host);
|
|
51
|
+
const chat = new Chat(client, transport, host, middleware, dispose);
|
|
47
52
|
try {
|
|
53
|
+
await client.connect(transport);
|
|
48
54
|
const { tools } = await client.listTools();
|
|
49
55
|
chat.tools = tools;
|
|
56
|
+
registerFileParameters(chat, middleware.fileParams);
|
|
50
57
|
}
|
|
51
58
|
catch (error) {
|
|
52
59
|
await transport.close();
|
|
@@ -55,12 +62,28 @@ export class Chat {
|
|
|
55
62
|
return chat;
|
|
56
63
|
}
|
|
57
64
|
/** Takes a turn. Assert on the result with `expect.chat`. */
|
|
58
|
-
async send(
|
|
59
|
-
this.
|
|
60
|
-
|
|
65
|
+
async send(content) {
|
|
66
|
+
if (this.closePromise) {
|
|
67
|
+
throw new Error("The conversation is closed");
|
|
68
|
+
}
|
|
69
|
+
const prepared = await this.middleware.files.prepare(content);
|
|
70
|
+
this.middleware.inputs.clear();
|
|
71
|
+
this.failures.clear();
|
|
72
|
+
this.messages.push({ role: "user", content: prepared });
|
|
73
|
+
this.turns.push({
|
|
74
|
+
role: "user",
|
|
75
|
+
text: typeof prepared === "string"
|
|
76
|
+
? prepared
|
|
77
|
+
: prepared
|
|
78
|
+
.filter((part) => part.type === "text")
|
|
79
|
+
.map((part) => part.text)
|
|
80
|
+
.join("\n"),
|
|
81
|
+
});
|
|
61
82
|
const result = await generateText({
|
|
62
83
|
model: this.host.model,
|
|
63
|
-
|
|
84
|
+
...(this.host.temperature === undefined
|
|
85
|
+
? {}
|
|
86
|
+
: { temperature: this.host.temperature }),
|
|
64
87
|
...(this.host.systemPrompt === undefined
|
|
65
88
|
? {}
|
|
66
89
|
: { system: this.host.systemPrompt }),
|
|
@@ -77,17 +100,31 @@ export class Chat {
|
|
|
77
100
|
role: "tool",
|
|
78
101
|
call: {
|
|
79
102
|
name: call.toolName,
|
|
80
|
-
arguments: (call.
|
|
103
|
+
arguments: this.middleware.inputs.get(call.toolCallId) ??
|
|
104
|
+
(call.input ?? {}),
|
|
81
105
|
...this.failureFor(call),
|
|
82
106
|
},
|
|
83
107
|
result: outputOf(step.toolResults, call.toolCallId),
|
|
84
108
|
});
|
|
85
109
|
}
|
|
86
110
|
}
|
|
87
|
-
this.messages.push(...result.
|
|
111
|
+
this.messages.push(...result.responseMessages);
|
|
88
112
|
}
|
|
89
|
-
|
|
90
|
-
|
|
113
|
+
close() {
|
|
114
|
+
this.closePromise ??= (async () => {
|
|
115
|
+
try {
|
|
116
|
+
await this.transport.close();
|
|
117
|
+
}
|
|
118
|
+
finally {
|
|
119
|
+
try {
|
|
120
|
+
await this.middleware.files.close();
|
|
121
|
+
}
|
|
122
|
+
finally {
|
|
123
|
+
await this.dispose();
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
})();
|
|
127
|
+
return this.closePromise;
|
|
91
128
|
}
|
|
92
129
|
failureFor(call) {
|
|
93
130
|
const reported = this.failures.get(call.toolCallId);
|
|
@@ -102,27 +139,42 @@ export class Chat {
|
|
|
102
139
|
toolSet() {
|
|
103
140
|
const entries = this.tools.map((definition) => {
|
|
104
141
|
const { $schema: _ignored, ...parameters } = definition.inputSchema;
|
|
142
|
+
const schema = fromJsonSchema(parameters);
|
|
105
143
|
return [
|
|
106
144
|
definition.name,
|
|
107
145
|
dynamicTool({
|
|
108
146
|
description: definition.description ?? "",
|
|
109
|
-
inputSchema: jsonSchema(parameters
|
|
147
|
+
inputSchema: jsonSchema(parameters, {
|
|
148
|
+
validate: async (input) => {
|
|
149
|
+
const result = await schema["~standard"].validate(input);
|
|
150
|
+
return result.issues
|
|
151
|
+
? {
|
|
152
|
+
success: false,
|
|
153
|
+
error: new Error(JSON.stringify(result.issues)),
|
|
154
|
+
}
|
|
155
|
+
: { success: true, value: result.value };
|
|
156
|
+
},
|
|
157
|
+
}),
|
|
110
158
|
execute: async (input, { toolCallId }) => {
|
|
111
159
|
let result;
|
|
112
160
|
try {
|
|
113
161
|
result = await this.client.callTool({
|
|
114
162
|
name: definition.name,
|
|
115
163
|
arguments: (input ?? {}),
|
|
164
|
+
_meta: { "skybridge/evalToolCallId": toolCallId },
|
|
116
165
|
});
|
|
117
166
|
}
|
|
118
167
|
catch (error) {
|
|
119
|
-
this.failures.set(toolCallId, `the call
|
|
168
|
+
this.failures.set(toolCallId, `the tool call failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
120
169
|
throw error;
|
|
121
170
|
}
|
|
122
171
|
if (result.isError === true) {
|
|
123
172
|
this.failures.set(toolCallId, `the tool returned an error: ${JSON.stringify(result.content)}`);
|
|
124
173
|
}
|
|
125
|
-
return
|
|
174
|
+
return {
|
|
175
|
+
content: result.content,
|
|
176
|
+
structuredContent: result.structuredContent,
|
|
177
|
+
};
|
|
126
178
|
},
|
|
127
179
|
}),
|
|
128
180
|
];
|
package/dist/chat.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,MAAM,EACN,6BAA6B,GAE9B,MAAM,8BAA8B,CAAC;AACtC,OAAO,EACL,WAAW,EACX,YAAY,
|
|
1
|
+
{"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,MAAM,EACN,cAAc,EAEd,6BAA6B,GAE9B,MAAM,8BAA8B,CAAC;AACtC,OAAO,EACL,WAAW,EACX,YAAY,EAEZ,UAAU,EAGV,WAAW,GAEZ,MAAM,IAAI,CAAC;AAEZ,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAUhE,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B,MAAM,cAAc,GAAG,sCAAsC,CAAC;AAE9D,SAAS,QAAQ,CACf,OAAmD,EACnD,UAAkB;IAElB,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,KAAK,UAAU,CAAC,EAAE,MAAM,CAAC;AAC5E,CAAC;AAWD;;;GAGG;AACH,MAAM,OAAO,IAAI;IAsCI;IACA;IAnCnB,0DAA0D;IACjD,KAAK,GAAgB,EAAE,CAAC;IAEjC,mEAAmE;IACnE,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,KAAK;aACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC;aACtC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAED,+EAA+E;IAC/E,IAAI,cAAc;QAChB,OAAO,IAAI,CAAC,KAAK;aACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,WAAW,CAAC;aAC3C,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAEgB,QAAQ,GAAmB,EAAE,CAAC;IAC9B,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrC,MAAM,CAAS;IACf,SAAS,CAAgC;IACzC,IAAI,CAAa;IAC1B,KAAK,GAAW,EAAE,CAAC;IACnB,YAAY,CAAiB;IAErC,mEAAmE;IACnE,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;IACzB,CAAC;IAED,YACE,MAAc,EACd,SAAwC,EACxC,IAAgB,EACC,UAA2B,EAC3B,OAA4B;QAD5B,eAAU,GAAV,UAAU,CAAiB;QAC3B,YAAO,GAAP,OAAO,CAAqB;QAE7C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,IAAI,CACf,IAAgB,EAChB,SAAyB,EACzB,UAA2B,EAC3B,UAA+B,KAAK,IAAI,EAAE,GAAE,CAAC;QAE7C,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,IAAI,6BAA6B,CACjD,IAAI,GAAG,CAAC,cAAc,CAAC,EACvB,EAAE,KAAK,EAAE,SAAS,EAAE,CACrB,CAAC;QACF,MAAM,IAAI,GAAG,IAAI,IAAI,CAAM,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;QACzE,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YAChC,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;YACnB,sBAAsB,CAAC,IAAI,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;QACtD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YACxB,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,6DAA6D;IAC7D,KAAK,CAAC,IAAI,CAAC,OAAoB;QAC7B,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAChD,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC9D,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;QACxD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;YACd,IAAI,EAAE,MAAM;YACZ,IAAI,EACF,OAAO,QAAQ,KAAK,QAAQ;gBAC1B,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,QAAQ;qBACL,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC;qBACtC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;qBACxB,IAAI,CAAC,IAAI,CAAC;SACpB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;YACtB,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,KAAK,SAAS;gBACrC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YAC3C,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,KAAK,SAAS;gBACtC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE;YACrB,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;SAC/D,CAAC,CAAC;QAEH,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,IAAI,CAAC,IAAI,KAAK,EAAE,EAAE,CAAC;gBACrB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YAC1D,CAAC;YACD,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAClC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;oBACd,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE;wBACJ,IAAI,EAAE,IAAI,CAAC,QAAQ;wBACnB,SAAS,EACP,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC;4BAC1C,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAA6B;wBACjD,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;qBACR;oBAClB,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,UAAU,CAAC;iBACpD,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;IACjD,CAAC;IAED,KAAK;QACH,IAAI,CAAC,YAAY,KAAK,CAAC,KAAK,IAAI,EAAE;YAChC,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;YAC/B,CAAC;oBAAS,CAAC;gBACT,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;gBACtC,CAAC;wBAAS,CAAC;oBACT,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;gBACvB,CAAC;YACH,CAAC;QACH,CAAC,CAAC,EAAE,CAAC;QACL,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAEO,UAAU,CAAC,IAA+C;QAGhE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QAC9B,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC1B,OAAO,EAAE,MAAM,EAAE,2CAA2C,EAAE,CAAC;QACjE,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAEO,OAAO;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;YAC5C,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,UAAU,EAAE,GAAG,UAAU,CAAC,WAAW,CAAC;YACpE,MAAM,MAAM,GAAG,cAAc,CAAC,UAA4B,CAAC,CAAC;YAC5D,OAAO;gBACL,UAAU,CAAC,IAAI;gBACf,WAAW,CAAC;oBACV,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,EAAE;oBACzC,WAAW,EAAE,UAAU,CAAC,UAAyB,EAAE;wBACjD,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;4BACxB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;4BACzD,OAAO,MAAM,CAAC,MAAM;gCAClB,CAAC,CAAC;oCACE,OAAO,EAAE,KAAK;oCACd,KAAK,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;iCAChD;gCACH,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;wBAC7C,CAAC;qBACF,CAAC;oBACF,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE;wBACvC,IAAI,MAA+C,CAAC;wBACpD,IAAI,CAAC;4BACH,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;gCAClC,IAAI,EAAE,UAAU,CAAC,IAAI;gCACrB,SAAS,EAAE,CAAC,KAAK,IAAI,EAAE,CAA4B;gCACnD,KAAK,EAAE,EAAE,0BAA0B,EAAE,UAAU,EAAE;6BAClD,CAAC,CAAC;wBACL,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BACf,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,yBAAyB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAClF,CAAC;4BACF,MAAM,KAAK,CAAC;wBACd,CAAC;wBACD,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;4BAC5B,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,+BAA+B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAChE,CAAC;wBACJ,CAAC;wBACD,OAAO;4BACL,OAAO,EAAE,MAAM,CAAC,OAAO;4BACvB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;yBAC5C,CAAC;oBACJ,CAAC;iBACF,CAAC;aACM,CAAC;QACb,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;CACF"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { InputFiles } from "./input-files.js";
|
|
2
|
+
import type { ToolStubs } from "./session-registry.js";
|
|
3
|
+
export declare function createEvalMiddlewares(stubs?: ToolStubs): {
|
|
4
|
+
files: InputFiles;
|
|
5
|
+
inputs: Map<string, Record<string, unknown>>;
|
|
6
|
+
fileParams: Map<string, string[]>;
|
|
7
|
+
listTools: import("skybridge/server").McpMiddlewareFn;
|
|
8
|
+
resolveFiles: import("skybridge/server").McpMiddlewareFn;
|
|
9
|
+
stubTool: import("skybridge/server").McpMiddlewareFn;
|
|
10
|
+
};
|
|
11
|
+
export type EvalMiddlewares = ReturnType<typeof createEvalMiddlewares>;
|
|
12
|
+
//# sourceMappingURL=eval-middlewares.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval-middlewares.d.ts","sourceRoot":"","sources":["../src/eval-middlewares.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAI9C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAEvD,wBAAgB,qBAAqB,CAAC,KAAK,GAAE,SAAc;;;;;;;EAa1D;AAED,MAAM,MAAM,eAAe,GAAG,UAAU,CAAC,OAAO,qBAAqB,CAAC,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { InputFiles } from "./input-files.js";
|
|
2
|
+
import { createListToolsMiddleware } from "./middleware/list-tools.js";
|
|
3
|
+
import { createResolveFilesMiddleware } from "./middleware/resolve-files.js";
|
|
4
|
+
import { createStubToolMiddleware } from "./middleware/stub-tool.js";
|
|
5
|
+
export function createEvalMiddlewares(stubs = {}) {
|
|
6
|
+
const files = new InputFiles();
|
|
7
|
+
const inputs = new Map();
|
|
8
|
+
const fileParams = new Map();
|
|
9
|
+
return {
|
|
10
|
+
files,
|
|
11
|
+
inputs,
|
|
12
|
+
fileParams,
|
|
13
|
+
listTools: createListToolsMiddleware(fileParams, stubs),
|
|
14
|
+
resolveFiles: createResolveFilesMiddleware(files, fileParams, inputs),
|
|
15
|
+
stubTool: createStubToolMiddleware(stubs),
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
//# sourceMappingURL=eval-middlewares.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval-middlewares.js","sourceRoot":"","sources":["../src/eval-middlewares.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AACvE,OAAO,EAAE,4BAA4B,EAAE,MAAM,+BAA+B,CAAC;AAC7E,OAAO,EAAE,wBAAwB,EAAE,MAAM,2BAA2B,CAAC;AAGrE,MAAM,UAAU,qBAAqB,CAAC,QAAmB,EAAE;IACzD,MAAM,KAAK,GAAG,IAAI,UAAU,EAAE,CAAC;IAC/B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAmC,CAAC;IAC1D,MAAM,UAAU,GAAG,IAAI,GAAG,EAAoB,CAAC;IAE/C,OAAO;QACL,KAAK;QACL,MAAM;QACN,UAAU;QACV,SAAS,EAAE,yBAAyB,CAAC,UAAU,EAAE,KAAK,CAAC;QACvD,YAAY,EAAE,4BAA4B,CAAC,KAAK,EAAE,UAAU,EAAE,MAAM,CAAC;QACrE,QAAQ,EAAE,wBAAwB,CAAC,KAAK,CAAC;KAC1C,CAAC;AACJ,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export type { Chat } from "./chat.js";
|
|
2
|
-
export type { ChatAssertion, ChatMatchers, JudgmentOptions, } from "./matchers/index.js";
|
|
3
|
-
export type { EvalApp, EvalIdentity } from "./session-registry.js";
|
|
2
|
+
export type { ChatAssertion, ChatMatchers, Judgment, JudgmentOptions, Verdict, } from "./matchers/index.js";
|
|
3
|
+
export type { EvalApp, EvalIdentity, ToolStubs, } from "./session-registry.js";
|
|
4
4
|
export { start } from "./session-registry.js";
|
|
5
5
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACtC,YAAY,EACV,aAAa,EACb,YAAY,EACZ,eAAe,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACtC,YAAY,EACV,aAAa,EACb,YAAY,EACZ,QAAQ,EACR,eAAe,EACf,OAAO,GACR,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EACV,OAAO,EACP,YAAY,EACZ,SAAS,GACV,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAaA,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { UserContent } from "ai";
|
|
2
|
+
export declare class InputFiles {
|
|
3
|
+
private closed;
|
|
4
|
+
private listening?;
|
|
5
|
+
private readonly files;
|
|
6
|
+
private readonly paths;
|
|
7
|
+
private readonly server;
|
|
8
|
+
prepare(content: UserContent): Promise<UserContent>;
|
|
9
|
+
rewrite(input: Record<string, unknown>, fields: string[]): Promise<{
|
|
10
|
+
[x: string]: unknown;
|
|
11
|
+
}>;
|
|
12
|
+
close(): Promise<void>;
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=input-files.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"input-files.d.ts","sourceRoot":"","sources":["../src/input-files.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAY,WAAW,EAAE,MAAM,IAAI,CAAC;AAkChD,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAC,CAAgB;IAClC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAGlB;IACJ,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA8B;IACpD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAQpB;IAEG,OAAO,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAyCnD,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE;;;IAkCxD,KAAK;CASZ"}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { once } from "node:events";
|
|
3
|
+
import { createServer } from "node:http";
|
|
4
|
+
import { registerInputFile } from "./matchers/matchers.js";
|
|
5
|
+
async function fileBytes(data) {
|
|
6
|
+
if (data instanceof URL) {
|
|
7
|
+
const response = await fetch(data);
|
|
8
|
+
if (!response.ok) {
|
|
9
|
+
throw new Error(`Input file download failed (HTTP ${response.status})`);
|
|
10
|
+
}
|
|
11
|
+
return Buffer.from(await response.arrayBuffer());
|
|
12
|
+
}
|
|
13
|
+
if (typeof data === "string") {
|
|
14
|
+
return data.startsWith("data:")
|
|
15
|
+
? fileBytes(new URL(data))
|
|
16
|
+
: Buffer.from(data, "base64");
|
|
17
|
+
}
|
|
18
|
+
if (data instanceof Uint8Array || data instanceof ArrayBuffer) {
|
|
19
|
+
return Buffer.from(new Uint8Array(data));
|
|
20
|
+
}
|
|
21
|
+
if (data.type === "data") {
|
|
22
|
+
return fileBytes(data.data);
|
|
23
|
+
}
|
|
24
|
+
if (data.type === "url") {
|
|
25
|
+
return fileBytes(new URL(data.url));
|
|
26
|
+
}
|
|
27
|
+
if (data.type === "text") {
|
|
28
|
+
return Buffer.from(data.text);
|
|
29
|
+
}
|
|
30
|
+
throw new Error("Input files need bytes, text, or a URL; provider-only file references cannot be downloaded by the eval app");
|
|
31
|
+
}
|
|
32
|
+
export class InputFiles {
|
|
33
|
+
closed = false;
|
|
34
|
+
listening;
|
|
35
|
+
files = new Map();
|
|
36
|
+
paths = new Map();
|
|
37
|
+
server = createServer((request, response) => {
|
|
38
|
+
const file = this.files.get(request.url?.slice(1) ?? "");
|
|
39
|
+
if (!file) {
|
|
40
|
+
response.writeHead(404).end();
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
response.writeHead(200, { "Content-Type": file.reference.mime_type });
|
|
44
|
+
response.end(file.bytes);
|
|
45
|
+
});
|
|
46
|
+
async prepare(content) {
|
|
47
|
+
if (this.closed) {
|
|
48
|
+
throw new Error("The conversation is closed");
|
|
49
|
+
}
|
|
50
|
+
if (typeof content === "string") {
|
|
51
|
+
return content;
|
|
52
|
+
}
|
|
53
|
+
const prepared = [];
|
|
54
|
+
for (const part of content) {
|
|
55
|
+
if (part.type !== "file") {
|
|
56
|
+
prepared.push(part);
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
const bytes = await fileBytes(part.data);
|
|
60
|
+
if (this.closed) {
|
|
61
|
+
throw new Error("The conversation is closed");
|
|
62
|
+
}
|
|
63
|
+
const id = randomUUID();
|
|
64
|
+
const file_id = `file_${id}`;
|
|
65
|
+
const reference = {
|
|
66
|
+
file_id,
|
|
67
|
+
download_url: "",
|
|
68
|
+
mime_type: part.mediaType,
|
|
69
|
+
...(part.filename === undefined ? {} : { file_name: part.filename }),
|
|
70
|
+
};
|
|
71
|
+
const uploaded_name = part.filename ?? "input";
|
|
72
|
+
const workspace_path = `/workspace/scratch/${id}/upload/${uploaded_name}`;
|
|
73
|
+
this.files.set(file_id, { reference, bytes });
|
|
74
|
+
this.paths.set(workspace_path, reference);
|
|
75
|
+
registerInputFile(reference, part);
|
|
76
|
+
prepared.push({
|
|
77
|
+
type: "text",
|
|
78
|
+
text: `Attached file: ${JSON.stringify({ uploaded_name, workspace_path })}`,
|
|
79
|
+
}, part);
|
|
80
|
+
}
|
|
81
|
+
return prepared;
|
|
82
|
+
}
|
|
83
|
+
async rewrite(input, fields) {
|
|
84
|
+
const rewritten = { ...input };
|
|
85
|
+
const resolve = (value) => {
|
|
86
|
+
if (typeof value !== "string") {
|
|
87
|
+
return value;
|
|
88
|
+
}
|
|
89
|
+
const reference = this.paths.get(value);
|
|
90
|
+
if (!reference) {
|
|
91
|
+
throw new Error(`Unknown input file path: ${value}`);
|
|
92
|
+
}
|
|
93
|
+
return reference;
|
|
94
|
+
};
|
|
95
|
+
for (const field of fields) {
|
|
96
|
+
if (field in rewritten) {
|
|
97
|
+
const value = rewritten[field];
|
|
98
|
+
rewritten[field] = Array.isArray(value)
|
|
99
|
+
? value.map(resolve)
|
|
100
|
+
: resolve(value);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
if (fields.some((field) => input[field] != null) && this.files.size) {
|
|
104
|
+
this.listening ??= (async () => {
|
|
105
|
+
this.server.listen(0, "127.0.0.1").unref();
|
|
106
|
+
await once(this.server, "listening");
|
|
107
|
+
})();
|
|
108
|
+
await this.listening;
|
|
109
|
+
const { port } = this.server.address();
|
|
110
|
+
for (const { reference } of this.files.values()) {
|
|
111
|
+
reference.download_url = `http://127.0.0.1:${port}/${reference.file_id}`;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
return rewritten;
|
|
115
|
+
}
|
|
116
|
+
async close() {
|
|
117
|
+
this.closed = true;
|
|
118
|
+
this.files.clear();
|
|
119
|
+
this.paths.clear();
|
|
120
|
+
if (this.server.listening) {
|
|
121
|
+
this.server.close();
|
|
122
|
+
await once(this.server, "close");
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
//# sourceMappingURL=input-files.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"input-files.js","sourceRoot":"","sources":["../src/input-files.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AACnC,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAIzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAE3D,KAAK,UAAU,SAAS,CAAC,IAAsB;IAC7C,IAAI,IAAI,YAAY,GAAG,EAAE,CAAC;QACxB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,oCAAoC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;QAC1E,CAAC;QACD,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;IACnD,CAAC;IACD,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;YAC7B,CAAC,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;YAC1B,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;IACD,IAAI,IAAI,YAAY,UAAU,IAAI,IAAI,YAAY,WAAW,EAAE,CAAC;QAC9D,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3C,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QACzB,OAAO,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;QACxB,OAAO,SAAS,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QACzB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IACD,MAAM,IAAI,KAAK,CACb,4GAA4G,CAC7G,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,UAAU;IACb,MAAM,GAAG,KAAK,CAAC;IACf,SAAS,CAAiB;IACjB,KAAK,GAAG,IAAI,GAAG,EAG7B,CAAC;IACa,KAAK,GAAG,IAAI,GAAG,EAAmB,CAAC;IACnC,MAAM,GAAG,YAAY,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE;QAC3D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACzD,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;YAC9B,OAAO;QACT,CAAC;QACD,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC,CAAC;QACtE,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,KAAK,CAAC,OAAO,CAAC,OAAoB;QAChC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAChD,CAAC;QACD,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YAChC,OAAO,OAAO,CAAC;QACjB,CAAC;QACD,MAAM,QAAQ,GAAiC,EAAE,CAAC;QAClD,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YAC3B,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACzB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACpB,SAAS;YACX,CAAC;YACD,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAChB,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;YAChD,CAAC;YACD,MAAM,EAAE,GAAG,UAAU,EAAE,CAAC;YACxB,MAAM,OAAO,GAAG,QAAQ,EAAE,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG;gBAChB,OAAO;gBACP,YAAY,EAAE,EAAE;gBAChB,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,GAAG,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;aACrE,CAAC;YACF,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC;YAC/C,MAAM,cAAc,GAAG,sBAAsB,EAAE,WAAW,aAAa,EAAE,CAAC;YAC1E,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;YAC9C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;YAC1C,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;YACnC,QAAQ,CAAC,IAAI,CACX;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,kBAAkB,IAAI,CAAC,SAAS,CAAC,EAAE,aAAa,EAAE,cAAc,EAAE,CAAC,EAAE;aAC5E,EACD,IAAI,CACL,CAAC;QACJ,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,KAA8B,EAAE,MAAgB;QAC5D,MAAM,SAAS,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;QAC/B,MAAM,OAAO,GAAG,CAAC,KAAc,EAAW,EAAE;YAC1C,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,OAAO,KAAK,CAAC;YACf,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACxC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,4BAA4B,KAAK,EAAE,CAAC,CAAC;YACvD,CAAC;YACD,OAAO,SAAS,CAAC;QACnB,CAAC,CAAC;QACF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,KAAK,IAAI,SAAS,EAAE,CAAC;gBACvB,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;gBAC/B,SAAS,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;oBACrC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC;oBACpB,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YACpE,IAAI,CAAC,SAAS,KAAK,CAAC,KAAK,IAAI,EAAE;gBAC7B,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,KAAK,EAAE,CAAC;gBAC3C,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;YACvC,CAAC,CAAC,EAAE,CAAC;YACL,MAAM,IAAI,CAAC,SAAS,CAAC;YACrB,MAAM,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,EAAiB,CAAC;YACtD,KAAK,MAAM,EAAE,SAAS,EAAE,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;gBAChD,SAAS,CAAC,YAAY,GAAG,oBAAoB,IAAI,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;YAC3E,CAAC;QACH,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YACpB,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;CACF"}
|
package/dist/matchers/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
import "./matchers.js";
|
|
2
|
-
export type { ChatAssertion, ChatMatchers, JudgmentOptions, } from "./matchers.js";
|
|
2
|
+
export type { ChatAssertion, ChatMatchers, Judgment, JudgmentOptions, Verdict, } from "./matchers.js";
|
|
3
3
|
export type { ChatLike, ToolCall, Turn } from "./types.js";
|
|
4
4
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/matchers/index.ts"],"names":[],"mappings":"AAAA,OAAO,eAAe,CAAC;AAEvB,YAAY,EACV,aAAa,EACb,YAAY,EACZ,eAAe,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/matchers/index.ts"],"names":[],"mappings":"AAAA,OAAO,eAAe,CAAC;AAEvB,YAAY,EACV,aAAa,EACb,YAAY,EACZ,QAAQ,EACR,eAAe,EACf,OAAO,GACR,MAAM,eAAe,CAAC;AACvB,YAAY,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC"}
|
|
@@ -1,11 +1,31 @@
|
|
|
1
1
|
import "vitest";
|
|
2
|
-
import type { LanguageModel } from "ai";
|
|
3
|
-
import type { ToolInput, ToolNames } from "skybridge/server";
|
|
2
|
+
import type { FilePart, LanguageModel } from "ai";
|
|
3
|
+
import type { FileRef, ToolInput, ToolNames } from "skybridge/server";
|
|
4
4
|
import type { ChatLike } from "./types.js";
|
|
5
|
+
export declare function registerInputFile(reference: FileRef, part: FilePart): void;
|
|
6
|
+
export declare function registerFileParameters(chat: object, fields: ReadonlyMap<string, readonly string[]>): void;
|
|
7
|
+
/** What a judge decided about a conversation. */
|
|
8
|
+
export interface Verdict {
|
|
9
|
+
pass: boolean;
|
|
10
|
+
/** Why, quoted in the failure message. Omit it when the judge gives no prose. */
|
|
11
|
+
reasoning?: string;
|
|
12
|
+
}
|
|
13
|
+
/** The conversation handed to a {@link JudgmentOptions.judge}. */
|
|
14
|
+
export interface Judgment {
|
|
15
|
+
criteria: string;
|
|
16
|
+
/** The rendered conversation, one turn per line, tool results included. */
|
|
17
|
+
transcript: string;
|
|
18
|
+
}
|
|
5
19
|
/** Options for {@link ChatMatchers.toPassJudgment}. */
|
|
6
20
|
export interface JudgmentOptions {
|
|
7
21
|
/** Overrides the judge for this assertion. Defaults to the chat's model. */
|
|
8
22
|
model?: LanguageModel;
|
|
23
|
+
/**
|
|
24
|
+
* Grades the conversation instead of the default model call, so an
|
|
25
|
+
* evaluation model or a scoring service can stand in for the judge. Takes
|
|
26
|
+
* precedence over {@link JudgmentOptions.model}.
|
|
27
|
+
*/
|
|
28
|
+
judge?: (judgment: Judgment) => Verdict | Promise<Verdict>;
|
|
9
29
|
}
|
|
10
30
|
/**
|
|
11
31
|
* Assertions about the tool calls a conversation produced. Kept off vitest's
|
|
@@ -27,6 +47,7 @@ export interface ChatAssertion<App> extends ChatMatchers<App> {
|
|
|
27
47
|
}
|
|
28
48
|
declare module "vitest" {
|
|
29
49
|
interface ExpectStatic {
|
|
50
|
+
inputFile(part: FilePart): FileRef;
|
|
30
51
|
chat<App>(chat: ChatLike<App>): ChatAssertion<App>;
|
|
31
52
|
}
|
|
32
53
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"matchers.d.ts","sourceRoot":"","sources":["../../src/matchers/matchers.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,CAAC;AAChB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"matchers.d.ts","sourceRoot":"","sources":["../../src/matchers/matchers.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,CAAC;AAChB,OAAO,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAClD,OAAO,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAEtE,OAAO,KAAK,EAAE,QAAQ,EAAkB,MAAM,YAAY,CAAC;AAS3D,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,QAInE;AAED,wBAAgB,sBAAsB,CACpC,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,QAG/C;AA0BD,iDAAiD;AACjD,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,OAAO,CAAC;IACd,iFAAiF;IACjF,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,kEAAkE;AAClE,MAAM,WAAW,QAAQ;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,2EAA2E;IAC3E,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,uDAAuD;AACvD,MAAM,WAAW,eAAe;IAC9B,4EAA4E;IAC5E,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB;;;;OAIG;IACH,KAAK,CAAC,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAC5D;AAED;;;;GAIG;AACH,MAAM,WAAW,YAAY,CAAC,GAAG;IAC/B,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAC9C,IAAI,EAAE,IAAI,EACV,IAAI,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GACnC,IAAI,CAAC;IACR,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAC9C,IAAI,EAAE,IAAI,EACV,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GAClC,IAAI,CAAC;IACR,qBAAqB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,CAAC;IACrE,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,CAAC;IACpE,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IACxC,wBAAwB,CACtB,GAAG,KAAK,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,GAAG,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,GAC9C,IAAI,CAAC;IACR,mBAAmB,IAAI,IAAI,CAAC;IAC5B,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5E;AAED,MAAM,WAAW,aAAa,CAAC,GAAG,CAAE,SAAQ,YAAY,CAAC,GAAG,CAAC;IAC3D,GAAG,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC;CACxB;AAED,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,YAAY;QACpB,SAAS,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC;QACnC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;KACpD;CACF"}
|
|
@@ -1,10 +1,48 @@
|
|
|
1
1
|
import "vitest";
|
|
2
2
|
import { expect } from "vitest";
|
|
3
|
+
const fileIds = new WeakMap();
|
|
4
|
+
const fileMatchers = new WeakSet();
|
|
5
|
+
const fileParameters = new WeakMap();
|
|
6
|
+
export function registerInputFile(reference, part) {
|
|
7
|
+
const ids = fileIds.get(part) ?? new Set();
|
|
8
|
+
ids.add(reference.file_id);
|
|
9
|
+
fileIds.set(part, ids);
|
|
10
|
+
}
|
|
11
|
+
export function registerFileParameters(chat, fields) {
|
|
12
|
+
fileParameters.set(chat, fields);
|
|
13
|
+
}
|
|
14
|
+
function inputFile(part) {
|
|
15
|
+
const label = () => `inputFile(${part.filename ?? part.mediaType})`;
|
|
16
|
+
const matcher = {
|
|
17
|
+
asymmetricMatch: (value) => typeof value === "object" &&
|
|
18
|
+
value !== null &&
|
|
19
|
+
"file_id" in value &&
|
|
20
|
+
typeof value.file_id === "string" &&
|
|
21
|
+
fileIds.get(part)?.has(value.file_id) === true,
|
|
22
|
+
toString: label,
|
|
23
|
+
toJSON: label,
|
|
24
|
+
};
|
|
25
|
+
fileMatchers.add(matcher);
|
|
26
|
+
return matcher;
|
|
27
|
+
}
|
|
28
|
+
function containsInputFile(value) {
|
|
29
|
+
return (typeof value === "object" &&
|
|
30
|
+
value !== null &&
|
|
31
|
+
(fileMatchers.has(value) || Object.values(value).some(containsInputFile)));
|
|
32
|
+
}
|
|
3
33
|
function calls(chat) {
|
|
4
34
|
return chat.turns
|
|
5
35
|
.filter((turn) => turn.role === "tool")
|
|
6
36
|
.map((turn) => turn.call);
|
|
7
37
|
}
|
|
38
|
+
function assertFileParameters(chat, name, args = {}) {
|
|
39
|
+
const fields = fileParameters.get(chat)?.get(name) ?? [];
|
|
40
|
+
for (const [field, value] of Object.entries(args)) {
|
|
41
|
+
if (containsInputFile(value) && !fields.includes(field)) {
|
|
42
|
+
throw new Error(`expect.inputFile cannot be used for "${name}.${field}": "${field}" is not listed in openai/fileParams`);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
}
|
|
8
46
|
function spokenTurns(chat) {
|
|
9
47
|
return chat.turns
|
|
10
48
|
.filter((turn) => turn.role === "assistant")
|
|
@@ -83,8 +121,27 @@ function report(chat, summary) {
|
|
|
83
121
|
Calls the model actually made:
|
|
84
122
|
${observed(chat)}`;
|
|
85
123
|
}
|
|
124
|
+
async function modelVerdict(chat, criteria, model) {
|
|
125
|
+
const { generateObject, jsonSchema } = await import("ai");
|
|
126
|
+
const { object } = await generateObject({
|
|
127
|
+
model: model ?? chat.model,
|
|
128
|
+
system: JUDGE_SYSTEM,
|
|
129
|
+
prompt: `Criteria:\n${criteria}\n\n<conversation>\n${transcriptFor(chat)}\n</conversation>`,
|
|
130
|
+
schema: jsonSchema({
|
|
131
|
+
type: "object",
|
|
132
|
+
properties: {
|
|
133
|
+
pass: { type: "boolean" },
|
|
134
|
+
reasoning: { type: "string" },
|
|
135
|
+
},
|
|
136
|
+
required: ["pass", "reasoning"],
|
|
137
|
+
additionalProperties: false,
|
|
138
|
+
}),
|
|
139
|
+
});
|
|
140
|
+
return object;
|
|
141
|
+
}
|
|
86
142
|
expect.extend({
|
|
87
143
|
toHaveCalledToolOnce(received, name, args) {
|
|
144
|
+
assertFileParameters(received, name, args);
|
|
88
145
|
const matching = acceptedCalls(received, name);
|
|
89
146
|
const argsMatch = args === undefined ||
|
|
90
147
|
(matching.length === 1 &&
|
|
@@ -100,6 +157,7 @@ expect.extend({
|
|
|
100
157
|
};
|
|
101
158
|
},
|
|
102
159
|
toHaveCalledToolWith(received, name, args) {
|
|
160
|
+
assertFileParameters(received, name, args);
|
|
103
161
|
const pass = acceptedCalls(received, name).some((call) => this.equals(call.arguments, expect.objectContaining(args)));
|
|
104
162
|
return {
|
|
105
163
|
pass,
|
|
@@ -164,23 +222,15 @@ ${spoken(received)}`,
|
|
|
164
222
|
async toPassJudgment(received, criteria, options) {
|
|
165
223
|
let verdict;
|
|
166
224
|
try {
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
pass: { type: "boolean" },
|
|
177
|
-
reasoning: { type: "string" },
|
|
178
|
-
},
|
|
179
|
-
required: ["pass", "reasoning"],
|
|
180
|
-
additionalProperties: false,
|
|
181
|
-
}),
|
|
182
|
-
});
|
|
183
|
-
verdict = object;
|
|
225
|
+
verdict = options?.judge
|
|
226
|
+
? await options.judge({
|
|
227
|
+
criteria,
|
|
228
|
+
transcript: transcriptFor(received),
|
|
229
|
+
})
|
|
230
|
+
: await modelVerdict(received, criteria, options?.model);
|
|
231
|
+
if (typeof verdict?.pass !== "boolean") {
|
|
232
|
+
throw new Error(`the judge returned ${JSON.stringify(verdict)} instead of a verdict with a boolean "pass"`);
|
|
233
|
+
}
|
|
184
234
|
}
|
|
185
235
|
catch (error) {
|
|
186
236
|
throw new Error(`judge unavailable: ${error instanceof Error ? error.message : String(error)}`);
|
|
@@ -190,8 +240,7 @@ ${spoken(received)}`,
|
|
|
190
240
|
message: () => `expected the conversation ${verdict.pass ? "not " : ""}to pass judgment:
|
|
191
241
|
"${criteria}"
|
|
192
242
|
|
|
193
|
-
judge: ${verdict.pass ? "PASS" : "FAIL"}
|
|
194
|
-
${verdict.reasoning}`,
|
|
243
|
+
judge: ${verdict.pass ? "PASS" : "FAIL"}${verdict.reasoning === undefined ? "" : `\n ${verdict.reasoning}`}`,
|
|
195
244
|
};
|
|
196
245
|
},
|
|
197
246
|
toNeverHaveCalledTool(received, name) {
|
|
@@ -205,4 +254,5 @@ judge: ${verdict.pass ? "PASS" : "FAIL"}
|
|
|
205
254
|
},
|
|
206
255
|
});
|
|
207
256
|
expect.chat = (chat) => expect(chat);
|
|
257
|
+
expect.inputFile = inputFile;
|
|
208
258
|
//# sourceMappingURL=matchers.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"matchers.js","sourceRoot":"","sources":["../../src/matchers/matchers.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,CAAC;AAGhB,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AA2ChC,SAAS,KAAK,CAAC,IAAuB;IACpC,OAAO,IAAI,CAAC,KAAK;SACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC;SACtC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC;AAED,SAAS,WAAW,CAAC,IAAuB;IAC1C,OAAO,IAAI,CAAC,KAAK;SACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,WAAW,CAAC;SAC3C,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC;AAED,SAAS,cAAc,CACrB,IAAuB,EACvB,IAAY;IAEZ,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,WAAW,CAAC,IAAuB;IAC1C,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;AACjE,CAAC;AAED,SAAS,aAAa,CACpB,IAAuB,EACvB,IAAY;IAEZ,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;AACxE,CAAC;AAED,SAAS,QAAQ,CAAC,IAAuB;IACvC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IACzB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,wBAAwB,CAAC;IAClC,CAAC;IACD,OAAO,IAAI;SACR,GAAG,CACF,CAAC,IAAuB,EAAE,KAAa,EAAE,EAAE,CACzC,KAAK,KAAK,GAAG,CAAC,KAAK,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,cAAc,IAAI,CAAC,MAAM,GAAG,EAAE,CACnI;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AAC1C,CAAC;AAED,SAAS,IAAI,CAAC,IAAuB,EAAE,IAAqB;IAC1D,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CACrC,OAAO,IAAI,KAAK,QAAQ;QACtB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;QACrE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CACpB,CAAC;AACJ,CAAC;AAED,SAAS,MAAM,CAAC,IAAuB;IACrC,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,gCAAgC,CAAC;IAC1C,CAAC;IACD,OAAO,KAAK;SACT,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,GAAG,CAAC,KAAK,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;SACzD,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,SAAS,KAAK,CAAC,IAAY;IACzB,OAAO,IAAI,CAAC,OAAO,CAAC,uBAAuB,EAAE,QAAQ,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,YAAY,GAAG;;;6KAGwJ,CAAC;AAE9K,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAE9B,SAAS,QAAQ,CAAC,IAAmB;IACnC,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QACzB,OAAO,GAAG,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;IAC7C,CAAC;IACD,MAAM,OAAO,GACX,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS;QAC5B,CAAC,CAAC,OAAO,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,EAAE;QACxD,CAAC,CAAC,aAAa,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;IACvC,OAAO,SAAS,KAAK,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,OAAO,EAAE,CAAC,EAAE,CAAC;AACxG,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,IAAI,CAAC,MAAM,GAAG,gBAAgB;QACnC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC,eAAe;QACnD,CAAC,CAAC,IAAI,CAAC;AACX,CAAC;AAED,SAAS,aAAa,CAAC,IAAuB;IAC5C,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,6BAA6B,CAAC;IACvC,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC7C,CAAC;AAED,SAAS,MAAM,CAAC,IAAuB,EAAE,OAAe;IACtD,OAAO,GAAG,OAAO;;;EAGjB,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;AACnB,CAAC;AAED,MAAM,CAAC,MAAM,CAAC;IACZ,oBAAoB,CAClB,QAA2B,EAC3B,IAAY,EACZ,IAA8B;QAE9B,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC/C,MAAM,SAAS,GACb,IAAI,KAAK,SAAS;YAClB,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACpB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxE,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC;QAEhD,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,aAAa,IAAI,kCAAkC;gBACrD,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;oBACrB,CAAC,CAAC,aAAa,IAAI,4BAA4B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,4BAA4B,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG;oBACxI,CAAC,CAAC,aAAa,IAAI,6DAA6D,QAAQ,CAAC,MAAM,wBAAwB,CAC5H;SACJ,CAAC;IACJ,CAAC;IAED,oBAAoB,CAClB,QAA2B,EAC3B,IAAY,EACZ,IAA6B;QAE7B,MAAM,IAAI,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CACvD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAC3D,CAAC;QAEF,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,wBAAwB,IAAI,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG;gBAC/D,CAAC,CAAC,aAAa,IAAI,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CACpE;SACJ,CAAC;IACJ,CAAC;IAED,oBAAoB,CAAC,QAA2B,EAAE,IAAY;QAC5D,MAAM,SAAS,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;QAErE,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC;YACvB,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,MAAM,CAAC,MAAM,GAAG,CAAC;gBACf,CAAC,CAAC,2BAA2B,IAAI,eAAe;gBAChD,CAAC,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;oBACtB,CAAC,CAAC,aAAa,IAAI,mDAAmD;oBACtE,CAAC,CAAC,aAAa,IAAI,sBAAsB,SAAS,CAAC,MAAM,mBAAmB,CACjF;SACJ,CAAC;IACJ,CAAC;IAED,UAAU,CAAC,QAA2B,EAAE,IAAqB;QAC3D,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAClC,MAAM,MAAM,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAErE,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,GACE,IAAI;gBACF,CAAC,CAAC,qCAAqC,MAAM,GAAG;gBAChD,CAAC,CAAC,iCAAiC,MAAM,GAC7C;;;EAGN,MAAM,CAAC,QAAQ,CAAC,EAAE;SACf,CAAC;IACJ,CAAC;IAED,mBAAmB,CAAC,QAA2B;QAC7C,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;QAErC,OAAO;YACL,IAAI,EAAE,KAAK,KAAK,CAAC;YACjB,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,KAAK,KAAK,CAAC;gBACT,CAAC,CAAC,+CAA+C;gBACjD,CAAC,CAAC,wDAAwD,KAAK,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,GAAG,CACvG;SACJ,CAAC;IACJ,CAAC;IAED,wBAAwB,CAAC,QAA2B,EAAE,GAAG,KAAe;QACtE,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;YACzC,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;gBACzC,OAAO,IAAI,CAAC,CAAC;YACf,CAAC;YACD,IAAI,OAAO,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;gBAC7B,MAAM;YACR,CAAC;QACH,CAAC;QACD,MAAM,IAAI,GAAG,OAAO,KAAK,KAAK,CAAC,MAAM,CAAC;QACtC,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAE/D,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,YAAY,MAAM,kCAAkC;gBACtD,CAAC,CAAC,YAAY,MAAM,wBAAwB,KAAK,CAAC,OAAO,CAAC,eAAe,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CACpI;SACJ,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,cAAc,CAClB,QAA2B,EAC3B,QAAgB,EAChB,OAAyB;QAEzB,IAAI,OAA6C,CAAC;QAClD,IAAI,CAAC;YACH,MAAM,EAAE,cAAc,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;YAC1D,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,cAAc,CAAC;gBACtC,KAAK,EAAE,OAAO,EAAE,KAAK,IAAI,QAAQ,CAAC,KAAK;gBACvC,WAAW,EAAE,CAAC;gBACd,MAAM,EAAE,YAAY;gBACpB,MAAM,EAAE,cAAc,QAAQ,uBAAuB,aAAa,CAAC,QAAQ,CAAC,mBAAmB;gBAC/F,MAAM,EAAE,UAAU,CAAuC;oBACvD,IAAI,EAAE,QAAQ;oBACd,UAAU,EAAE;wBACV,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;wBACzB,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC9B;oBACD,QAAQ,EAAE,CAAC,MAAM,EAAE,WAAW,CAAC;oBAC/B,oBAAoB,EAAE,KAAK;iBAC5B,CAAC;aACH,CAAC,CAAC;YACH,OAAO,GAAG,MAAM,CAAC;QACnB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,sBAAsB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC/E,CAAC;QACJ,CAAC;QAED,OAAO;YACL,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,OAAO,EAAE,GAAG,EAAE,CACZ,6BAA6B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;KAC1D,QAAQ;;SAEJ,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;IACnC,OAAO,CAAC,SAAS,EAAE;SAClB,CAAC;IACJ,CAAC;IAED,qBAAqB,CAAC,QAA2B,EAAE,IAAY;QAC7D,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAEhD,OAAO;YACL,IAAI,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC;YAC3B,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACnB,CAAC,CAAC,aAAa,IAAI,+BAA+B;gBAClD,CAAC,CAAC,aAAa,IAAI,2CAA2C,QAAQ,CAAC,MAAM,SAAS,CACzF;SACJ,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,IAAI,GAAG,CAAM,IAAmB,EAAsB,EAAE,CAC7D,MAAM,CAAC,IAAI,CAAkC,CAAC"}
|
|
1
|
+
{"version":3,"file":"matchers.js","sourceRoot":"","sources":["../../src/matchers/matchers.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,CAAC;AAGhB,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,OAAO,GAAG,IAAI,OAAO,EAAyB,CAAC;AACrD,MAAM,YAAY,GAAG,IAAI,OAAO,EAAU,CAAC;AAC3C,MAAM,cAAc,GAAG,IAAI,OAAO,EAG/B,CAAC;AAEJ,MAAM,UAAU,iBAAiB,CAAC,SAAkB,EAAE,IAAc;IAClE,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,EAAU,CAAC;IACnD,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IAC3B,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,IAAY,EACZ,MAA8C;IAE9C,cAAc,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;AACnC,CAAC;AAED,SAAS,SAAS,CAAC,IAAc;IAC/B,MAAM,KAAK,GAAG,GAAG,EAAE,CAAC,aAAa,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC;IACpE,MAAM,OAAO,GAAG;QACd,eAAe,EAAE,CAAC,KAAc,EAAE,EAAE,CAClC,OAAO,KAAK,KAAK,QAAQ;YACzB,KAAK,KAAK,IAAI;YACd,SAAS,IAAI,KAAK;YAClB,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ;YACjC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,IAAI;QAChD,QAAQ,EAAE,KAAK;QACf,MAAM,EAAE,KAAK;KACd,CAAC;IACF,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC1B,OAAO,OAA6B,CAAC;AACvC,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc;IACvC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAC1E,CAAC;AACJ,CAAC;AA+DD,SAAS,KAAK,CAAC,IAAuB;IACpC,OAAO,IAAI,CAAC,KAAK;SACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC;SACtC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC;AAED,SAAS,oBAAoB,CAC3B,IAAuB,EACvB,IAAY,EACZ,OAAgC,EAAE;IAElC,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IACzD,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QAClD,IAAI,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACxD,MAAM,IAAI,KAAK,CACb,wCAAwC,IAAI,IAAI,KAAK,OAAO,KAAK,sCAAsC,CACxG,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,IAAuB;IAC1C,OAAO,IAAI,CAAC,KAAK;SACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,WAAW,CAAC;SAC3C,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC;AAED,SAAS,cAAc,CACrB,IAAuB,EACvB,IAAY;IAEZ,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,WAAW,CAAC,IAAuB;IAC1C,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;AACjE,CAAC;AAED,SAAS,aAAa,CACpB,IAAuB,EACvB,IAAY;IAEZ,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;AACxE,CAAC;AAED,SAAS,QAAQ,CAAC,IAAuB;IACvC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IACzB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,wBAAwB,CAAC;IAClC,CAAC;IACD,OAAO,IAAI;SACR,GAAG,CACF,CAAC,IAAuB,EAAE,KAAa,EAAE,EAAE,CACzC,KAAK,KAAK,GAAG,CAAC,KAAK,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,cAAc,IAAI,CAAC,MAAM,GAAG,EAAE,CACnI;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AAC1C,CAAC;AAED,SAAS,IAAI,CAAC,IAAuB,EAAE,IAAqB;IAC1D,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CACrC,OAAO,IAAI,KAAK,QAAQ;QACtB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;QACrE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CACpB,CAAC;AACJ,CAAC;AAED,SAAS,MAAM,CAAC,IAAuB;IACrC,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,gCAAgC,CAAC;IAC1C,CAAC;IACD,OAAO,KAAK;SACT,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,GAAG,CAAC,KAAK,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;SACzD,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,SAAS,KAAK,CAAC,IAAY;IACzB,OAAO,IAAI,CAAC,OAAO,CAAC,uBAAuB,EAAE,QAAQ,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,YAAY,GAAG;;;6KAGwJ,CAAC;AAE9K,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAE9B,SAAS,QAAQ,CAAC,IAAmB;IACnC,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QACzB,OAAO,GAAG,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;IAC7C,CAAC;IACD,MAAM,OAAO,GACX,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS;QAC5B,CAAC,CAAC,OAAO,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,EAAE;QACxD,CAAC,CAAC,aAAa,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;IACvC,OAAO,SAAS,KAAK,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,OAAO,EAAE,CAAC,EAAE,CAAC;AACxG,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,IAAI,CAAC,MAAM,GAAG,gBAAgB;QACnC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC,eAAe;QACnD,CAAC,CAAC,IAAI,CAAC;AACX,CAAC;AAED,SAAS,aAAa,CAAC,IAAuB;IAC5C,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,6BAA6B,CAAC;IACvC,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC7C,CAAC;AAED,SAAS,MAAM,CAAC,IAAuB,EAAE,OAAe;IACtD,OAAO,GAAG,OAAO;;;EAGjB,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;AACnB,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,IAAuB,EACvB,QAAgB,EAChB,KAAqB;IAErB,MAAM,EAAE,cAAc,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;IAC1D,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,cAAc,CAAC;QACtC,KAAK,EAAE,KAAK,IAAI,IAAI,CAAC,KAAK;QAC1B,MAAM,EAAE,YAAY;QACpB,MAAM,EAAE,cAAc,QAAQ,uBAAuB,aAAa,CAAC,IAAI,CAAC,mBAAmB;QAC3F,MAAM,EAAE,UAAU,CAAkC;YAClD,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;gBACzB,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;aAC9B;YACD,QAAQ,EAAE,CAAC,MAAM,EAAE,WAAW,CAAC;YAC/B,oBAAoB,EAAE,KAAK;SAC5B,CAAC;KACH,CAAC,CAAC;IACH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,MAAM,CAAC;IACZ,oBAAoB,CAClB,QAA2B,EAC3B,IAAY,EACZ,IAA8B;QAE9B,oBAAoB,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;QAC3C,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC/C,MAAM,SAAS,GACb,IAAI,KAAK,SAAS;YAClB,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACpB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxE,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC;QAEhD,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,aAAa,IAAI,kCAAkC;gBACrD,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;oBACrB,CAAC,CAAC,aAAa,IAAI,4BAA4B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,4BAA4B,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG;oBACxI,CAAC,CAAC,aAAa,IAAI,6DAA6D,QAAQ,CAAC,MAAM,wBAAwB,CAC5H;SACJ,CAAC;IACJ,CAAC;IAED,oBAAoB,CAClB,QAA2B,EAC3B,IAAY,EACZ,IAA6B;QAE7B,oBAAoB,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;QAC3C,MAAM,IAAI,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CACvD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAC3D,CAAC;QAEF,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,wBAAwB,IAAI,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG;gBAC/D,CAAC,CAAC,aAAa,IAAI,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CACpE;SACJ,CAAC;IACJ,CAAC;IAED,oBAAoB,CAAC,QAA2B,EAAE,IAAY;QAC5D,MAAM,SAAS,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;QAErE,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC;YACvB,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,MAAM,CAAC,MAAM,GAAG,CAAC;gBACf,CAAC,CAAC,2BAA2B,IAAI,eAAe;gBAChD,CAAC,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;oBACtB,CAAC,CAAC,aAAa,IAAI,mDAAmD;oBACtE,CAAC,CAAC,aAAa,IAAI,sBAAsB,SAAS,CAAC,MAAM,mBAAmB,CACjF;SACJ,CAAC;IACJ,CAAC;IAED,UAAU,CAAC,QAA2B,EAAE,IAAqB;QAC3D,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAClC,MAAM,MAAM,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAErE,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,GACE,IAAI;gBACF,CAAC,CAAC,qCAAqC,MAAM,GAAG;gBAChD,CAAC,CAAC,iCAAiC,MAAM,GAC7C;;;EAGN,MAAM,CAAC,QAAQ,CAAC,EAAE;SACf,CAAC;IACJ,CAAC;IAED,mBAAmB,CAAC,QAA2B;QAC7C,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;QAErC,OAAO;YACL,IAAI,EAAE,KAAK,KAAK,CAAC;YACjB,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,KAAK,KAAK,CAAC;gBACT,CAAC,CAAC,+CAA+C;gBACjD,CAAC,CAAC,wDAAwD,KAAK,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,GAAG,CACvG;SACJ,CAAC;IACJ,CAAC;IAED,wBAAwB,CAAC,QAA2B,EAAE,GAAG,KAAe;QACtE,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;YACzC,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;gBACzC,OAAO,IAAI,CAAC,CAAC;YACf,CAAC;YACD,IAAI,OAAO,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;gBAC7B,MAAM;YACR,CAAC;QACH,CAAC;QACD,MAAM,IAAI,GAAG,OAAO,KAAK,KAAK,CAAC,MAAM,CAAC;QACtC,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAE/D,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,YAAY,MAAM,kCAAkC;gBACtD,CAAC,CAAC,YAAY,MAAM,wBAAwB,KAAK,CAAC,OAAO,CAAC,eAAe,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CACpI;SACJ,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,cAAc,CAClB,QAA2B,EAC3B,QAAgB,EAChB,OAAyB;QAEzB,IAAI,OAAgB,CAAC;QACrB,IAAI,CAAC;YACH,OAAO,GAAG,OAAO,EAAE,KAAK;gBACtB,CAAC,CAAC,MAAM,OAAO,CAAC,KAAK,CAAC;oBAClB,QAAQ;oBACR,UAAU,EAAE,aAAa,CAAC,QAAQ,CAAC;iBACpC,CAAC;gBACJ,CAAC,CAAC,MAAM,YAAY,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;YAE3D,IAAI,OAAO,OAAO,EAAE,IAAI,KAAK,SAAS,EAAE,CAAC;gBACvC,MAAM,IAAI,KAAK,CACb,sBAAsB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,6CAA6C,CAC3F,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,sBAAsB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC/E,CAAC;QACJ,CAAC;QAED,OAAO;YACL,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,OAAO,EAAE,GAAG,EAAE,CACZ,6BAA6B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;KAC1D,QAAQ;;SAEJ,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,OAAO,CAAC,SAAS,EAAE,EAAE;SACxG,CAAC;IACJ,CAAC;IAED,qBAAqB,CAAC,QAA2B,EAAE,IAAY;QAC7D,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAEhD,OAAO;YACL,IAAI,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC;YAC3B,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACnB,CAAC,CAAC,aAAa,IAAI,+BAA+B;gBAClD,CAAC,CAAC,aAAa,IAAI,2CAA2C,QAAQ,CAAC,MAAM,SAAS,CACzF;SACJ,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,IAAI,GAAG,CAAM,IAAmB,EAAsB,EAAE,CAC7D,MAAM,CAAC,IAAI,CAAkC,CAAC;AAChD,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { McpMiddlewareFn } from "skybridge/server";
|
|
2
|
+
import type { ToolStubs } from "../session-registry.js";
|
|
3
|
+
export declare function createListToolsMiddleware(fileParams: Map<string, string[]>, stubs: ToolStubs): McpMiddlewareFn;
|
|
4
|
+
//# sourceMappingURL=list-tools.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"list-tools.d.ts","sourceRoot":"","sources":["../../src/middleware/list-tools.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAoDxD,wBAAgB,yBAAyB,CACvC,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,EACjC,KAAK,EAAE,SAAS,GACf,eAAe,CA2CjB"}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
function fileInputSchema(schema, allowArray = true) {
|
|
2
|
+
if (!schema || typeof schema === "boolean") {
|
|
3
|
+
throw new Error("expected a FileRef schema");
|
|
4
|
+
}
|
|
5
|
+
if (schema.anyOf) {
|
|
6
|
+
if (schema.anyOf.every((branch) => typeof branch !== "boolean" && branch.type === "null")) {
|
|
7
|
+
throw new Error("expected a FileRef schema");
|
|
8
|
+
}
|
|
9
|
+
return {
|
|
10
|
+
...schema,
|
|
11
|
+
anyOf: schema.anyOf.map((branch) => typeof branch !== "boolean" && branch.type === "null"
|
|
12
|
+
? branch
|
|
13
|
+
: fileInputSchema(branch, allowArray)),
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
if (allowArray && schema.type === "array" && !Array.isArray(schema.items)) {
|
|
17
|
+
return { ...schema, items: fileInputSchema(schema.items, false) };
|
|
18
|
+
}
|
|
19
|
+
const fields = ["download_url", "file_id", "mime_type", "file_name"];
|
|
20
|
+
if (schema.type !== "object" ||
|
|
21
|
+
(schema.additionalProperties !== undefined &&
|
|
22
|
+
schema.additionalProperties !== false) ||
|
|
23
|
+
schema.required?.length !== 2 ||
|
|
24
|
+
!["download_url", "file_id"].every((field) => schema.required?.includes(field)) ||
|
|
25
|
+
Object.keys(schema.properties ?? {}).length !== fields.length ||
|
|
26
|
+
!fields.every((field) => {
|
|
27
|
+
const property = schema.properties?.[field];
|
|
28
|
+
return typeof property === "object" && property.type === "string";
|
|
29
|
+
})) {
|
|
30
|
+
throw new Error("expected a FileRef schema");
|
|
31
|
+
}
|
|
32
|
+
return {
|
|
33
|
+
type: "string",
|
|
34
|
+
description: "This parameter expects an absolute local file path.",
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
export function createListToolsMiddleware(fileParams, stubs) {
|
|
38
|
+
function toolList(tools) {
|
|
39
|
+
return tools.map((tool) => {
|
|
40
|
+
const fields = tool._meta?.["openai/fileParams"] ?? [];
|
|
41
|
+
if (!Array.isArray(fields) ||
|
|
42
|
+
!fields.every((field) => typeof field === "string")) {
|
|
43
|
+
throw new Error(`Invalid openai/fileParams for "${tool.name}": expected parameter names`);
|
|
44
|
+
}
|
|
45
|
+
fileParams.set(tool.name, fields);
|
|
46
|
+
const properties = { ...tool.inputSchema.properties };
|
|
47
|
+
for (const field of fields) {
|
|
48
|
+
try {
|
|
49
|
+
properties[field] = fileInputSchema(properties[field]);
|
|
50
|
+
}
|
|
51
|
+
catch {
|
|
52
|
+
throw new Error(`Invalid openai/fileParams for "${tool.name}.${field}": expected a FileRef schema (or an optional, nullable, or array form)`);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return { ...tool, inputSchema: { ...tool.inputSchema, properties } };
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
return async (_request, _extra, next) => {
|
|
59
|
+
const result = (await next());
|
|
60
|
+
const unknown = Object.keys(stubs).filter((name) => !result.tools.some((tool) => tool.name === name));
|
|
61
|
+
if (unknown.length) {
|
|
62
|
+
throw new Error(`the app exposes no tool named ${unknown.map((name) => `"${name}"`).join(", ")}, so the stub would never run`);
|
|
63
|
+
}
|
|
64
|
+
return { ...result, tools: toolList(result.tools) };
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
//# sourceMappingURL=list-tools.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"list-tools.js","sourceRoot":"","sources":["../../src/middleware/list-tools.ts"],"names":[],"mappings":"AAKA,SAAS,eAAe,CACtB,MAAyC,EACzC,UAAU,GAAG,IAAI;IAEjB,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,SAAS,EAAE,CAAC;QAC3C,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IACD,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,IACE,MAAM,CAAC,KAAK,CAAC,KAAK,CAChB,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,CAClE,EACD,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/C,CAAC;QACD,OAAO;YACL,GAAG,MAAM;YACT,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CACjC,OAAO,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM;gBACnD,CAAC,CAAC,MAAM;gBACR,CAAC,CAAC,eAAe,CAAC,MAAM,EAAE,UAAU,CAAC,CACxC;SACF,CAAC;IACJ,CAAC;IACD,IAAI,UAAU,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1E,OAAO,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,EAAE,CAAC;IACpE,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,cAAc,EAAE,SAAS,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC;IACrE,IACE,MAAM,CAAC,IAAI,KAAK,QAAQ;QACxB,CAAC,MAAM,CAAC,oBAAoB,KAAK,SAAS;YACxC,MAAM,CAAC,oBAAoB,KAAK,KAAK,CAAC;QACxC,MAAM,CAAC,QAAQ,EAAE,MAAM,KAAK,CAAC;QAC7B,CAAC,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAC3C,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,CACjC;QACD,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM;QAC7D,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACtB,MAAM,QAAQ,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC;YAC5C,OAAO,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC;QACpE,CAAC,CAAC,EACF,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,WAAW,EAAE,qDAAqD;KACnE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,yBAAyB,CACvC,UAAiC,EACjC,KAAgB;IAEhB,SAAS,QAAQ,CAAC,KAAa;QAC7B,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACxB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC;YACvD,IACE,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;gBACtB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,EACnD,CAAC;gBACD,MAAM,IAAI,KAAK,CACb,kCAAkC,IAAI,CAAC,IAAI,6BAA6B,CACzE,CAAC;YACJ,CAAC;YACD,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAClC,MAAM,UAAU,GAAG,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,EAElD,CAAC;YACF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,IAAI,CAAC;oBACH,UAAU,CAAC,KAAK,CAAC,GAAG,eAAe,CACjC,UAAU,CAAC,KAAK,CAAsC,CACvD,CAAC;gBACJ,CAAC;gBAAC,MAAM,CAAC;oBACP,MAAM,IAAI,KAAK,CACb,kCAAkC,IAAI,CAAC,IAAI,IAAI,KAAK,wEAAwE,CAC7H,CAAC;gBACJ,CAAC;YACH,CAAC;YACD,OAAO,EAAE,GAAG,IAAI,EAAE,WAAW,EAAE,EAAE,GAAG,IAAI,CAAC,WAAW,EAAE,UAAU,EAAE,EAAE,CAAC;QACvE,CAAC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE;QACtC,MAAM,MAAM,GAAG,CAAC,MAAM,IAAI,EAAE,CAAsB,CAAC;QACnD,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CACvC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,CAC3D,CAAC;QACF,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CACb,iCAAiC,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,+BAA+B,CAC9G,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;IACtD,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { McpMiddlewareFn } from "skybridge/server";
|
|
2
|
+
import type { InputFiles } from "../input-files.js";
|
|
3
|
+
export declare function createResolveFilesMiddleware(files: InputFiles, fileParams: Map<string, string[]>, inputs: Map<string, Record<string, unknown>>): McpMiddlewareFn;
|
|
4
|
+
//# sourceMappingURL=resolve-files.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"resolve-files.d.ts","sourceRoot":"","sources":["../../src/middleware/resolve-files.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAEpD,wBAAgB,4BAA4B,CAC1C,KAAK,EAAE,UAAU,EACjB,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,EACjC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,GAC3C,eAAe,CAgBjB"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export function createResolveFilesMiddleware(files, fileParams, inputs) {
|
|
2
|
+
return async (request, _extra, next) => {
|
|
3
|
+
const params = request.params;
|
|
4
|
+
const args = await files.rewrite((params.arguments ?? {}), fileParams.get(params.name) ?? []);
|
|
5
|
+
params.arguments = args;
|
|
6
|
+
const callId = params._meta?.["skybridge/evalToolCallId"];
|
|
7
|
+
if (typeof callId === "string") {
|
|
8
|
+
inputs.set(callId, args);
|
|
9
|
+
}
|
|
10
|
+
return next();
|
|
11
|
+
};
|
|
12
|
+
}
|
|
13
|
+
//# sourceMappingURL=resolve-files.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"resolve-files.js","sourceRoot":"","sources":["../../src/middleware/resolve-files.ts"],"names":[],"mappings":"AAGA,MAAM,UAAU,4BAA4B,CAC1C,KAAiB,EACjB,UAAiC,EACjC,MAA4C;IAE5C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE;QACrC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC9B,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,OAAO,CAC9B,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAA4B,EACnD,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,IAAc,CAAC,IAAI,EAAE,CAC5C,CAAC;QACF,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,MAAM,MAAM,GAAI,MAAM,CAAC,KAA6C,EAAE,CACpE,0BAA0B,CAC3B,CAAC;QACF,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC/B,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAC3B,CAAC;QACD,OAAO,IAAI,EAAE,CAAC;IAChB,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stub-tool.d.ts","sourceRoot":"","sources":["../../src/middleware/stub-tool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAExD,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,SAAS,GAAG,eAAe,CAiB1E"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export function createStubToolMiddleware(stubs) {
|
|
2
|
+
return async (request, _extra, next) => {
|
|
3
|
+
const result = await stubs[request.params.name]?.((request.params.arguments ?? {}));
|
|
4
|
+
return result === undefined
|
|
5
|
+
? next()
|
|
6
|
+
: {
|
|
7
|
+
content: [
|
|
8
|
+
{
|
|
9
|
+
type: "text",
|
|
10
|
+
text: typeof result === "string" ? result : JSON.stringify(result),
|
|
11
|
+
},
|
|
12
|
+
],
|
|
13
|
+
};
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
//# sourceMappingURL=stub-tool.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stub-tool.js","sourceRoot":"","sources":["../../src/middleware/stub-tool.ts"],"names":[],"mappings":"AAGA,MAAM,UAAU,wBAAwB,CAAC,KAAgB;IACvD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE;QACrC,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAc,CAAC,EAAE,CACzD,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAA4B,CAC5D,CAAC;QACF,OAAO,MAAM,KAAK,SAAS;YACzB,CAAC,CAAC,IAAI,EAAE;YACR,CAAC,CAAC;gBACE,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EACF,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;qBAC/D;iBACF;aACF,CAAC;IACR,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -1,18 +1,8 @@
|
|
|
1
|
-
import { type Server } from "@modelcontextprotocol/server";
|
|
2
1
|
import type { LanguageModel } from "ai";
|
|
2
|
+
import type { Skybridge, ToolDef, ToolInput, ToolNames } from "skybridge/server";
|
|
3
3
|
import { Chat } from "./chat.js";
|
|
4
|
-
/**
|
|
5
|
-
|
|
6
|
-
* `$types` marker the assertions infer tool names from, and the per-request
|
|
7
|
-
* server builder the session's handler is built on. Structural on purpose, so
|
|
8
|
-
* this package needs no runtime dependency on `skybridge`.
|
|
9
|
-
*/
|
|
10
|
-
export interface EvalApp {
|
|
11
|
-
readonly $types: {
|
|
12
|
-
readonly tools: object;
|
|
13
|
-
};
|
|
14
|
-
createServerInstance(): Promise<Server>;
|
|
15
|
-
}
|
|
4
|
+
/** The Skybridge surface used by an in-process eval session. */
|
|
5
|
+
export type EvalApp = Pick<Skybridge, "$types" | "createServerInstance">;
|
|
16
6
|
/**
|
|
17
7
|
* The identity an in-process session claims, handed to the app as the
|
|
18
8
|
* request's `authInfo`. Mirrors the SDK's `AuthInfo` structurally; `extra`
|
|
@@ -26,6 +16,15 @@ export interface EvalIdentity {
|
|
|
26
16
|
resource?: URL;
|
|
27
17
|
extra?: Record<string, unknown>;
|
|
28
18
|
}
|
|
19
|
+
/**
|
|
20
|
+
* Results the runner answers with instead of calling the app, keyed by tool
|
|
21
|
+
* name and typed against the project's own registry. A stub may be async, and
|
|
22
|
+
* one that returns `undefined` falls through to the real server, so a scenario
|
|
23
|
+
* can pin one set of arguments and leave the rest live.
|
|
24
|
+
*/
|
|
25
|
+
export type ToolStubs<App = Skybridge<Record<string, ToolDef<Record<string, unknown>>>>> = {
|
|
26
|
+
[Name in ToolNames<App>]?: (args: ToolInput<App, Name>) => unknown;
|
|
27
|
+
};
|
|
29
28
|
interface StartOptions {
|
|
30
29
|
/** Any AI SDK model instance, built by the project. */
|
|
31
30
|
model: LanguageModel;
|
|
@@ -43,6 +42,11 @@ interface StartOptions {
|
|
|
43
42
|
* The assertions are inferred from the app value itself, so `expect.chat` gets
|
|
44
43
|
* the project's tool names and argument shapes with no type parameter.
|
|
45
44
|
*
|
|
45
|
+
* `stubs` answers a tool from the scenario instead of the app, so a case that
|
|
46
|
+
* depends on today's date or on a live catalogue still reads the same weeks
|
|
47
|
+
* later. A stubbed call bypasses the application pipeline, and shows up in
|
|
48
|
+
* `chat.toolCalls` like any other. See {@link ToolStubs}.
|
|
49
|
+
*
|
|
46
50
|
* `authInfo` claims an identity for the session: the app's per-tool scheme and
|
|
47
51
|
* scope enforcement runs against it for real, only token verification is
|
|
48
52
|
* skipped. Omit it to exercise the anonymous path, challenges included.
|
|
@@ -50,6 +54,7 @@ interface StartOptions {
|
|
|
50
54
|
export declare function start<App extends EvalApp>(options: StartOptions & {
|
|
51
55
|
app: App;
|
|
52
56
|
authInfo?: EvalIdentity;
|
|
57
|
+
stubs?: ToolStubs<App>;
|
|
53
58
|
}): Promise<Chat<App>>;
|
|
54
59
|
export {};
|
|
55
60
|
//# sourceMappingURL=session-registry.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACxC,OAAO,KAAK,EACV,SAAS,EACT,OAAO,EACP,SAAS,EACT,SAAS,EACV,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC,gEAAgE;AAChE,MAAM,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,EAAE,QAAQ,GAAG,sBAAsB,CAAC,CAAC;AAEzE;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,GAAG,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACjC;AAED;;;;;GAKG;AACH,MAAM,MAAM,SAAS,CACnB,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAC/D;KACD,IAAI,IAAI,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,EAAE,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,OAAO;CACnE,CAAC;AAEF,UAAU,YAAY;IACpB,uDAAuD;IACvD,KAAK,EAAE,aAAa,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAUD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,KAAK,CAAC,GAAG,SAAS,OAAO,EAC7C,OAAO,EAAE,YAAY,GAAG;IACtB,GAAG,EAAE,GAAG,CAAC;IACT,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,KAAK,CAAC,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC;CACxB,GACA,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAoCpB"}
|
package/dist/session-registry.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { createMcpHandler } from "@modelcontextprotocol/server";
|
|
2
2
|
import { inject, onTestFinished } from "vitest";
|
|
3
3
|
import { Chat } from "./chat.js";
|
|
4
|
+
import { createEvalMiddlewares } from "./eval-middlewares.js";
|
|
4
5
|
function sharedDefaults() {
|
|
5
6
|
try {
|
|
6
7
|
return inject("skybridgeEvals");
|
|
@@ -19,6 +20,11 @@ function sharedDefaults() {
|
|
|
19
20
|
* The assertions are inferred from the app value itself, so `expect.chat` gets
|
|
20
21
|
* the project's tool names and argument shapes with no type parameter.
|
|
21
22
|
*
|
|
23
|
+
* `stubs` answers a tool from the scenario instead of the app, so a case that
|
|
24
|
+
* depends on today's date or on a live catalogue still reads the same weeks
|
|
25
|
+
* later. A stubbed call bypasses the application pipeline, and shows up in
|
|
26
|
+
* `chat.toolCalls` like any other. See {@link ToolStubs}.
|
|
27
|
+
*
|
|
22
28
|
* `authInfo` claims an identity for the session: the app's per-tool scheme and
|
|
23
29
|
* scope enforcement runs against it for real, only token verification is
|
|
24
30
|
* skipped. Omit it to exercise the anonymous path, challenges included.
|
|
@@ -26,16 +32,27 @@ function sharedDefaults() {
|
|
|
26
32
|
export async function start(options) {
|
|
27
33
|
const config = sharedDefaults();
|
|
28
34
|
const { app, authInfo } = options;
|
|
29
|
-
const
|
|
30
|
-
const
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
35
|
+
const middleware = createEvalMiddlewares((options.stubs ?? {}));
|
|
36
|
+
const handler = createMcpHandler(() => app.createServerInstance([
|
|
37
|
+
{ filter: "tools/list", handler: middleware.listTools },
|
|
38
|
+
{ filter: "tools/call", handler: middleware.resolveFiles },
|
|
39
|
+
{ filter: "tools/call", handler: middleware.stubTool },
|
|
40
|
+
]));
|
|
41
|
+
let chat;
|
|
42
|
+
try {
|
|
43
|
+
chat = await Chat.open({
|
|
44
|
+
model: options.model,
|
|
45
|
+
temperature: options.temperature ?? config?.temperature,
|
|
46
|
+
systemPrompt: options.systemPrompt ?? config?.systemPrompt,
|
|
47
|
+
maxSteps: options.maxSteps ?? config?.maxSteps,
|
|
48
|
+
}, (url, init) => handler.fetch(new Request(url, init), authInfo === undefined ? undefined : { authInfo }), middleware, () => handler.close());
|
|
49
|
+
}
|
|
50
|
+
catch (error) {
|
|
51
|
+
await Promise.allSettled([middleware.files.close(), handler.close()]);
|
|
52
|
+
throw error;
|
|
53
|
+
}
|
|
36
54
|
onTestFinished(async () => {
|
|
37
55
|
await chat.close();
|
|
38
|
-
await handler.close();
|
|
39
56
|
});
|
|
40
57
|
return chat;
|
|
41
58
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-registry.js","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,
|
|
1
|
+
{"version":3,"file":"session-registry.js","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAQhE,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAuC9D,SAAS,cAAc;IACrB,IAAI,CAAC;QACH,OAAO,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CACzB,OAIC;IAED,MAAM,MAAM,GAAG,cAAc,EAAE,CAAC;IAChC,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAClC,MAAM,UAAU,GAAG,qBAAqB,CAAC,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE,CAAc,CAAC,CAAC;IAC7E,MAAM,OAAO,GAAG,gBAAgB,CAAC,GAAG,EAAE,CACpC,GAAG,CAAC,oBAAoB,CAAC;QACvB,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,UAAU,CAAC,SAAS,EAAE;QACvD,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,UAAU,CAAC,YAAY,EAAE;QAC1D,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,UAAU,CAAC,QAAQ,EAAE;KACvD,CAAC,CACH,CAAC;IACF,IAAI,IAAe,CAAC;IACpB,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CACpB;YACE,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,MAAM,EAAE,WAAW;YACvD,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,MAAM,EAAE,YAAY;YAC1D,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,MAAM,EAAE,QAAQ;SAC/C,EACD,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CACZ,OAAO,CAAC,KAAK,CACX,IAAI,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,EACtB,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAClD,EACH,UAAU,EACV,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,CACtB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACtE,MAAM,KAAK,CAAC;IACd,CAAC;IACD,cAAc,CAAC,KAAK,IAAI,EAAE;QACxB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC,CAAC,CAAC;IACH,OAAO,IAAI,CAAC;AACd,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@skybridge/test",
|
|
3
|
-
"version": "0.0.0-dev.
|
|
4
|
-
"description": "Eval runner for Skybridge MCP
|
|
3
|
+
"version": "0.0.0-dev.1b18ac7",
|
|
4
|
+
"description": "Eval runner for Skybridge MCP Apps: does a prompt reach the right tool with the right arguments",
|
|
5
5
|
"license": "ISC",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"repository": {
|
|
@@ -37,8 +37,8 @@
|
|
|
37
37
|
"typescript": "^5.9.3",
|
|
38
38
|
"vitest": "^4.1.10",
|
|
39
39
|
"zod": "^4.4.3",
|
|
40
|
-
"@skybridge/vite-plugin": "0.0.0-dev.
|
|
41
|
-
"skybridge": "0.0.0-dev.
|
|
40
|
+
"@skybridge/vite-plugin": "0.0.0-dev.1b18ac7",
|
|
41
|
+
"skybridge": "0.0.0-dev.1b18ac7"
|
|
42
42
|
},
|
|
43
43
|
"engines": {
|
|
44
44
|
"node": ">=22.23.1"
|