@skybridge/test 0.0.0-dev.965222c → 0.0.0-dev.97e4bf7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -0
- package/dist/chat.d.ts +5 -4
- package/dist/chat.d.ts.map +1 -1
- package/dist/chat.js +39 -34
- package/dist/chat.js.map +1 -1
- package/dist/eval-middleware.d.ts +14 -0
- package/dist/eval-middleware.d.ts.map +1 -0
- package/dist/eval-middleware.js +147 -0
- package/dist/eval-middleware.js.map +1 -0
- package/dist/input-files.d.ts +12 -0
- package/dist/input-files.d.ts.map +1 -0
- package/dist/input-files.js +109 -0
- package/dist/input-files.js.map +1 -0
- package/dist/matchers/matchers.d.ts +5 -2
- package/dist/matchers/matchers.d.ts.map +1 -1
- package/dist/matchers/matchers.js +35 -0
- package/dist/matchers/matchers.js.map +1 -1
- package/dist/session-registry.d.ts.map +1 -1
- package/dist/session-registry.js +16 -6
- package/dist/session-registry.js.map +1 -1
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -82,6 +82,36 @@ from, so a project can set them once instead of per scenario.
|
|
|
82
82
|
skybridge({ evals: { temperature: 0, maxSteps: 6 } });
|
|
83
83
|
```
|
|
84
84
|
|
|
85
|
+
## Input files
|
|
86
|
+
|
|
87
|
+
`chat.send` also accepts AI SDK `UserContent`. Attach a `FilePart` and use the same object in `expect.inputFile` to check which file a tool received:
|
|
88
|
+
|
|
89
|
+
```ts
|
|
90
|
+
import { readFile } from "node:fs/promises";
|
|
91
|
+
import type { FilePart } from "ai";
|
|
92
|
+
|
|
93
|
+
const portrait: FilePart = {
|
|
94
|
+
type: "file",
|
|
95
|
+
mediaType: "image/png",
|
|
96
|
+
filename: "photo.png",
|
|
97
|
+
data: await readFile(new URL("./fixtures/photo.png", import.meta.url)),
|
|
98
|
+
};
|
|
99
|
+
|
|
100
|
+
await chat.send([{ type: "text", text: "Resize this photo" }, portrait]);
|
|
101
|
+
|
|
102
|
+
expect.chat(chat).toHaveCalledToolWith("image_crop_and_resize", {
|
|
103
|
+
file: expect.inputFile(portrait),
|
|
104
|
+
});
|
|
105
|
+
expect.chat(chat).toHaveCalledToolOnce("asset_openai_file_upload", {
|
|
106
|
+
files: [expect.inputFile(portrait)],
|
|
107
|
+
});
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
`file` and `files` must be listed in their tools' `_meta["openai/fileParams"]`.
|
|
111
|
+
Misplaced file matchers throw immediately, including nested matchers and `.not`.
|
|
112
|
+
`start` validates that each declared parameter uses `FileRef`, including optional, nullable, and array forms. The eval middleware exposes paths to the model and resolves them before handlers or stubs run.
|
|
113
|
+
See [Evals](https://docs.skybridge.tech/test/evals) for how files reach the model and the tool.
|
|
114
|
+
|
|
85
115
|
## Matchers
|
|
86
116
|
|
|
87
117
|
Assertions on what the assistant said work the same way, though model prose
|
package/dist/chat.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { type LanguageModel } from "ai";
|
|
1
|
+
import { type LanguageModel, type UserContent } from "ai";
|
|
2
|
+
import type { EvalMiddleware } from "./eval-middleware.js";
|
|
2
3
|
import type { ToolCall, Turn } from "./matchers/types.js";
|
|
3
4
|
interface HostConfig {
|
|
4
5
|
model: LanguageModel;
|
|
@@ -23,6 +24,7 @@ export type TransportFetch = (url: string | URL, init?: RequestInit) => Promise<
|
|
|
23
24
|
* {@link Chat.toolCalls}, the sequence of calls the model made.
|
|
24
25
|
*/
|
|
25
26
|
export declare class Chat<App = unknown> {
|
|
27
|
+
private readonly middleware;
|
|
26
28
|
/** Type-only anchor so `expect.chat` infers `App` from the conversation. */
|
|
27
29
|
readonly $app: App;
|
|
28
30
|
/** Everything that happened, in the order it happened. */
|
|
@@ -36,14 +38,13 @@ export declare class Chat<App = unknown> {
|
|
|
36
38
|
private readonly client;
|
|
37
39
|
private readonly transport;
|
|
38
40
|
private readonly host;
|
|
39
|
-
private readonly stubs;
|
|
40
41
|
private tools;
|
|
41
42
|
/** The model the conversation runs on, and the judge's default. */
|
|
42
43
|
get model(): LanguageModel;
|
|
43
44
|
private constructor();
|
|
44
|
-
static open<App>(host: HostConfig,
|
|
45
|
+
static open<App>(host: HostConfig, middleware: EvalMiddleware): Promise<Chat<App>>;
|
|
45
46
|
/** Takes a turn. Assert on the result with `expect.chat`. */
|
|
46
|
-
send(
|
|
47
|
+
send(content: UserContent): Promise<void>;
|
|
47
48
|
close(): Promise<void>;
|
|
48
49
|
private failureFor;
|
|
49
50
|
private toolSet;
|
package/dist/chat.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAOA,OAAO,EAKL,KAAK,aAAa,EAGlB,KAAK,WAAW,EACjB,MAAM,IAAI,CAAC;AACZ,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAE3D,OAAO,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AAE1D,UAAU,UAAU;IAClB,KAAK,EAAE,aAAa,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;GAKG;AACH,MAAM,MAAM,OAAO,GAAG,MAAM,CAC1B,MAAM,EACN,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,OAAO,CAC3C,CAAC;AAaF;;;GAGG;AACH,MAAM,MAAM,cAAc,GAAG,CAC3B,GAAG,EAAE,MAAM,GAAG,GAAG,EACjB,IAAI,CAAC,EAAE,WAAW,KACf,OAAO,CAAC,QAAQ,CAAC,CAAC;AAEvB;;;GAGG;AACH,qBAAa,IAAI,CAAC,GAAG,GAAG,OAAO;IAqC3B,OAAO,CAAC,QAAQ,CAAC,UAAU;IApC7B,4EAA4E;IAC5E,SAAiB,IAAI,EAAE,GAAG,CAAC;IAE3B,0DAA0D;IAC1D,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAM;IAEjC,mEAAmE;IACnE,IAAI,SAAS,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE,CAI/B;IAED,+EAA+E;IAC/E,IAAI,cAAc,IAAI,MAAM,EAAE,CAI7B;IAED,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA6B;IACtD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAa;IAClC,OAAO,CAAC,KAAK,CAAc;IAE3B,mEAAmE;IACnE,IAAI,KAAK,IAAI,aAAa,CAEzB;IAED,OAAO;WAWM,IAAI,CAAC,GAAG,EACnB,IAAI,EAAE,UAAU,EAChB,UAAU,EAAE,cAAc,GACzB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAmBrB,6DAA6D;IACvD,IAAI,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAmDzC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAK5B,OAAO,CAAC,UAAU;IAalB,OAAO,CAAC,OAAO;CAkDhB"}
|
package/dist/chat.js
CHANGED
|
@@ -1,21 +1,8 @@
|
|
|
1
|
-
import { Client, StreamableHTTPClientTransport, } from "@modelcontextprotocol/client";
|
|
1
|
+
import { Client, fromJsonSchema, StreamableHTTPClientTransport, } from "@modelcontextprotocol/client";
|
|
2
2
|
import { dynamicTool, generateText, jsonSchema, stepCountIs, } from "ai";
|
|
3
|
+
import { registerFileParameters } from "./matchers/matchers.js";
|
|
3
4
|
const DEFAULT_MAX_STEPS = 8;
|
|
4
5
|
const IN_PROCESS_URL = "http://in-process.skybridge.test/mcp";
|
|
5
|
-
/**
|
|
6
|
-
* Dress a stubbed value the way the server dresses a real one, so the model
|
|
7
|
-
* reads the same envelope under a stub as it does live.
|
|
8
|
-
*/
|
|
9
|
-
function asToolResult(value) {
|
|
10
|
-
return {
|
|
11
|
-
content: [
|
|
12
|
-
{
|
|
13
|
-
type: "text",
|
|
14
|
-
text: typeof value === "string" ? value : JSON.stringify(value),
|
|
15
|
-
},
|
|
16
|
-
],
|
|
17
|
-
};
|
|
18
|
-
}
|
|
19
6
|
function outputOf(results, toolCallId) {
|
|
20
7
|
return results.find((result) => result.toolCallId === toolCallId)?.output;
|
|
21
8
|
}
|
|
@@ -24,6 +11,7 @@ function outputOf(results, toolCallId) {
|
|
|
24
11
|
* {@link Chat.toolCalls}, the sequence of calls the model made.
|
|
25
12
|
*/
|
|
26
13
|
export class Chat {
|
|
14
|
+
middleware;
|
|
27
15
|
/** Everything that happened, in the order it happened. */
|
|
28
16
|
turns = [];
|
|
29
17
|
/** Every call the model made, in order, as it crossed the wire. */
|
|
@@ -43,30 +31,26 @@ export class Chat {
|
|
|
43
31
|
client;
|
|
44
32
|
transport;
|
|
45
33
|
host;
|
|
46
|
-
stubs;
|
|
47
34
|
tools = [];
|
|
48
35
|
/** The model the conversation runs on, and the judge's default. */
|
|
49
36
|
get model() {
|
|
50
37
|
return this.host.model;
|
|
51
38
|
}
|
|
52
|
-
constructor(client, transport, host,
|
|
39
|
+
constructor(client, transport, host, middleware) {
|
|
40
|
+
this.middleware = middleware;
|
|
53
41
|
this.client = client;
|
|
54
42
|
this.transport = transport;
|
|
55
43
|
this.host = host;
|
|
56
|
-
this.stubs = new Map(Object.entries(stubs));
|
|
57
44
|
}
|
|
58
|
-
static async open(host,
|
|
45
|
+
static async open(host, middleware) {
|
|
59
46
|
const client = new Client({ name: "skybridge-eval", version: "0" });
|
|
60
|
-
const transport = new StreamableHTTPClientTransport(new URL(IN_PROCESS_URL), { fetch:
|
|
47
|
+
const transport = new StreamableHTTPClientTransport(new URL(IN_PROCESS_URL), { fetch: (url, init) => middleware.fetch(url, init) });
|
|
61
48
|
await client.connect(transport);
|
|
62
|
-
const chat = new Chat(client, transport, host,
|
|
49
|
+
const chat = new Chat(client, transport, host, middleware);
|
|
63
50
|
try {
|
|
64
51
|
const { tools } = await client.listTools();
|
|
65
52
|
chat.tools = tools;
|
|
66
|
-
|
|
67
|
-
if (unknownStubs.length > 0) {
|
|
68
|
-
throw new Error(`the app exposes no tool named ${unknownStubs.map((name) => `"${name}"`).join(", ")}, so the stub would never run`);
|
|
69
|
-
}
|
|
53
|
+
registerFileParameters(chat, middleware.fileParams);
|
|
70
54
|
}
|
|
71
55
|
catch (error) {
|
|
72
56
|
await transport.close();
|
|
@@ -75,9 +59,20 @@ export class Chat {
|
|
|
75
59
|
return chat;
|
|
76
60
|
}
|
|
77
61
|
/** Takes a turn. Assert on the result with `expect.chat`. */
|
|
78
|
-
async send(
|
|
79
|
-
this.
|
|
80
|
-
this.
|
|
62
|
+
async send(content) {
|
|
63
|
+
const prepared = await this.middleware.files.prepare(content);
|
|
64
|
+
this.middleware.inputs.clear();
|
|
65
|
+
this.failures.clear();
|
|
66
|
+
this.messages.push({ role: "user", content: prepared });
|
|
67
|
+
this.turns.push({
|
|
68
|
+
role: "user",
|
|
69
|
+
text: typeof prepared === "string"
|
|
70
|
+
? prepared
|
|
71
|
+
: prepared
|
|
72
|
+
.filter((part) => part.type === "text")
|
|
73
|
+
.map((part) => part.text)
|
|
74
|
+
.join("\n"),
|
|
75
|
+
});
|
|
81
76
|
const result = await generateText({
|
|
82
77
|
model: this.host.model,
|
|
83
78
|
...(this.host.temperature === undefined
|
|
@@ -99,7 +94,8 @@ export class Chat {
|
|
|
99
94
|
role: "tool",
|
|
100
95
|
call: {
|
|
101
96
|
name: call.toolName,
|
|
102
|
-
arguments: (call.
|
|
97
|
+
arguments: this.middleware.inputs.get(call.toolCallId) ??
|
|
98
|
+
(call.input ?? {}),
|
|
103
99
|
...this.failureFor(call),
|
|
104
100
|
},
|
|
105
101
|
result: outputOf(step.toolResults, call.toolCallId),
|
|
@@ -109,6 +105,7 @@ export class Chat {
|
|
|
109
105
|
this.messages.push(...result.response.messages);
|
|
110
106
|
}
|
|
111
107
|
async close() {
|
|
108
|
+
await this.middleware.files.close();
|
|
112
109
|
await this.transport.close();
|
|
113
110
|
}
|
|
114
111
|
failureFor(call) {
|
|
@@ -124,21 +121,29 @@ export class Chat {
|
|
|
124
121
|
toolSet() {
|
|
125
122
|
const entries = this.tools.map((definition) => {
|
|
126
123
|
const { $schema: _ignored, ...parameters } = definition.inputSchema;
|
|
124
|
+
const schema = fromJsonSchema(parameters);
|
|
127
125
|
return [
|
|
128
126
|
definition.name,
|
|
129
127
|
dynamicTool({
|
|
130
128
|
description: definition.description ?? "",
|
|
131
|
-
inputSchema: jsonSchema(parameters
|
|
129
|
+
inputSchema: jsonSchema(parameters, {
|
|
130
|
+
validate: async (input) => {
|
|
131
|
+
const result = await schema["~standard"].validate(input);
|
|
132
|
+
return result.issues
|
|
133
|
+
? {
|
|
134
|
+
success: false,
|
|
135
|
+
error: new Error(JSON.stringify(result.issues)),
|
|
136
|
+
}
|
|
137
|
+
: { success: true, value: result.value };
|
|
138
|
+
},
|
|
139
|
+
}),
|
|
132
140
|
execute: async (input, { toolCallId }) => {
|
|
133
|
-
const stubbed = await this.stubs.get(definition.name)?.((input ?? {}));
|
|
134
|
-
if (stubbed !== undefined) {
|
|
135
|
-
return asToolResult(stubbed);
|
|
136
|
-
}
|
|
137
141
|
let result;
|
|
138
142
|
try {
|
|
139
143
|
result = await this.client.callTool({
|
|
140
144
|
name: definition.name,
|
|
141
145
|
arguments: (input ?? {}),
|
|
146
|
+
_meta: { "skybridge/evalToolCallId": toolCallId },
|
|
142
147
|
});
|
|
143
148
|
}
|
|
144
149
|
catch (error) {
|
package/dist/chat.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,MAAM,EACN,6BAA6B,GAE9B,MAAM,8BAA8B,CAAC;AACtC,OAAO,EACL,WAAW,EACX,YAAY,
|
|
1
|
+
{"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,MAAM,EACN,cAAc,EAEd,6BAA6B,GAE9B,MAAM,8BAA8B,CAAC;AACtC,OAAO,EACL,WAAW,EACX,YAAY,EAEZ,UAAU,EAGV,WAAW,GAEZ,MAAM,IAAI,CAAC;AAEZ,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAqBhE,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B,MAAM,cAAc,GAAG,sCAAsC,CAAC;AAE9D,SAAS,QAAQ,CACf,OAAmD,EACnD,UAAkB;IAElB,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,KAAK,UAAU,CAAC,EAAE,MAAM,CAAC;AAC5E,CAAC;AAWD;;;GAGG;AACH,MAAM,OAAO,IAAI;IAqCI;IAjCnB,0DAA0D;IACjD,KAAK,GAAgB,EAAE,CAAC;IAEjC,mEAAmE;IACnE,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,KAAK;aACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC;aACtC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAED,+EAA+E;IAC/E,IAAI,cAAc;QAChB,OAAO,IAAI,CAAC,KAAK;aACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,WAAW,CAAC;aAC3C,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAEgB,QAAQ,GAAmB,EAAE,CAAC;IAC9B,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrC,MAAM,CAAS;IACf,SAAS,CAAgC;IACzC,IAAI,CAAa;IAC1B,KAAK,GAAW,EAAE,CAAC;IAE3B,mEAAmE;IACnE,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;IACzB,CAAC;IAED,YACE,MAAc,EACd,SAAwC,EACxC,IAAgB,EACC,UAA0B;QAA1B,eAAU,GAAV,UAAU,CAAgB;QAE3C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,IAAI,CACf,IAAgB,EAChB,UAA0B;QAE1B,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,IAAI,6BAA6B,CACjD,IAAI,GAAG,CAAC,cAAc,CAAC,EACvB,EAAE,KAAK,EAAE,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,CACtD,CAAC;QACF,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAM,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;QAChE,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;YACnB,sBAAsB,CAAC,IAAI,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;QACtD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YACxB,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,6DAA6D;IAC7D,KAAK,CAAC,IAAI,CAAC,OAAoB;QAC7B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC9D,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;QACxD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;YACd,IAAI,EAAE,MAAM;YACZ,IAAI,EACF,OAAO,QAAQ,KAAK,QAAQ;gBAC1B,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,QAAQ;qBACL,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC;qBACtC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;qBACxB,IAAI,CAAC,IAAI,CAAC;SACpB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;YACtB,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,KAAK,SAAS;gBACrC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YAC3C,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,KAAK,SAAS;gBACtC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE;YACrB,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;SAC/D,CAAC,CAAC;QAEH,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,IAAI,CAAC,IAAI,KAAK,EAAE,EAAE,CAAC;gBACrB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YAC1D,CAAC;YACD,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAClC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;oBACd,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE;wBACJ,IAAI,EAAE,IAAI,CAAC,QAAQ;wBACnB,SAAS,EACP,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC;4BAC1C,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAA6B;wBACjD,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;qBACR;oBAClB,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,UAAU,CAAC;iBACpD,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACpC,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;IAEO,UAAU,CAAC,IAA+C;QAGhE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QAC9B,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC1B,OAAO,EAAE,MAAM,EAAE,2CAA2C,EAAE,CAAC;QACjE,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAEO,OAAO;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;YAC5C,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,UAAU,EAAE,GAAG,UAAU,CAAC,WAAW,CAAC;YACpE,MAAM,MAAM,GAAG,cAAc,CAAC,UAA4B,CAAC,CAAC;YAC5D,OAAO;gBACL,UAAU,CAAC,IAAI;gBACf,WAAW,CAAC;oBACV,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,EAAE;oBACzC,WAAW,EAAE,UAAU,CAAC,UAAyB,EAAE;wBACjD,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;4BACxB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;4BACzD,OAAO,MAAM,CAAC,MAAM;gCAClB,CAAC,CAAC;oCACE,OAAO,EAAE,KAAK;oCACd,KAAK,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;iCAChD;gCACH,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;wBAC7C,CAAC;qBACF,CAAC;oBACF,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE;wBACvC,IAAI,MAA+C,CAAC;wBACpD,IAAI,CAAC;4BACH,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;gCAClC,IAAI,EAAE,UAAU,CAAC,IAAI;gCACrB,SAAS,EAAE,CAAC,KAAK,IAAI,EAAE,CAA4B;gCACnD,KAAK,EAAE,EAAE,0BAA0B,EAAE,UAAU,EAAE;6BAClD,CAAC,CAAC;wBACL,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BACf,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,sCAAsC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC/F,CAAC;4BACF,MAAM,KAAK,CAAC;wBACd,CAAC;wBACD,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;4BAC5B,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,+BAA+B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAChE,CAAC;wBACJ,CAAC;wBACD,OAAO;4BACL,OAAO,EAAE,MAAM,CAAC,OAAO;4BACvB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;yBAC5C,CAAC;oBACJ,CAAC;iBACF,CAAC;aACM,CAAC;QACb,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;CACF"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { StubMap, TransportFetch } from "./chat.js";
|
|
2
|
+
import { InputFiles } from "./input-files.js";
|
|
3
|
+
export declare class EvalMiddleware {
|
|
4
|
+
private readonly next;
|
|
5
|
+
private readonly stubs;
|
|
6
|
+
readonly files: InputFiles;
|
|
7
|
+
readonly inputs: Map<string, Record<string, unknown>>;
|
|
8
|
+
readonly fileParams: Map<string, string[]>;
|
|
9
|
+
private readonly schemas;
|
|
10
|
+
constructor(next: TransportFetch, stubs: StubMap);
|
|
11
|
+
private toolList;
|
|
12
|
+
fetch(url: string | URL, init?: RequestInit): Promise<Response>;
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=eval-middleware.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval-middleware.d.ts","sourceRoot":"","sources":["../src/eval-middleware.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AACzD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAoD9C,qBAAa,cAAc;IAUvB,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,KAAK;IAVxB,QAAQ,CAAC,KAAK,aAAoB;IAClC,QAAQ,CAAC,MAAM,uCAA8C;IAC7D,QAAQ,CAAC,UAAU,wBAA+B;IAClD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAGpB;gBAGe,IAAI,EAAE,cAAc,EACpB,KAAK,EAAE,OAAO;IAGjC,OAAO,CAAC,QAAQ;IAoDV,KAAK,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,EAAE,IAAI,CAAC,EAAE,WAAW;CA0ElD"}
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
import { fromJsonSchema, } from "@modelcontextprotocol/client";
|
|
2
|
+
import { InputFiles } from "./input-files.js";
|
|
3
|
+
function fileInputSchema(schema, allowArray = true) {
|
|
4
|
+
if (!schema || typeof schema === "boolean") {
|
|
5
|
+
throw new Error("expected a FileRef schema");
|
|
6
|
+
}
|
|
7
|
+
if (schema.anyOf) {
|
|
8
|
+
if (schema.anyOf.every((branch) => typeof branch !== "boolean" && branch.type === "null")) {
|
|
9
|
+
throw new Error("expected a FileRef schema");
|
|
10
|
+
}
|
|
11
|
+
return {
|
|
12
|
+
...schema,
|
|
13
|
+
anyOf: schema.anyOf.map((branch) => typeof branch !== "boolean" && branch.type === "null"
|
|
14
|
+
? branch
|
|
15
|
+
: fileInputSchema(branch, allowArray)),
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
if (allowArray && schema.type === "array" && !Array.isArray(schema.items)) {
|
|
19
|
+
return { ...schema, items: fileInputSchema(schema.items, false) };
|
|
20
|
+
}
|
|
21
|
+
const fields = ["download_url", "file_id", "mime_type", "file_name"];
|
|
22
|
+
if (schema.type !== "object" ||
|
|
23
|
+
(schema.additionalProperties !== undefined &&
|
|
24
|
+
schema.additionalProperties !== false) ||
|
|
25
|
+
schema.required?.length !== 2 ||
|
|
26
|
+
!["download_url", "file_id"].every((field) => schema.required?.includes(field)) ||
|
|
27
|
+
Object.keys(schema.properties ?? {}).length !== fields.length ||
|
|
28
|
+
!fields.every((field) => {
|
|
29
|
+
const property = schema.properties?.[field];
|
|
30
|
+
return typeof property === "object" && property.type === "string";
|
|
31
|
+
})) {
|
|
32
|
+
throw new Error("expected a FileRef schema");
|
|
33
|
+
}
|
|
34
|
+
return {
|
|
35
|
+
type: "string",
|
|
36
|
+
description: "This parameter expects an absolute local file path.",
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
export class EvalMiddleware {
|
|
40
|
+
next;
|
|
41
|
+
stubs;
|
|
42
|
+
files = new InputFiles();
|
|
43
|
+
inputs = new Map();
|
|
44
|
+
fileParams = new Map();
|
|
45
|
+
schemas = new Map();
|
|
46
|
+
constructor(next, stubs) {
|
|
47
|
+
this.next = next;
|
|
48
|
+
this.stubs = stubs;
|
|
49
|
+
}
|
|
50
|
+
toolList(message) {
|
|
51
|
+
if (!("result" in message) || !Array.isArray(message.result.tools)) {
|
|
52
|
+
return message;
|
|
53
|
+
}
|
|
54
|
+
const tools = message.result.tools;
|
|
55
|
+
const unknownStubs = Object.keys(this.stubs).filter((name) => !tools.some((tool) => tool.name === name));
|
|
56
|
+
if (unknownStubs.length > 0) {
|
|
57
|
+
throw new Error(`the app exposes no tool named ${unknownStubs.map((name) => `"${name}"`).join(", ")}, so the stub would never run`);
|
|
58
|
+
}
|
|
59
|
+
return {
|
|
60
|
+
...message,
|
|
61
|
+
result: {
|
|
62
|
+
...message.result,
|
|
63
|
+
tools: tools.map((tool) => {
|
|
64
|
+
this.schemas.set(tool.name, fromJsonSchema(tool.inputSchema));
|
|
65
|
+
const fields = tool._meta?.["openai/fileParams"] ?? [];
|
|
66
|
+
if (!Array.isArray(fields) ||
|
|
67
|
+
!fields.every((field) => typeof field === "string")) {
|
|
68
|
+
throw new Error(`Invalid openai/fileParams for "${tool.name}": expected parameter names`);
|
|
69
|
+
}
|
|
70
|
+
this.fileParams.set(tool.name, fields);
|
|
71
|
+
const properties = { ...tool.inputSchema.properties };
|
|
72
|
+
for (const field of fields) {
|
|
73
|
+
try {
|
|
74
|
+
properties[field] = fileInputSchema(properties[field]);
|
|
75
|
+
}
|
|
76
|
+
catch {
|
|
77
|
+
throw new Error(`Invalid openai/fileParams for "${tool.name}.${field}": expected a FileRef schema (or an optional, nullable, or array form)`);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
return { ...tool, inputSchema: { ...tool.inputSchema, properties } };
|
|
81
|
+
}),
|
|
82
|
+
},
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
async fetch(url, init) {
|
|
86
|
+
if (typeof init?.body !== "string") {
|
|
87
|
+
return this.next(url, init);
|
|
88
|
+
}
|
|
89
|
+
const message = JSON.parse(init.body);
|
|
90
|
+
if ("method" in message && message.method === "tools/call") {
|
|
91
|
+
const params = message.params;
|
|
92
|
+
const args = this.files.rewrite(params.arguments ?? {}, this.fileParams.get(params.name) ?? []);
|
|
93
|
+
const callId = params._meta?.["skybridge/evalToolCallId"];
|
|
94
|
+
if (typeof callId === "string") {
|
|
95
|
+
this.inputs.set(callId, args);
|
|
96
|
+
}
|
|
97
|
+
const validation = await this.schemas
|
|
98
|
+
.get(params.name)?.["~standard"].validate(args);
|
|
99
|
+
if (validation?.issues) {
|
|
100
|
+
throw new Error(`Invalid tool arguments for "${params.name}": ${JSON.stringify(validation.issues)}`);
|
|
101
|
+
}
|
|
102
|
+
const stubbed = await this.stubs[params.name]?.(args);
|
|
103
|
+
if (stubbed !== undefined) {
|
|
104
|
+
return Response.json({
|
|
105
|
+
jsonrpc: "2.0",
|
|
106
|
+
id: "id" in message ? message.id : null,
|
|
107
|
+
result: {
|
|
108
|
+
content: [
|
|
109
|
+
{
|
|
110
|
+
type: "text",
|
|
111
|
+
text: typeof stubbed === "string"
|
|
112
|
+
? stubbed
|
|
113
|
+
: JSON.stringify(stubbed),
|
|
114
|
+
},
|
|
115
|
+
],
|
|
116
|
+
},
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
init = {
|
|
120
|
+
...init,
|
|
121
|
+
body: JSON.stringify({
|
|
122
|
+
...message,
|
|
123
|
+
params: { ...params, arguments: args },
|
|
124
|
+
}),
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
const response = await this.next(url, init);
|
|
128
|
+
if (!("method" in message) || message.method !== "tools/list") {
|
|
129
|
+
return response;
|
|
130
|
+
}
|
|
131
|
+
const text = await response.text();
|
|
132
|
+
const body = response.headers
|
|
133
|
+
.get("content-type")
|
|
134
|
+
?.includes("text/event-stream")
|
|
135
|
+
? text
|
|
136
|
+
.split("\n")
|
|
137
|
+
.map((line) => line.startsWith("data: ")
|
|
138
|
+
? `data: ${JSON.stringify(this.toolList(JSON.parse(line.slice(6))))}`
|
|
139
|
+
: line)
|
|
140
|
+
.join("\n")
|
|
141
|
+
: JSON.stringify(this.toolList(JSON.parse(text)));
|
|
142
|
+
const headers = new Headers(response.headers);
|
|
143
|
+
headers.delete("content-length");
|
|
144
|
+
return new Response(body, { status: response.status, headers });
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
//# sourceMappingURL=eval-middleware.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval-middleware.js","sourceRoot":"","sources":["../src/eval-middleware.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,cAAc,GAIf,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,SAAS,eAAe,CACtB,MAAyC,EACzC,UAAU,GAAG,IAAI;IAEjB,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,SAAS,EAAE,CAAC;QAC3C,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IACD,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,IACE,MAAM,CAAC,KAAK,CAAC,KAAK,CAChB,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,CAClE,EACD,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/C,CAAC;QACD,OAAO;YACL,GAAG,MAAM;YACT,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CACjC,OAAO,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM;gBACnD,CAAC,CAAC,MAAM;gBACR,CAAC,CAAC,eAAe,CAAC,MAAM,EAAE,UAAU,CAAC,CACxC;SACF,CAAC;IACJ,CAAC;IACD,IAAI,UAAU,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1E,OAAO,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,EAAE,CAAC;IACpE,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,cAAc,EAAE,SAAS,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC;IACrE,IACE,MAAM,CAAC,IAAI,KAAK,QAAQ;QACxB,CAAC,MAAM,CAAC,oBAAoB,KAAK,SAAS;YACxC,MAAM,CAAC,oBAAoB,KAAK,KAAK,CAAC;QACxC,MAAM,CAAC,QAAQ,EAAE,MAAM,KAAK,CAAC;QAC7B,CAAC,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAC3C,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,CACjC;QACD,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM;QAC7D,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACtB,MAAM,QAAQ,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC;YAC5C,OAAO,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC;QACpE,CAAC,CAAC,EACF,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,WAAW,EAAE,qDAAqD;KACnE,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,cAAc;IAUN;IACA;IAVV,KAAK,GAAG,IAAI,UAAU,EAAE,CAAC;IACzB,MAAM,GAAG,IAAI,GAAG,EAAmC,CAAC;IACpD,UAAU,GAAG,IAAI,GAAG,EAAoB,CAAC;IACjC,OAAO,GAAG,IAAI,GAAG,EAG/B,CAAC;IAEJ,YACmB,IAAoB,EACpB,KAAc;QADd,SAAI,GAAJ,IAAI,CAAgB;QACpB,UAAK,GAAL,KAAK,CAAS;IAC9B,CAAC;IAEI,QAAQ,CAAC,OAAuB;QACtC,IAAI,CAAC,CAAC,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YACnE,OAAO,OAAO,CAAC;QACjB,CAAC;QACD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAe,CAAC;QAC7C,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CACjD,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,CACpD,CAAC;QACF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CACb,iCAAiC,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,+BAA+B,CACnH,CAAC;QACJ,CAAC;QACD,OAAO;YACL,GAAG,OAAO;YACV,MAAM,EAAE;gBACN,GAAG,OAAO,CAAC,MAAM;gBACjB,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;oBACxB,IAAI,CAAC,OAAO,CAAC,GAAG,CACd,IAAI,CAAC,IAAI,EACT,cAAc,CAAC,IAAI,CAAC,WAA6B,CAAC,CACnD,CAAC;oBACF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC;oBACvD,IACE,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;wBACtB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,EACnD,CAAC;wBACD,MAAM,IAAI,KAAK,CACb,kCAAkC,IAAI,CAAC,IAAI,6BAA6B,CACzE,CAAC;oBACJ,CAAC;oBACD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;oBACvC,MAAM,UAAU,GAAG,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,EAElD,CAAC;oBACF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;wBAC3B,IAAI,CAAC;4BACH,UAAU,CAAC,KAAK,CAAC,GAAG,eAAe,CACjC,UAAU,CAAC,KAAK,CAAsC,CACvD,CAAC;wBACJ,CAAC;wBAAC,MAAM,CAAC;4BACP,MAAM,IAAI,KAAK,CACb,kCAAkC,IAAI,CAAC,IAAI,IAAI,KAAK,wEAAwE,CAC7H,CAAC;wBACJ,CAAC;oBACH,CAAC;oBACD,OAAO,EAAE,GAAG,IAAI,EAAE,WAAW,EAAE,EAAE,GAAG,IAAI,CAAC,WAAW,EAAE,UAAU,EAAE,EAAE,CAAC;gBACvE,CAAC,CAAC;aACH;SACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,GAAiB,EAAE,IAAkB;QAC/C,IAAI,OAAO,IAAI,EAAE,IAAI,KAAK,QAAQ,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC9B,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAmB,CAAC;QACxD,IAAI,QAAQ,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;YAC3D,MAAM,MAAM,GAAG,OAAO,CAAC,MAItB,CAAC;YACF,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAC7B,MAAM,CAAC,SAAS,IAAI,EAAE,EACtB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CACvC,CAAC;YACF,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC,0BAA0B,CAAC,CAAC;YAC1D,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAC/B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YAChC,CAAC;YACD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,OAAO;iBAClC,GAAG,CAAC,MAAM,CAAC,IAAI,CAChB,EAAE,CAAC,WAAW,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YACjC,IAAI,UAAU,EAAE,MAAM,EAAE,CAAC;gBACvB,MAAM,IAAI,KAAK,CACb,+BAA+B,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CACpF,CAAC;YACJ,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;YACtD,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,OAAO,QAAQ,CAAC,IAAI,CAAC;oBACnB,OAAO,EAAE,KAAK;oBACd,EAAE,EAAE,IAAI,IAAI,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI;oBACvC,MAAM,EAAE;wBACN,OAAO,EAAE;4BACP;gCACE,IAAI,EAAE,MAAM;gCACZ,IAAI,EACF,OAAO,OAAO,KAAK,QAAQ;oCACzB,CAAC,CAAC,OAAO;oCACT,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;6BAC9B;yBACF;qBACF;iBACF,CAAC,CAAC;YACL,CAAC;YACD,IAAI,GAAG;gBACL,GAAG,IAAI;gBACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,GAAG,OAAO;oBACV,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE;iBACvC,CAAC;aACH,CAAC;QACJ,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC5C,IAAI,CAAC,CAAC,QAAQ,IAAI,OAAO,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;YAC9D,OAAO,QAAQ,CAAC;QAClB,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO;aAC1B,GAAG,CAAC,cAAc,CAAC;YACpB,EAAE,QAAQ,CAAC,mBAAmB,CAAC;YAC/B,CAAC,CAAC,IAAI;iBACD,KAAK,CAAC,IAAI,CAAC;iBACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACZ,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;gBACvB,CAAC,CAAC,SAAS,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE;gBACrE,CAAC,CAAC,IAAI,CACT;iBACA,IAAI,CAAC,IAAI,CAAC;YACf,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAC9C,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;QACjC,OAAO,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;IAClE,CAAC;CACF"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { UserContent } from "ai";
|
|
2
|
+
export declare class InputFiles {
|
|
3
|
+
private readonly files;
|
|
4
|
+
private readonly paths;
|
|
5
|
+
private readonly server;
|
|
6
|
+
prepare(content: UserContent): Promise<UserContent>;
|
|
7
|
+
rewrite(input: Record<string, unknown>, fields: string[]): {
|
|
8
|
+
[x: string]: unknown;
|
|
9
|
+
};
|
|
10
|
+
close(): Promise<void>;
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=input-files.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"input-files.d.ts","sourceRoot":"","sources":["../src/input-files.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAY,WAAW,EAAE,MAAM,IAAI,CAAC;AAkChD,qBAAa,UAAU;IACrB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAGlB;IACJ,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA8B;IACpD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAQpB;IAEG,OAAO,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAwCzD,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE;;;IAuBlD,KAAK;CAMZ"}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { once } from "node:events";
|
|
3
|
+
import { createServer } from "node:http";
|
|
4
|
+
import { registerInputFile } from "./matchers/matchers.js";
|
|
5
|
+
async function fileBytes(data) {
|
|
6
|
+
if (data instanceof URL) {
|
|
7
|
+
const response = await fetch(data);
|
|
8
|
+
if (!response.ok) {
|
|
9
|
+
throw new Error(`Input file download failed (HTTP ${response.status})`);
|
|
10
|
+
}
|
|
11
|
+
return Buffer.from(await response.arrayBuffer());
|
|
12
|
+
}
|
|
13
|
+
if (typeof data === "string") {
|
|
14
|
+
return data.startsWith("data:")
|
|
15
|
+
? fileBytes(new URL(data))
|
|
16
|
+
: Buffer.from(data, "base64");
|
|
17
|
+
}
|
|
18
|
+
if (data instanceof Uint8Array || data instanceof ArrayBuffer) {
|
|
19
|
+
return Buffer.from(new Uint8Array(data));
|
|
20
|
+
}
|
|
21
|
+
if (data.type === "data") {
|
|
22
|
+
return fileBytes(data.data);
|
|
23
|
+
}
|
|
24
|
+
if (data.type === "url") {
|
|
25
|
+
return fileBytes(new URL(data.url));
|
|
26
|
+
}
|
|
27
|
+
if (data.type === "text") {
|
|
28
|
+
return Buffer.from(data.text);
|
|
29
|
+
}
|
|
30
|
+
throw new Error("Input files need bytes, text, or a URL; provider-only file references cannot be downloaded by the eval app");
|
|
31
|
+
}
|
|
32
|
+
export class InputFiles {
|
|
33
|
+
files = new Map();
|
|
34
|
+
paths = new Map();
|
|
35
|
+
server = createServer((request, response) => {
|
|
36
|
+
const file = this.files.get(request.url?.slice(1) ?? "");
|
|
37
|
+
if (!file) {
|
|
38
|
+
response.writeHead(404).end();
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
41
|
+
response.writeHead(200, { "Content-Type": file.reference.mime_type });
|
|
42
|
+
response.end(file.bytes);
|
|
43
|
+
});
|
|
44
|
+
async prepare(content) {
|
|
45
|
+
if (typeof content === "string") {
|
|
46
|
+
return content;
|
|
47
|
+
}
|
|
48
|
+
const prepared = [];
|
|
49
|
+
for (const part of content) {
|
|
50
|
+
if (part.type !== "file") {
|
|
51
|
+
prepared.push(part);
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
const bytes = await fileBytes(part.data);
|
|
55
|
+
if (!this.server.listening) {
|
|
56
|
+
this.server.listen(0, "127.0.0.1").unref();
|
|
57
|
+
await once(this.server, "listening");
|
|
58
|
+
}
|
|
59
|
+
const { port } = this.server.address();
|
|
60
|
+
const id = randomUUID();
|
|
61
|
+
const file_id = `file_${id}`;
|
|
62
|
+
const reference = {
|
|
63
|
+
file_id,
|
|
64
|
+
download_url: `http://127.0.0.1:${port}/${file_id}`,
|
|
65
|
+
mime_type: part.mediaType,
|
|
66
|
+
...(part.filename === undefined ? {} : { file_name: part.filename }),
|
|
67
|
+
};
|
|
68
|
+
const uploaded_name = part.filename ?? "input";
|
|
69
|
+
const workspace_path = `/workspace/scratch/${id}/upload/${uploaded_name}`;
|
|
70
|
+
this.files.set(file_id, { reference, bytes });
|
|
71
|
+
this.paths.set(workspace_path, reference);
|
|
72
|
+
registerInputFile(reference, part);
|
|
73
|
+
prepared.push({
|
|
74
|
+
type: "text",
|
|
75
|
+
text: `Attached file: ${JSON.stringify({ uploaded_name, workspace_path })}`,
|
|
76
|
+
}, part);
|
|
77
|
+
}
|
|
78
|
+
return prepared;
|
|
79
|
+
}
|
|
80
|
+
rewrite(input, fields) {
|
|
81
|
+
const rewritten = { ...input };
|
|
82
|
+
const resolve = (value) => {
|
|
83
|
+
if (typeof value !== "string") {
|
|
84
|
+
return value;
|
|
85
|
+
}
|
|
86
|
+
const reference = this.paths.get(value);
|
|
87
|
+
if (!reference) {
|
|
88
|
+
throw new Error(`Unknown input file path: ${value}`);
|
|
89
|
+
}
|
|
90
|
+
return reference;
|
|
91
|
+
};
|
|
92
|
+
for (const field of fields) {
|
|
93
|
+
if (field in rewritten) {
|
|
94
|
+
const value = rewritten[field];
|
|
95
|
+
rewritten[field] = Array.isArray(value)
|
|
96
|
+
? value.map(resolve)
|
|
97
|
+
: resolve(value);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return rewritten;
|
|
101
|
+
}
|
|
102
|
+
async close() {
|
|
103
|
+
if (this.server.listening) {
|
|
104
|
+
this.server.close();
|
|
105
|
+
await once(this.server, "close");
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
//# sourceMappingURL=input-files.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"input-files.js","sourceRoot":"","sources":["../src/input-files.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AACnC,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAIzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAE3D,KAAK,UAAU,SAAS,CAAC,IAAsB;IAC7C,IAAI,IAAI,YAAY,GAAG,EAAE,CAAC;QACxB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,oCAAoC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;QAC1E,CAAC;QACD,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;IACnD,CAAC;IACD,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;YAC7B,CAAC,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;YAC1B,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;IACD,IAAI,IAAI,YAAY,UAAU,IAAI,IAAI,YAAY,WAAW,EAAE,CAAC;QAC9D,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3C,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QACzB,OAAO,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;QACxB,OAAO,SAAS,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QACzB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IACD,MAAM,IAAI,KAAK,CACb,4GAA4G,CAC7G,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,UAAU;IACJ,KAAK,GAAG,IAAI,GAAG,EAG7B,CAAC;IACa,KAAK,GAAG,IAAI,GAAG,EAAmB,CAAC;IACnC,MAAM,GAAG,YAAY,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE;QAC3D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACzD,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;YAC9B,OAAO;QACT,CAAC;QACD,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC,CAAC;QACtE,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,KAAK,CAAC,OAAO,CAAC,OAAoB;QAChC,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YAChC,OAAO,OAAO,CAAC;QACjB,CAAC;QACD,MAAM,QAAQ,GAAiC,EAAE,CAAC;QAClD,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YAC3B,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACzB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACpB,SAAS;YACX,CAAC;YACD,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;gBAC3B,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,KAAK,EAAE,CAAC;gBAC3C,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;YACvC,CAAC;YACD,MAAM,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,EAAiB,CAAC;YACtD,MAAM,EAAE,GAAG,UAAU,EAAE,CAAC;YACxB,MAAM,OAAO,GAAG,QAAQ,EAAE,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG;gBAChB,OAAO;gBACP,YAAY,EAAE,oBAAoB,IAAI,IAAI,OAAO,EAAE;gBACnD,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,GAAG,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;aACrE,CAAC;YACF,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC;YAC/C,MAAM,cAAc,GAAG,sBAAsB,EAAE,WAAW,aAAa,EAAE,CAAC;YAC1E,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;YAC9C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;YAC1C,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;YACnC,QAAQ,CAAC,IAAI,CACX;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,kBAAkB,IAAI,CAAC,SAAS,CAAC,EAAE,aAAa,EAAE,cAAc,EAAE,CAAC,EAAE;aAC5E,EACD,IAAI,CACL,CAAC;QACJ,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,OAAO,CAAC,KAA8B,EAAE,MAAgB;QACtD,MAAM,SAAS,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;QAC/B,MAAM,OAAO,GAAG,CAAC,KAAc,EAAW,EAAE;YAC1C,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,OAAO,KAAK,CAAC;YACf,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACxC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,4BAA4B,KAAK,EAAE,CAAC,CAAC;YACvD,CAAC;YACD,OAAO,SAAS,CAAC;QACnB,CAAC,CAAC;QACF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,KAAK,IAAI,SAAS,EAAE,CAAC;gBACvB,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;gBAC/B,SAAS,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;oBACrC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC;oBACpB,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YACpB,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;CACF"}
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import "vitest";
|
|
2
|
-
import type { LanguageModel } from "ai";
|
|
3
|
-
import type { ToolInput, ToolNames } from "skybridge/server";
|
|
2
|
+
import type { FilePart, LanguageModel } from "ai";
|
|
3
|
+
import type { FileRef, ToolInput, ToolNames } from "skybridge/server";
|
|
4
4
|
import type { ChatLike } from "./types.js";
|
|
5
|
+
export declare function registerInputFile(reference: FileRef, part: FilePart): void;
|
|
6
|
+
export declare function registerFileParameters(chat: object, fields: ReadonlyMap<string, readonly string[]>): void;
|
|
5
7
|
/** What a judge decided about a conversation. */
|
|
6
8
|
export interface Verdict {
|
|
7
9
|
pass: boolean;
|
|
@@ -45,6 +47,7 @@ export interface ChatAssertion<App> extends ChatMatchers<App> {
|
|
|
45
47
|
}
|
|
46
48
|
declare module "vitest" {
|
|
47
49
|
interface ExpectStatic {
|
|
50
|
+
inputFile(part: FilePart): FileRef;
|
|
48
51
|
chat<App>(chat: ChatLike<App>): ChatAssertion<App>;
|
|
49
52
|
}
|
|
50
53
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"matchers.d.ts","sourceRoot":"","sources":["../../src/matchers/matchers.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,CAAC;AAChB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"matchers.d.ts","sourceRoot":"","sources":["../../src/matchers/matchers.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,CAAC;AAChB,OAAO,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAClD,OAAO,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAEtE,OAAO,KAAK,EAAE,QAAQ,EAAkB,MAAM,YAAY,CAAC;AAS3D,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,QAEnE;AAED,wBAAgB,sBAAsB,CACpC,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,QAG/C;AAsBD,iDAAiD;AACjD,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,OAAO,CAAC;IACd,iFAAiF;IACjF,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,kEAAkE;AAClE,MAAM,WAAW,QAAQ;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,2EAA2E;IAC3E,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,uDAAuD;AACvD,MAAM,WAAW,eAAe;IAC9B,4EAA4E;IAC5E,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB;;;;OAIG;IACH,KAAK,CAAC,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAC5D;AAED;;;;GAIG;AACH,MAAM,WAAW,YAAY,CAAC,GAAG;IAC/B,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAC9C,IAAI,EAAE,IAAI,EACV,IAAI,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GACnC,IAAI,CAAC;IACR,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAC9C,IAAI,EAAE,IAAI,EACV,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GAClC,IAAI,CAAC;IACR,qBAAqB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,CAAC;IACrE,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,CAAC;IACpE,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IACxC,wBAAwB,CACtB,GAAG,KAAK,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,GAAG,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,GAC9C,IAAI,CAAC;IACR,mBAAmB,IAAI,IAAI,CAAC;IAC5B,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5E;AAED,MAAM,WAAW,aAAa,CAAC,GAAG,CAAE,SAAQ,YAAY,CAAC,GAAG,CAAC;IAC3D,GAAG,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC;CACxB;AAED,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,YAAY;QACpB,SAAS,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC;QACnC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;KACpD;CACF"}
|
|
@@ -1,10 +1,42 @@
|
|
|
1
1
|
import "vitest";
|
|
2
2
|
import { expect } from "vitest";
|
|
3
|
+
const owners = new WeakMap();
|
|
4
|
+
const fileMatchers = new WeakSet();
|
|
5
|
+
const fileParameters = new WeakMap();
|
|
6
|
+
export function registerInputFile(reference, part) {
|
|
7
|
+
owners.set(reference, part);
|
|
8
|
+
}
|
|
9
|
+
export function registerFileParameters(chat, fields) {
|
|
10
|
+
fileParameters.set(chat, fields);
|
|
11
|
+
}
|
|
12
|
+
function inputFile(part) {
|
|
13
|
+
const label = () => `inputFile(${part.filename ?? part.mediaType})`;
|
|
14
|
+
const matcher = {
|
|
15
|
+
asymmetricMatch: (value) => typeof value === "object" && value !== null && owners.get(value) === part,
|
|
16
|
+
toString: label,
|
|
17
|
+
toJSON: label,
|
|
18
|
+
};
|
|
19
|
+
fileMatchers.add(matcher);
|
|
20
|
+
return matcher;
|
|
21
|
+
}
|
|
22
|
+
function containsInputFile(value) {
|
|
23
|
+
return (typeof value === "object" &&
|
|
24
|
+
value !== null &&
|
|
25
|
+
(fileMatchers.has(value) || Object.values(value).some(containsInputFile)));
|
|
26
|
+
}
|
|
3
27
|
function calls(chat) {
|
|
4
28
|
return chat.turns
|
|
5
29
|
.filter((turn) => turn.role === "tool")
|
|
6
30
|
.map((turn) => turn.call);
|
|
7
31
|
}
|
|
32
|
+
function assertFileParameters(chat, name, args = {}) {
|
|
33
|
+
const fields = fileParameters.get(chat)?.get(name) ?? [];
|
|
34
|
+
for (const [field, value] of Object.entries(args)) {
|
|
35
|
+
if (containsInputFile(value) && !fields.includes(field)) {
|
|
36
|
+
throw new Error(`expect.inputFile cannot be used for "${name}.${field}": "${field}" is not listed in openai/fileParams`);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
}
|
|
8
40
|
function spokenTurns(chat) {
|
|
9
41
|
return chat.turns
|
|
10
42
|
.filter((turn) => turn.role === "assistant")
|
|
@@ -103,6 +135,7 @@ async function modelVerdict(chat, criteria, model) {
|
|
|
103
135
|
}
|
|
104
136
|
expect.extend({
|
|
105
137
|
toHaveCalledToolOnce(received, name, args) {
|
|
138
|
+
assertFileParameters(received, name, args);
|
|
106
139
|
const matching = acceptedCalls(received, name);
|
|
107
140
|
const argsMatch = args === undefined ||
|
|
108
141
|
(matching.length === 1 &&
|
|
@@ -118,6 +151,7 @@ expect.extend({
|
|
|
118
151
|
};
|
|
119
152
|
},
|
|
120
153
|
toHaveCalledToolWith(received, name, args) {
|
|
154
|
+
assertFileParameters(received, name, args);
|
|
121
155
|
const pass = acceptedCalls(received, name).some((call) => this.equals(call.arguments, expect.objectContaining(args)));
|
|
122
156
|
return {
|
|
123
157
|
pass,
|
|
@@ -214,4 +248,5 @@ judge: ${verdict.pass ? "PASS" : "FAIL"}${verdict.reasoning === undefined ? "" :
|
|
|
214
248
|
},
|
|
215
249
|
});
|
|
216
250
|
expect.chat = (chat) => expect(chat);
|
|
251
|
+
expect.inputFile = inputFile;
|
|
217
252
|
//# sourceMappingURL=matchers.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"matchers.js","sourceRoot":"","sources":["../../src/matchers/matchers.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,CAAC;AAGhB,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AA+DhC,SAAS,KAAK,CAAC,IAAuB;IACpC,OAAO,IAAI,CAAC,KAAK;SACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC;SACtC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC;AAED,SAAS,WAAW,CAAC,IAAuB;IAC1C,OAAO,IAAI,CAAC,KAAK;SACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,WAAW,CAAC;SAC3C,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC;AAED,SAAS,cAAc,CACrB,IAAuB,EACvB,IAAY;IAEZ,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,WAAW,CAAC,IAAuB;IAC1C,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;AACjE,CAAC;AAED,SAAS,aAAa,CACpB,IAAuB,EACvB,IAAY;IAEZ,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;AACxE,CAAC;AAED,SAAS,QAAQ,CAAC,IAAuB;IACvC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IACzB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,wBAAwB,CAAC;IAClC,CAAC;IACD,OAAO,IAAI;SACR,GAAG,CACF,CAAC,IAAuB,EAAE,KAAa,EAAE,EAAE,CACzC,KAAK,KAAK,GAAG,CAAC,KAAK,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,cAAc,IAAI,CAAC,MAAM,GAAG,EAAE,CACnI;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AAC1C,CAAC;AAED,SAAS,IAAI,CAAC,IAAuB,EAAE,IAAqB;IAC1D,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CACrC,OAAO,IAAI,KAAK,QAAQ;QACtB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;QACrE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CACpB,CAAC;AACJ,CAAC;AAED,SAAS,MAAM,CAAC,IAAuB;IACrC,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,gCAAgC,CAAC;IAC1C,CAAC;IACD,OAAO,KAAK;SACT,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,GAAG,CAAC,KAAK,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;SACzD,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,SAAS,KAAK,CAAC,IAAY;IACzB,OAAO,IAAI,CAAC,OAAO,CAAC,uBAAuB,EAAE,QAAQ,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,YAAY,GAAG;;;6KAGwJ,CAAC;AAE9K,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAE9B,SAAS,QAAQ,CAAC,IAAmB;IACnC,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QACzB,OAAO,GAAG,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;IAC7C,CAAC;IACD,MAAM,OAAO,GACX,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS;QAC5B,CAAC,CAAC,OAAO,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,EAAE;QACxD,CAAC,CAAC,aAAa,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;IACvC,OAAO,SAAS,KAAK,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,OAAO,EAAE,CAAC,EAAE,CAAC;AACxG,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,IAAI,CAAC,MAAM,GAAG,gBAAgB;QACnC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC,eAAe;QACnD,CAAC,CAAC,IAAI,CAAC;AACX,CAAC;AAED,SAAS,aAAa,CAAC,IAAuB;IAC5C,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,6BAA6B,CAAC;IACvC,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC7C,CAAC;AAED,SAAS,MAAM,CAAC,IAAuB,EAAE,OAAe;IACtD,OAAO,GAAG,OAAO;;;EAGjB,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;AACnB,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,IAAuB,EACvB,QAAgB,EAChB,KAAqB;IAErB,MAAM,EAAE,cAAc,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;IAC1D,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,cAAc,CAAC;QACtC,KAAK,EAAE,KAAK,IAAI,IAAI,CAAC,KAAK;QAC1B,MAAM,EAAE,YAAY;QACpB,MAAM,EAAE,cAAc,QAAQ,uBAAuB,aAAa,CAAC,IAAI,CAAC,mBAAmB;QAC3F,MAAM,EAAE,UAAU,CAAkC;YAClD,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;gBACzB,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;aAC9B;YACD,QAAQ,EAAE,CAAC,MAAM,EAAE,WAAW,CAAC;YAC/B,oBAAoB,EAAE,KAAK;SAC5B,CAAC;KACH,CAAC,CAAC;IACH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,MAAM,CAAC;IACZ,oBAAoB,CAClB,QAA2B,EAC3B,IAAY,EACZ,IAA8B;QAE9B,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC/C,MAAM,SAAS,GACb,IAAI,KAAK,SAAS;YAClB,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACpB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxE,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC;QAEhD,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,aAAa,IAAI,kCAAkC;gBACrD,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;oBACrB,CAAC,CAAC,aAAa,IAAI,4BAA4B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,4BAA4B,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG;oBACxI,CAAC,CAAC,aAAa,IAAI,6DAA6D,QAAQ,CAAC,MAAM,wBAAwB,CAC5H;SACJ,CAAC;IACJ,CAAC;IAED,oBAAoB,CAClB,QAA2B,EAC3B,IAAY,EACZ,IAA6B;QAE7B,MAAM,IAAI,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CACvD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAC3D,CAAC;QAEF,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,wBAAwB,IAAI,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG;gBAC/D,CAAC,CAAC,aAAa,IAAI,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CACpE;SACJ,CAAC;IACJ,CAAC;IAED,oBAAoB,CAAC,QAA2B,EAAE,IAAY;QAC5D,MAAM,SAAS,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;QAErE,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC;YACvB,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,MAAM,CAAC,MAAM,GAAG,CAAC;gBACf,CAAC,CAAC,2BAA2B,IAAI,eAAe;gBAChD,CAAC,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;oBACtB,CAAC,CAAC,aAAa,IAAI,mDAAmD;oBACtE,CAAC,CAAC,aAAa,IAAI,sBAAsB,SAAS,CAAC,MAAM,mBAAmB,CACjF;SACJ,CAAC;IACJ,CAAC;IAED,UAAU,CAAC,QAA2B,EAAE,IAAqB;QAC3D,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAClC,MAAM,MAAM,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAErE,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,GACE,IAAI;gBACF,CAAC,CAAC,qCAAqC,MAAM,GAAG;gBAChD,CAAC,CAAC,iCAAiC,MAAM,GAC7C;;;EAGN,MAAM,CAAC,QAAQ,CAAC,EAAE;SACf,CAAC;IACJ,CAAC;IAED,mBAAmB,CAAC,QAA2B;QAC7C,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;QAErC,OAAO;YACL,IAAI,EAAE,KAAK,KAAK,CAAC;YACjB,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,KAAK,KAAK,CAAC;gBACT,CAAC,CAAC,+CAA+C;gBACjD,CAAC,CAAC,wDAAwD,KAAK,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,GAAG,CACvG;SACJ,CAAC;IACJ,CAAC;IAED,wBAAwB,CAAC,QAA2B,EAAE,GAAG,KAAe;QACtE,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;YACzC,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;gBACzC,OAAO,IAAI,CAAC,CAAC;YACf,CAAC;YACD,IAAI,OAAO,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;gBAC7B,MAAM;YACR,CAAC;QACH,CAAC;QACD,MAAM,IAAI,GAAG,OAAO,KAAK,KAAK,CAAC,MAAM,CAAC;QACtC,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAE/D,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,YAAY,MAAM,kCAAkC;gBACtD,CAAC,CAAC,YAAY,MAAM,wBAAwB,KAAK,CAAC,OAAO,CAAC,eAAe,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CACpI;SACJ,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,cAAc,CAClB,QAA2B,EAC3B,QAAgB,EAChB,OAAyB;QAEzB,IAAI,OAAgB,CAAC;QACrB,IAAI,CAAC;YACH,OAAO,GAAG,OAAO,EAAE,KAAK;gBACtB,CAAC,CAAC,MAAM,OAAO,CAAC,KAAK,CAAC;oBAClB,QAAQ;oBACR,UAAU,EAAE,aAAa,CAAC,QAAQ,CAAC;iBACpC,CAAC;gBACJ,CAAC,CAAC,MAAM,YAAY,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;YAE3D,IAAI,OAAO,OAAO,EAAE,IAAI,KAAK,SAAS,EAAE,CAAC;gBACvC,MAAM,IAAI,KAAK,CACb,sBAAsB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,6CAA6C,CAC3F,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,sBAAsB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC/E,CAAC;QACJ,CAAC;QAED,OAAO;YACL,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,OAAO,EAAE,GAAG,EAAE,CACZ,6BAA6B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;KAC1D,QAAQ;;SAEJ,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,OAAO,CAAC,SAAS,EAAE,EAAE;SACxG,CAAC;IACJ,CAAC;IAED,qBAAqB,CAAC,QAA2B,EAAE,IAAY;QAC7D,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAEhD,OAAO;YACL,IAAI,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC;YAC3B,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACnB,CAAC,CAAC,aAAa,IAAI,+BAA+B;gBAClD,CAAC,CAAC,aAAa,IAAI,2CAA2C,QAAQ,CAAC,MAAM,SAAS,CACzF;SACJ,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,IAAI,GAAG,CAAM,IAAmB,EAAsB,EAAE,CAC7D,MAAM,CAAC,IAAI,CAAkC,CAAC"}
|
|
1
|
+
{"version":3,"file":"matchers.js","sourceRoot":"","sources":["../../src/matchers/matchers.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,CAAC;AAGhB,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,MAAM,GAAG,IAAI,OAAO,EAAoB,CAAC;AAC/C,MAAM,YAAY,GAAG,IAAI,OAAO,EAAU,CAAC;AAC3C,MAAM,cAAc,GAAG,IAAI,OAAO,EAG/B,CAAC;AAEJ,MAAM,UAAU,iBAAiB,CAAC,SAAkB,EAAE,IAAc;IAClE,MAAM,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,IAAY,EACZ,MAA8C;IAE9C,cAAc,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;AACnC,CAAC;AAED,SAAS,SAAS,CAAC,IAAc;IAC/B,MAAM,KAAK,GAAG,GAAG,EAAE,CAAC,aAAa,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC;IACpE,MAAM,OAAO,GAAG;QACd,eAAe,EAAE,CAAC,KAAc,EAAE,EAAE,CAClC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI;QAC3E,QAAQ,EAAE,KAAK;QACf,MAAM,EAAE,KAAK;KACd,CAAC;IACF,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC1B,OAAO,OAA6B,CAAC;AACvC,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc;IACvC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAC1E,CAAC;AACJ,CAAC;AA+DD,SAAS,KAAK,CAAC,IAAuB;IACpC,OAAO,IAAI,CAAC,KAAK;SACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC;SACtC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC;AAED,SAAS,oBAAoB,CAC3B,IAAuB,EACvB,IAAY,EACZ,OAAgC,EAAE;IAElC,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IACzD,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QAClD,IAAI,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACxD,MAAM,IAAI,KAAK,CACb,wCAAwC,IAAI,IAAI,KAAK,OAAO,KAAK,sCAAsC,CACxG,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,IAAuB;IAC1C,OAAO,IAAI,CAAC,KAAK;SACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,WAAW,CAAC;SAC3C,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC;AAED,SAAS,cAAc,CACrB,IAAuB,EACvB,IAAY;IAEZ,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,WAAW,CAAC,IAAuB;IAC1C,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;AACjE,CAAC;AAED,SAAS,aAAa,CACpB,IAAuB,EACvB,IAAY;IAEZ,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;AACxE,CAAC;AAED,SAAS,QAAQ,CAAC,IAAuB;IACvC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IACzB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,wBAAwB,CAAC;IAClC,CAAC;IACD,OAAO,IAAI;SACR,GAAG,CACF,CAAC,IAAuB,EAAE,KAAa,EAAE,EAAE,CACzC,KAAK,KAAK,GAAG,CAAC,KAAK,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,cAAc,IAAI,CAAC,MAAM,GAAG,EAAE,CACnI;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AAC1C,CAAC;AAED,SAAS,IAAI,CAAC,IAAuB,EAAE,IAAqB;IAC1D,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CACrC,OAAO,IAAI,KAAK,QAAQ;QACtB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;QACrE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CACpB,CAAC;AACJ,CAAC;AAED,SAAS,MAAM,CAAC,IAAuB;IACrC,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,gCAAgC,CAAC;IAC1C,CAAC;IACD,OAAO,KAAK;SACT,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,GAAG,CAAC,KAAK,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;SACzD,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,SAAS,KAAK,CAAC,IAAY;IACzB,OAAO,IAAI,CAAC,OAAO,CAAC,uBAAuB,EAAE,QAAQ,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,YAAY,GAAG;;;6KAGwJ,CAAC;AAE9K,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAE9B,SAAS,QAAQ,CAAC,IAAmB;IACnC,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QACzB,OAAO,GAAG,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;IAC7C,CAAC;IACD,MAAM,OAAO,GACX,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS;QAC5B,CAAC,CAAC,OAAO,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,EAAE;QACxD,CAAC,CAAC,aAAa,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;IACvC,OAAO,SAAS,KAAK,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,OAAO,EAAE,CAAC,EAAE,CAAC;AACxG,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,IAAI,CAAC,MAAM,GAAG,gBAAgB;QACnC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC,eAAe;QACnD,CAAC,CAAC,IAAI,CAAC;AACX,CAAC;AAED,SAAS,aAAa,CAAC,IAAuB;IAC5C,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,6BAA6B,CAAC;IACvC,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC7C,CAAC;AAED,SAAS,MAAM,CAAC,IAAuB,EAAE,OAAe;IACtD,OAAO,GAAG,OAAO;;;EAGjB,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;AACnB,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,IAAuB,EACvB,QAAgB,EAChB,KAAqB;IAErB,MAAM,EAAE,cAAc,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;IAC1D,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,cAAc,CAAC;QACtC,KAAK,EAAE,KAAK,IAAI,IAAI,CAAC,KAAK;QAC1B,MAAM,EAAE,YAAY;QACpB,MAAM,EAAE,cAAc,QAAQ,uBAAuB,aAAa,CAAC,IAAI,CAAC,mBAAmB;QAC3F,MAAM,EAAE,UAAU,CAAkC;YAClD,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;gBACzB,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;aAC9B;YACD,QAAQ,EAAE,CAAC,MAAM,EAAE,WAAW,CAAC;YAC/B,oBAAoB,EAAE,KAAK;SAC5B,CAAC;KACH,CAAC,CAAC;IACH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,MAAM,CAAC;IACZ,oBAAoB,CAClB,QAA2B,EAC3B,IAAY,EACZ,IAA8B;QAE9B,oBAAoB,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;QAC3C,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC/C,MAAM,SAAS,GACb,IAAI,KAAK,SAAS;YAClB,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACpB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxE,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC;QAEhD,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,aAAa,IAAI,kCAAkC;gBACrD,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;oBACrB,CAAC,CAAC,aAAa,IAAI,4BAA4B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,4BAA4B,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG;oBACxI,CAAC,CAAC,aAAa,IAAI,6DAA6D,QAAQ,CAAC,MAAM,wBAAwB,CAC5H;SACJ,CAAC;IACJ,CAAC;IAED,oBAAoB,CAClB,QAA2B,EAC3B,IAAY,EACZ,IAA6B;QAE7B,oBAAoB,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;QAC3C,MAAM,IAAI,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CACvD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAC3D,CAAC;QAEF,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,wBAAwB,IAAI,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG;gBAC/D,CAAC,CAAC,aAAa,IAAI,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CACpE;SACJ,CAAC;IACJ,CAAC;IAED,oBAAoB,CAAC,QAA2B,EAAE,IAAY;QAC5D,MAAM,SAAS,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;QAErE,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC;YACvB,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,MAAM,CAAC,MAAM,GAAG,CAAC;gBACf,CAAC,CAAC,2BAA2B,IAAI,eAAe;gBAChD,CAAC,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;oBACtB,CAAC,CAAC,aAAa,IAAI,mDAAmD;oBACtE,CAAC,CAAC,aAAa,IAAI,sBAAsB,SAAS,CAAC,MAAM,mBAAmB,CACjF;SACJ,CAAC;IACJ,CAAC;IAED,UAAU,CAAC,QAA2B,EAAE,IAAqB;QAC3D,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAClC,MAAM,MAAM,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAErE,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,GACE,IAAI;gBACF,CAAC,CAAC,qCAAqC,MAAM,GAAG;gBAChD,CAAC,CAAC,iCAAiC,MAAM,GAC7C;;;EAGN,MAAM,CAAC,QAAQ,CAAC,EAAE;SACf,CAAC;IACJ,CAAC;IAED,mBAAmB,CAAC,QAA2B;QAC7C,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;QAErC,OAAO;YACL,IAAI,EAAE,KAAK,KAAK,CAAC;YACjB,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,KAAK,KAAK,CAAC;gBACT,CAAC,CAAC,+CAA+C;gBACjD,CAAC,CAAC,wDAAwD,KAAK,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,GAAG,CACvG;SACJ,CAAC;IACJ,CAAC;IAED,wBAAwB,CAAC,QAA2B,EAAE,GAAG,KAAe;QACtE,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;YACzC,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;gBACzC,OAAO,IAAI,CAAC,CAAC;YACf,CAAC;YACD,IAAI,OAAO,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;gBAC7B,MAAM;YACR,CAAC;QACH,CAAC;QACD,MAAM,IAAI,GAAG,OAAO,KAAK,KAAK,CAAC,MAAM,CAAC;QACtC,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAE/D,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,YAAY,MAAM,kCAAkC;gBACtD,CAAC,CAAC,YAAY,MAAM,wBAAwB,KAAK,CAAC,OAAO,CAAC,eAAe,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CACpI;SACJ,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,cAAc,CAClB,QAA2B,EAC3B,QAAgB,EAChB,OAAyB;QAEzB,IAAI,OAAgB,CAAC;QACrB,IAAI,CAAC;YACH,OAAO,GAAG,OAAO,EAAE,KAAK;gBACtB,CAAC,CAAC,MAAM,OAAO,CAAC,KAAK,CAAC;oBAClB,QAAQ;oBACR,UAAU,EAAE,aAAa,CAAC,QAAQ,CAAC;iBACpC,CAAC;gBACJ,CAAC,CAAC,MAAM,YAAY,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;YAE3D,IAAI,OAAO,OAAO,EAAE,IAAI,KAAK,SAAS,EAAE,CAAC;gBACvC,MAAM,IAAI,KAAK,CACb,sBAAsB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,6CAA6C,CAC3F,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,sBAAsB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC/E,CAAC;QACJ,CAAC;QAED,OAAO;YACL,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,OAAO,EAAE,GAAG,EAAE,CACZ,6BAA6B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;KAC1D,QAAQ;;SAEJ,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,OAAO,CAAC,SAAS,EAAE,EAAE;SACxG,CAAC;IACJ,CAAC;IAED,qBAAqB,CAAC,QAA2B,EAAE,IAAY;QAC7D,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAEhD,OAAO;YACL,IAAI,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC;YAC3B,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACnB,CAAC,CAAC,aAAa,IAAI,+BAA+B;gBAClD,CAAC,CAAC,aAAa,IAAI,2CAA2C,QAAQ,CAAC,MAAM,SAAS,CACzF;SACJ,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,IAAI,GAAG,CAAM,IAAmB,EAAsB,EAAE,CAC7D,MAAM,CAAC,IAAI,CAAkC,CAAC;AAChD,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAoB,KAAK,MAAM,EAAE,MAAM,8BAA8B,CAAC;AAC7E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACxC,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7D,OAAO,EAAE,IAAI,EAAgB,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAoB,KAAK,MAAM,EAAE,MAAM,8BAA8B,CAAC;AAC7E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACxC,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7D,OAAO,EAAE,IAAI,EAAgB,MAAM,WAAW,CAAC;AAG/C;;;;;GAKG;AACH,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5C,oBAAoB,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;CACzC;AAED;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,GAAG,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACjC;AAED;;;;;GAKG;AACH,MAAM,MAAM,SAAS,CAAC,GAAG,IAAI;KAC1B,IAAI,IAAI,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,EAAE,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,OAAO;CACnE,CAAC;AAEF,UAAU,YAAY;IACpB,uDAAuD;IACvD,KAAK,EAAE,aAAa,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAUD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,KAAK,CAAC,GAAG,SAAS,OAAO,EAC7C,OAAO,EAAE,YAAY,GAAG;IACtB,GAAG,EAAE,GAAG,CAAC;IACT,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,KAAK,CAAC,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC;CACxB,GACA,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAmCpB"}
|
package/dist/session-registry.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { createMcpHandler } from "@modelcontextprotocol/server";
|
|
2
2
|
import { inject, onTestFinished } from "vitest";
|
|
3
3
|
import { Chat } from "./chat.js";
|
|
4
|
+
import { EvalMiddleware } from "./eval-middleware.js";
|
|
4
5
|
function sharedDefaults() {
|
|
5
6
|
try {
|
|
6
7
|
return inject("skybridgeEvals");
|
|
@@ -32,12 +33,21 @@ export async function start(options) {
|
|
|
32
33
|
const config = sharedDefaults();
|
|
33
34
|
const { app, authInfo } = options;
|
|
34
35
|
const handler = createMcpHandler(() => app.createServerInstance());
|
|
35
|
-
const
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
36
|
+
const middleware = new EvalMiddleware((url, init) => handler.fetch(new Request(url, init), authInfo === undefined ? undefined : { authInfo }), (options.stubs ?? {}));
|
|
37
|
+
let chat;
|
|
38
|
+
try {
|
|
39
|
+
chat = await Chat.open({
|
|
40
|
+
model: options.model,
|
|
41
|
+
temperature: options.temperature ?? config?.temperature,
|
|
42
|
+
systemPrompt: options.systemPrompt ?? config?.systemPrompt,
|
|
43
|
+
maxSteps: options.maxSteps ?? config?.maxSteps,
|
|
44
|
+
}, middleware);
|
|
45
|
+
}
|
|
46
|
+
catch (error) {
|
|
47
|
+
await middleware.files.close();
|
|
48
|
+
await handler.close();
|
|
49
|
+
throw error;
|
|
50
|
+
}
|
|
41
51
|
onTestFinished(async () => {
|
|
42
52
|
await chat.close();
|
|
43
53
|
await handler.close();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-registry.js","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAe,MAAM,8BAA8B,CAAC;AAG7E,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAC;AAChD,OAAO,EAAE,IAAI,EAAgB,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"session-registry.js","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAe,MAAM,8BAA8B,CAAC;AAG7E,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAC;AAChD,OAAO,EAAE,IAAI,EAAgB,MAAM,WAAW,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AA6CtD,SAAS,cAAc;IACrB,IAAI,CAAC;QACH,OAAO,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CACzB,OAIC;IAED,MAAM,MAAM,GAAG,cAAc,EAAE,CAAC;IAChC,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAClC,MAAM,OAAO,GAAG,gBAAgB,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,oBAAoB,EAAE,CAAC,CAAC;IAEnE,MAAM,UAAU,GAAG,IAAI,cAAc,CACnC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CACZ,OAAO,CAAC,KAAK,CACX,IAAI,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,EACtB,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAClD,EACH,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE,CAAY,CACjC,CAAC;IACF,IAAI,IAAe,CAAC;IACpB,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CACpB;YACE,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,MAAM,EAAE,WAAW;YACvD,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,MAAM,EAAE,YAAY;YAC1D,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,MAAM,EAAE,QAAQ;SAC/C,EACD,UAAU,CACX,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,UAAU,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QAC/B,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACtB,MAAM,KAAK,CAAC;IACd,CAAC;IAED,cAAc,CAAC,KAAK,IAAI,EAAE;QACxB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC,CAAC,CAAC;IACH,OAAO,IAAI,CAAC;AACd,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@skybridge/test",
|
|
3
|
-
"version": "0.0.0-dev.
|
|
3
|
+
"version": "0.0.0-dev.97e4bf7",
|
|
4
4
|
"description": "Eval runner for Skybridge MCP Apps: does a prompt reach the right tool with the right arguments",
|
|
5
5
|
"license": "ISC",
|
|
6
6
|
"type": "module",
|
|
@@ -37,8 +37,8 @@
|
|
|
37
37
|
"typescript": "^5.9.3",
|
|
38
38
|
"vitest": "^4.1.10",
|
|
39
39
|
"zod": "^4.4.3",
|
|
40
|
-
"
|
|
41
|
-
"skybridge": "0.0.0-dev.
|
|
40
|
+
"skybridge": "0.0.0-dev.97e4bf7",
|
|
41
|
+
"@skybridge/vite-plugin": "0.0.0-dev.97e4bf7"
|
|
42
42
|
},
|
|
43
43
|
"engines": {
|
|
44
44
|
"node": ">=22.23.1"
|