@skybridge/test 0.0.0-dev.69d93ba → 0.0.0-dev.6e30880
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -1
- package/dist/chat.d.ts +6 -2
- package/dist/chat.d.ts.map +1 -1
- package/dist/chat.js +99 -8
- package/dist/chat.js.map +1 -1
- package/dist/input-files.d.ts +20 -0
- package/dist/input-files.d.ts.map +1 -0
- package/dist/input-files.js +163 -0
- package/dist/input-files.js.map +1 -0
- package/dist/matchers/matchers.d.ts +12 -3
- package/dist/matchers/matchers.d.ts.map +1 -1
- package/dist/matchers/matchers.js +12 -0
- package/dist/matchers/matchers.js.map +1 -1
- package/dist/matchers/types.d.ts +1 -0
- package/dist/matchers/types.d.ts.map +1 -1
- package/package.json +4 -4
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# @skybridge/test
|
|
2
2
|
|
|
3
|
-
Evals for [Skybridge](https://skybridge.tech) MCP
|
|
3
|
+
Evals for [Skybridge](https://skybridge.tech) MCP Apps: they answer whether a
|
|
4
4
|
prompt reaches the right tool with the right arguments, which no unit test can
|
|
5
5
|
check.
|
|
6
6
|
|
|
@@ -82,6 +82,48 @@ from, so a project can set them once instead of per scenario.
|
|
|
82
82
|
skybridge({ evals: { temperature: 0, maxSteps: 6 } });
|
|
83
83
|
```
|
|
84
84
|
|
|
85
|
+
## Input files
|
|
86
|
+
|
|
87
|
+
`chat.send` also accepts AI SDK `UserContent`. Attach a `FilePart` and use the
|
|
88
|
+
same object in `expect.inputFile` to check which file a tool received:
|
|
89
|
+
|
|
90
|
+
```ts
|
|
91
|
+
import { readFile } from "node:fs/promises";
|
|
92
|
+
import type { FilePart } from "ai";
|
|
93
|
+
|
|
94
|
+
const portrait: FilePart = {
|
|
95
|
+
type: "file",
|
|
96
|
+
mediaType: "image/png",
|
|
97
|
+
filename: "photo.png",
|
|
98
|
+
data: await readFile(new URL("./fixtures/photo.png", import.meta.url)),
|
|
99
|
+
};
|
|
100
|
+
|
|
101
|
+
await chat.send([{ type: "text", text: "Resize this photo" }, portrait]);
|
|
102
|
+
|
|
103
|
+
expect.chat(chat).toHaveCalledToolWith("image_crop_and_resize", {
|
|
104
|
+
file: expect.inputFile(portrait),
|
|
105
|
+
});
|
|
106
|
+
expect.chat(chat).toHaveCalledToolOnce("asset_openai_file_upload", {
|
|
107
|
+
files: [expect.inputFile(portrait)],
|
|
108
|
+
});
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
The example assumes `file` and `files` are declared in their tools'
|
|
112
|
+
`_meta["openai/fileParams"]`. The model sees each attachment's name, synthetic
|
|
113
|
+
absolute workspace path, and content. Declared file fields appear as path strings
|
|
114
|
+
or arrays of path strings. The host turns the selected paths into OpenAI file
|
|
115
|
+
objects before the handler or stub runs. The local download URLs serve the
|
|
116
|
+
attached bytes until the conversation closes; paths are host-local identifiers
|
|
117
|
+
and do not require files on disk.
|
|
118
|
+
|
|
119
|
+
`expect.inputFile(part)` accepts that file's path, id, URL, file object, or an array
|
|
120
|
+
containing one of them. Distinct file objects never match each other, even with
|
|
121
|
+
identical bytes. It works with both tool argument matchers and their `.not` forms.
|
|
122
|
+
|
|
123
|
+
Using it for a parameter not listed in `openai/fileParams` throws immediately,
|
|
124
|
+
including nested matchers and negative assertions. Ordinary URL parameters remain
|
|
125
|
+
plain strings and do not receive automatic file conversion or attachment URLs.
|
|
126
|
+
|
|
85
127
|
## Matchers
|
|
86
128
|
|
|
87
129
|
Assertions on what the assistant said work the same way, though model prose
|
package/dist/chat.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type LanguageModel } from "ai";
|
|
1
|
+
import { type LanguageModel, type UserContent } from "ai";
|
|
2
2
|
import type { ToolCall, Turn } from "./matchers/types.js";
|
|
3
3
|
interface HostConfig {
|
|
4
4
|
model: LanguageModel;
|
|
@@ -33,6 +33,9 @@ export declare class Chat<App = unknown> {
|
|
|
33
33
|
get assistantTurns(): string[];
|
|
34
34
|
private readonly messages;
|
|
35
35
|
private readonly failures;
|
|
36
|
+
private readonly inputs;
|
|
37
|
+
private readonly inputFiles;
|
|
38
|
+
private readonly schemaValidator;
|
|
36
39
|
private readonly client;
|
|
37
40
|
private readonly transport;
|
|
38
41
|
private readonly host;
|
|
@@ -40,10 +43,11 @@ export declare class Chat<App = unknown> {
|
|
|
40
43
|
private tools;
|
|
41
44
|
/** The model the conversation runs on, and the judge's default. */
|
|
42
45
|
get model(): LanguageModel;
|
|
46
|
+
get fileParams(): Map<string, string[]>;
|
|
43
47
|
private constructor();
|
|
44
48
|
static open<App>(host: HostConfig, fetchImpl: TransportFetch, stubs: StubMap): Promise<Chat<App>>;
|
|
45
49
|
/** Takes a turn. Assert on the result with `expect.chat`. */
|
|
46
|
-
send(
|
|
50
|
+
send(content: UserContent): Promise<void>;
|
|
47
51
|
close(): Promise<void>;
|
|
48
52
|
private failureFor;
|
|
49
53
|
private toolSet;
|
package/dist/chat.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAMA,OAAO,EAKL,KAAK,aAAa,EAGlB,KAAK,WAAW,EACjB,MAAM,IAAI,CAAC;AAEZ,OAAO,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AAE1D,UAAU,UAAU;IAClB,KAAK,EAAE,aAAa,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;GAKG;AACH,MAAM,MAAM,OAAO,GAAG,MAAM,CAC1B,MAAM,EACN,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,OAAO,CAC3C,CAAC;AAgEF;;;GAGG;AACH,MAAM,MAAM,cAAc,GAAG,CAC3B,GAAG,EAAE,MAAM,GAAG,GAAG,EACjB,IAAI,CAAC,EAAE,WAAW,KACf,OAAO,CAAC,QAAQ,CAAC,CAAC;AAEvB;;;GAGG;AACH,qBAAa,IAAI,CAAC,GAAG,GAAG,OAAO;IAC7B,4EAA4E;IAC5E,SAAiB,IAAI,EAAE,GAAG,CAAC;IAE3B,0DAA0D;IAC1D,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAM;IAEjC,mEAAmE;IACnE,IAAI,SAAS,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE,CAI/B;IAED,+EAA+E;IAC/E,IAAI,cAAc,IAAI,MAAM,EAAE,CAI7B;IAED,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA6B;IACtD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA8C;IACrE,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAoB;IAC/C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAgC;IAChE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAa;IAClC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA+B;IACrD,OAAO,CAAC,KAAK,CAAc;IAE3B,mEAAmE;IACnE,IAAI,KAAK,IAAI,aAAa,CAEzB;IAED,IAAI,UAAU,0BAEb;IAED,OAAO;WAYM,IAAI,CAAC,GAAG,EACnB,IAAI,EAAE,UAAU,EAChB,SAAS,EAAE,cAAc,EACzB,KAAK,EAAE,OAAO,GACb,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IA0BrB,6DAA6D;IACvD,IAAI,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAiDzC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ5B,OAAO,CAAC,UAAU;IAalB,OAAO,CAAC,OAAO;CA+EhB"}
|
package/dist/chat.js
CHANGED
|
@@ -1,7 +1,43 @@
|
|
|
1
1
|
import { Client, StreamableHTTPClientTransport, } from "@modelcontextprotocol/client";
|
|
2
|
+
import { AjvJsonSchemaValidator } from "@modelcontextprotocol/client/validators/ajv";
|
|
2
3
|
import { dynamicTool, generateText, jsonSchema, stepCountIs, } from "ai";
|
|
4
|
+
import { InputFiles } from "./input-files.js";
|
|
3
5
|
const DEFAULT_MAX_STEPS = 8;
|
|
4
6
|
const IN_PROCESS_URL = "http://in-process.skybridge.test/mcp";
|
|
7
|
+
function fileParameters(tool) {
|
|
8
|
+
const fields = tool._meta?.["openai/fileParams"];
|
|
9
|
+
return Array.isArray(fields)
|
|
10
|
+
? fields.filter((field) => typeof field === "string")
|
|
11
|
+
: [];
|
|
12
|
+
}
|
|
13
|
+
function fileInputSchema(schema) {
|
|
14
|
+
if (typeof schema === "boolean") {
|
|
15
|
+
return schema;
|
|
16
|
+
}
|
|
17
|
+
if (schema.anyOf) {
|
|
18
|
+
return { ...schema, anyOf: schema.anyOf.map(fileInputSchema) };
|
|
19
|
+
}
|
|
20
|
+
if (schema.oneOf) {
|
|
21
|
+
return { ...schema, oneOf: schema.oneOf.map(fileInputSchema) };
|
|
22
|
+
}
|
|
23
|
+
if (schema.type === "array") {
|
|
24
|
+
return {
|
|
25
|
+
...schema,
|
|
26
|
+
items: Array.isArray(schema.items)
|
|
27
|
+
? schema.items.map(fileInputSchema)
|
|
28
|
+
: schema.items === undefined
|
|
29
|
+
? undefined
|
|
30
|
+
: fileInputSchema(schema.items),
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
if (schema.type === "object" || schema.$ref) {
|
|
34
|
+
return {
|
|
35
|
+
type: "string",
|
|
36
|
+
description: "This parameter expects an absolute local file path.",
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
return schema;
|
|
40
|
+
}
|
|
5
41
|
/**
|
|
6
42
|
* Dress a stubbed value the way the server dresses a real one, so the model
|
|
7
43
|
* reads the same envelope under a stub as it does live.
|
|
@@ -40,6 +76,9 @@ export class Chat {
|
|
|
40
76
|
}
|
|
41
77
|
messages = [];
|
|
42
78
|
failures = new Map();
|
|
79
|
+
inputs = new Map();
|
|
80
|
+
inputFiles = new InputFiles();
|
|
81
|
+
schemaValidator = new AjvJsonSchemaValidator();
|
|
43
82
|
client;
|
|
44
83
|
transport;
|
|
45
84
|
host;
|
|
@@ -49,6 +88,9 @@ export class Chat {
|
|
|
49
88
|
get model() {
|
|
50
89
|
return this.host.model;
|
|
51
90
|
}
|
|
91
|
+
get fileParams() {
|
|
92
|
+
return new Map(this.tools.map((tool) => [tool.name, fileParameters(tool)]));
|
|
93
|
+
}
|
|
52
94
|
constructor(client, transport, host, stubs) {
|
|
53
95
|
this.client = client;
|
|
54
96
|
this.transport = transport;
|
|
@@ -75,9 +117,20 @@ export class Chat {
|
|
|
75
117
|
return chat;
|
|
76
118
|
}
|
|
77
119
|
/** Takes a turn. Assert on the result with `expect.chat`. */
|
|
78
|
-
async send(
|
|
79
|
-
this.
|
|
80
|
-
this.
|
|
120
|
+
async send(content) {
|
|
121
|
+
const prepared = await this.inputFiles.prepare(content);
|
|
122
|
+
this.inputs.clear();
|
|
123
|
+
this.failures.clear();
|
|
124
|
+
this.messages.push({ role: "user", content: prepared });
|
|
125
|
+
this.turns.push({
|
|
126
|
+
role: "user",
|
|
127
|
+
text: typeof prepared === "string"
|
|
128
|
+
? prepared
|
|
129
|
+
: prepared
|
|
130
|
+
.filter((part) => part.type === "text")
|
|
131
|
+
.map((part) => part.text)
|
|
132
|
+
.join("\n"),
|
|
133
|
+
});
|
|
81
134
|
const result = await generateText({
|
|
82
135
|
model: this.host.model,
|
|
83
136
|
temperature: this.host.temperature ?? 0,
|
|
@@ -97,7 +150,8 @@ export class Chat {
|
|
|
97
150
|
role: "tool",
|
|
98
151
|
call: {
|
|
99
152
|
name: call.toolName,
|
|
100
|
-
arguments: (call.
|
|
153
|
+
arguments: this.inputs.get(call.toolCallId) ??
|
|
154
|
+
(call.input ?? {}),
|
|
101
155
|
...this.failureFor(call),
|
|
102
156
|
},
|
|
103
157
|
result: outputOf(step.toolResults, call.toolCallId),
|
|
@@ -107,7 +161,12 @@ export class Chat {
|
|
|
107
161
|
this.messages.push(...result.response.messages);
|
|
108
162
|
}
|
|
109
163
|
async close() {
|
|
110
|
-
|
|
164
|
+
try {
|
|
165
|
+
await this.transport.close();
|
|
166
|
+
}
|
|
167
|
+
finally {
|
|
168
|
+
await this.inputFiles.close();
|
|
169
|
+
}
|
|
111
170
|
}
|
|
112
171
|
failureFor(call) {
|
|
113
172
|
const reported = this.failures.get(call.toolCallId);
|
|
@@ -122,13 +181,45 @@ export class Chat {
|
|
|
122
181
|
toolSet() {
|
|
123
182
|
const entries = this.tools.map((definition) => {
|
|
124
183
|
const { $schema: _ignored, ...parameters } = definition.inputSchema;
|
|
184
|
+
const fields = fileParameters(definition);
|
|
185
|
+
const properties = { ...parameters.properties };
|
|
186
|
+
for (const field of fields) {
|
|
187
|
+
if (field in properties) {
|
|
188
|
+
properties[field] = fileInputSchema(properties[field]);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
const inputSchema = { ...parameters, properties };
|
|
192
|
+
const validate = fields.length === 0
|
|
193
|
+
? undefined
|
|
194
|
+
: this.schemaValidator.getValidator({
|
|
195
|
+
...definition.inputSchema,
|
|
196
|
+
properties,
|
|
197
|
+
});
|
|
125
198
|
return [
|
|
126
199
|
definition.name,
|
|
127
200
|
dynamicTool({
|
|
128
201
|
description: definition.description ?? "",
|
|
129
|
-
inputSchema: jsonSchema(
|
|
202
|
+
inputSchema: jsonSchema(inputSchema, {
|
|
203
|
+
validate: validate
|
|
204
|
+
? (input) => {
|
|
205
|
+
const result = validate(input);
|
|
206
|
+
return result.valid
|
|
207
|
+
? { success: true, value: result.data }
|
|
208
|
+
: { success: false, error: new Error(result.errorMessage) };
|
|
209
|
+
}
|
|
210
|
+
: undefined,
|
|
211
|
+
}),
|
|
130
212
|
execute: async (input, { toolCallId }) => {
|
|
131
|
-
|
|
213
|
+
let args;
|
|
214
|
+
try {
|
|
215
|
+
args = this.inputFiles.rewrite((input ?? {}), fields);
|
|
216
|
+
}
|
|
217
|
+
catch (error) {
|
|
218
|
+
this.failures.set(toolCallId, String(error));
|
|
219
|
+
throw error;
|
|
220
|
+
}
|
|
221
|
+
this.inputs.set(toolCallId, args);
|
|
222
|
+
const stubbed = await this.stubs.get(definition.name)?.(args);
|
|
132
223
|
if (stubbed !== undefined) {
|
|
133
224
|
return asToolResult(stubbed);
|
|
134
225
|
}
|
|
@@ -136,7 +227,7 @@ export class Chat {
|
|
|
136
227
|
try {
|
|
137
228
|
result = await this.client.callTool({
|
|
138
229
|
name: definition.name,
|
|
139
|
-
arguments:
|
|
230
|
+
arguments: args,
|
|
140
231
|
});
|
|
141
232
|
}
|
|
142
233
|
catch (error) {
|
package/dist/chat.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,MAAM,EACN,6BAA6B,GAE9B,MAAM,8BAA8B,CAAC;AACtC,OAAO,EACL,WAAW,EACX,YAAY,
|
|
1
|
+
{"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,MAAM,EACN,6BAA6B,GAE9B,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAAE,sBAAsB,EAAE,MAAM,6CAA6C,CAAC;AACrF,OAAO,EACL,WAAW,EACX,YAAY,EAEZ,UAAU,EAGV,WAAW,GAEZ,MAAM,IAAI,CAAC;AACZ,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAqB9C,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B,MAAM,cAAc,GAAG,sCAAsC,CAAC;AAE9D,SAAS,cAAc,CAAC,IAAU;IAChC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,mBAAmB,CAAC,CAAC;IACjD,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;QAC1B,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC;QACtE,CAAC,CAAC,EAAE,CAAC;AACT,CAAC;AAED,SAAS,eAAe,CAAC,MAA6B;IACpD,IAAI,OAAO,MAAM,KAAK,SAAS,EAAE,CAAC;QAChC,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,OAAO,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,CAAC;IACjE,CAAC;IACD,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,OAAO,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,CAAC;IACjE,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC5B,OAAO;YACL,GAAG,MAAM;YACT,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;gBAChC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,eAAe,CAAC;gBACnC,CAAC,CAAC,MAAM,CAAC,KAAK,KAAK,SAAS;oBAC1B,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC;SACpC,CAAC;IACJ,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QAC5C,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,qDAAqD;SACnE,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO;QACL,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;aAChE;SACF;KACF,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CACf,OAAmD,EACnD,UAAkB;IAElB,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,KAAK,UAAU,CAAC,EAAE,MAAM,CAAC;AAC5E,CAAC;AAWD;;;GAGG;AACH,MAAM,OAAO,IAAI;IAIf,0DAA0D;IACjD,KAAK,GAAgB,EAAE,CAAC;IAEjC,mEAAmE;IACnE,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,KAAK;aACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC;aACtC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAED,+EAA+E;IAC/E,IAAI,cAAc;QAChB,OAAO,IAAI,CAAC,KAAK;aACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,WAAW,CAAC;aAC3C,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAEgB,QAAQ,GAAmB,EAAE,CAAC;IAC9B,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrC,MAAM,GAAG,IAAI,GAAG,EAAmC,CAAC;IACpD,UAAU,GAAG,IAAI,UAAU,EAAE,CAAC;IAC9B,eAAe,GAAG,IAAI,sBAAsB,EAAE,CAAC;IAC/C,MAAM,CAAS;IACf,SAAS,CAAgC;IACzC,IAAI,CAAa;IACjB,KAAK,CAA+B;IAC7C,KAAK,GAAW,EAAE,CAAC;IAE3B,mEAAmE;IACnE,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;IACzB,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9E,CAAC;IAED,YACE,MAAc,EACd,SAAwC,EACxC,IAAgB,EAChB,KAAc;QAEd,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,IAAI,CACf,IAAgB,EAChB,SAAyB,EACzB,KAAc;QAEd,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,IAAI,6BAA6B,CACjD,IAAI,GAAG,CAAC,cAAc,CAAC,EACvB,EAAE,KAAK,EAAE,SAAS,EAAE,CACrB,CAAC;QACF,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAM,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;QAC3D,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;YACnB,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAC5C,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,CACpD,CAAC;YACF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,IAAI,KAAK,CACb,iCAAiC,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,+BAA+B,CACnH,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YACxB,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,6DAA6D;IAC7D,KAAK,CAAC,IAAI,CAAC,OAAoB;QAC7B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACxD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;QACxD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;YACd,IAAI,EAAE,MAAM;YACZ,IAAI,EACF,OAAO,QAAQ,KAAK,QAAQ;gBAC1B,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,QAAQ;qBACL,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC;qBACtC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;qBACxB,IAAI,CAAC,IAAI,CAAC;SACpB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;YACtB,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC;YACvC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,KAAK,SAAS;gBACtC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE;YACrB,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;SAC/D,CAAC,CAAC;QAEH,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,IAAI,CAAC,IAAI,KAAK,EAAE,EAAE,CAAC;gBACrB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YAC1D,CAAC;YACD,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAClC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;oBACd,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE;wBACJ,IAAI,EAAE,IAAI,CAAC,QAAQ;wBACnB,SAAS,EACP,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC;4BAC/B,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAA6B;wBACjD,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;qBACR;oBAClB,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,UAAU,CAAC;iBACpD,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QAC/B,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QAChC,CAAC;IACH,CAAC;IAEO,UAAU,CAAC,IAA+C;QAGhE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QAC9B,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC1B,OAAO,EAAE,MAAM,EAAE,2CAA2C,EAAE,CAAC;QACjE,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAEO,OAAO;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;YAC5C,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,UAAU,EAAE,GAAG,UAAU,CAAC,WAAW,CAAC;YACpE,MAAM,MAAM,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC;YAC1C,MAAM,UAAU,GAAG,EAAE,GAAG,UAAU,CAAC,UAAU,EAE5C,CAAC;YACF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,IAAI,KAAK,IAAI,UAAU,EAAE,CAAC;oBACxB,UAAU,CAAC,KAAK,CAAC,GAAG,eAAe,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;gBACzD,CAAC;YACH,CAAC;YACD,MAAM,WAAW,GAAG,EAAE,GAAG,UAAU,EAAE,UAAU,EAAE,CAAC;YAClD,MAAM,QAAQ,GACZ,MAAM,CAAC,MAAM,KAAK,CAAC;gBACjB,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,YAAY,CAAU;oBACzC,GAAG,UAAU,CAAC,WAAW;oBACzB,UAAU;iBAC8C,CAAC,CAAC;YAClE,OAAO;gBACL,UAAU,CAAC,IAAI;gBACf,WAAW,CAAC;oBACV,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,EAAE;oBACzC,WAAW,EAAE,UAAU,CAAC,WAA0B,EAAE;wBAClD,QAAQ,EAAE,QAAQ;4BAChB,CAAC,CAAC,CAAC,KAAK,EAAE,EAAE;gCACR,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;gCAC/B,OAAO,MAAM,CAAC,KAAK;oCACjB,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,EAAE;oCACvC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,KAAK,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC;4BAChE,CAAC;4BACH,CAAC,CAAC,SAAS;qBACd,CAAC;oBACF,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE;wBACvC,IAAI,IAA6B,CAAC;wBAClC,IAAI,CAAC;4BACH,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAC5B,CAAC,KAAK,IAAI,EAAE,CAA4B,EACxC,MAAM,CACP,CAAC;wBACJ,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BACf,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;4BAC7C,MAAM,KAAK,CAAC;wBACd,CAAC;wBACD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;wBAClC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;wBAC9D,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;4BAC1B,OAAO,YAAY,CAAC,OAAO,CAAC,CAAC;wBAC/B,CAAC;wBACD,IAAI,MAA+C,CAAC;wBACpD,IAAI,CAAC;4BACH,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;gCAClC,IAAI,EAAE,UAAU,CAAC,IAAI;gCACrB,SAAS,EAAE,IAAI;6BAChB,CAAC,CAAC;wBACL,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BACf,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,sCAAsC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC/F,CAAC;4BACF,MAAM,KAAK,CAAC;wBACd,CAAC;wBACD,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;4BAC5B,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,+BAA+B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAChE,CAAC;wBACJ,CAAC;wBACD,OAAO;4BACL,OAAO,EAAE,MAAM,CAAC,OAAO;4BACvB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;yBAC5C,CAAC;oBACJ,CAAC;iBACF,CAAC;aACM,CAAC;QACb,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;CACF"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { FilePart, UserContent } from "ai";
|
|
2
|
+
export interface InputFileMatcher {
|
|
3
|
+
asymmetricMatch(value: unknown): boolean;
|
|
4
|
+
toString(): string;
|
|
5
|
+
toJSON(): string;
|
|
6
|
+
}
|
|
7
|
+
export declare function inputFile(part: FilePart): InputFileMatcher;
|
|
8
|
+
export declare function containsInputFile(value: unknown): boolean;
|
|
9
|
+
export declare class InputFiles {
|
|
10
|
+
private readonly files;
|
|
11
|
+
private readonly parts;
|
|
12
|
+
private readonly paths;
|
|
13
|
+
private readonly server;
|
|
14
|
+
prepare(content: UserContent): Promise<UserContent>;
|
|
15
|
+
rewrite(input: Record<string, unknown>, fields: string[]): {
|
|
16
|
+
[x: string]: unknown;
|
|
17
|
+
};
|
|
18
|
+
close(): Promise<void>;
|
|
19
|
+
}
|
|
20
|
+
//# sourceMappingURL=input-files.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"input-files.d.ts","sourceRoot":"","sources":["../src/input-files.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,IAAI,CAAC;AAiBhD,MAAM,WAAW,gBAAgB;IAC/B,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC;IACzC,QAAQ,IAAI,MAAM,CAAC;IACnB,MAAM,IAAI,MAAM,CAAC;CAClB;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,QAAQ,GAAG,gBAAgB,CA8B1D;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAOzD;AAoCD,qBAAa,UAAU;IACrB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAGlB;IACJ,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAuC;IAC7D,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAoC;IAC1D,OAAO,CAAC,QAAQ,CAAC,MAAM,CAQpB;IAEG,OAAO,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAyDzD,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE;;;IAuBlD,KAAK;CASZ"}
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { createServer } from "node:http";
|
|
3
|
+
const references = new WeakMap();
|
|
4
|
+
const fileMatchers = new WeakSet();
|
|
5
|
+
export function inputFile(part) {
|
|
6
|
+
function matches(value) {
|
|
7
|
+
if (Array.isArray(value)) {
|
|
8
|
+
return value.some(matches);
|
|
9
|
+
}
|
|
10
|
+
return [...(references.get(part) ?? [])].some(({ reference, workspace_path }) => {
|
|
11
|
+
if (typeof value === "string") {
|
|
12
|
+
return (value === workspace_path ||
|
|
13
|
+
value === reference.file_id ||
|
|
14
|
+
value === reference.download_url);
|
|
15
|
+
}
|
|
16
|
+
return (value !== null &&
|
|
17
|
+
typeof value === "object" &&
|
|
18
|
+
"file_id" in value &&
|
|
19
|
+
"download_url" in value &&
|
|
20
|
+
value.file_id === reference.file_id &&
|
|
21
|
+
value.download_url === reference.download_url);
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
const label = () => `inputFile(${part.filename ?? part.mediaType}: ${[...(references.get(part) ?? [])].map(({ reference }) => reference.file_id).join(", ") || "not sent"})`;
|
|
25
|
+
const matcher = { asymmetricMatch: matches, toString: label, toJSON: label };
|
|
26
|
+
fileMatchers.add(matcher);
|
|
27
|
+
return matcher;
|
|
28
|
+
}
|
|
29
|
+
export function containsInputFile(value) {
|
|
30
|
+
if (value === null || typeof value !== "object") {
|
|
31
|
+
return false;
|
|
32
|
+
}
|
|
33
|
+
return (fileMatchers.has(value) || Object.values(value).some(containsInputFile));
|
|
34
|
+
}
|
|
35
|
+
async function fileBytes(data) {
|
|
36
|
+
if (data instanceof URL) {
|
|
37
|
+
const response = await fetch(data);
|
|
38
|
+
if (!response.ok) {
|
|
39
|
+
throw new Error(`Input file download failed (HTTP ${response.status})`);
|
|
40
|
+
}
|
|
41
|
+
return Buffer.from(await response.arrayBuffer());
|
|
42
|
+
}
|
|
43
|
+
if (typeof data === "string") {
|
|
44
|
+
if (data.startsWith("data:")) {
|
|
45
|
+
return fileBytes(new URL(data));
|
|
46
|
+
}
|
|
47
|
+
return Buffer.from(data, "base64");
|
|
48
|
+
}
|
|
49
|
+
if (data instanceof ArrayBuffer) {
|
|
50
|
+
return Buffer.from(data);
|
|
51
|
+
}
|
|
52
|
+
if (data instanceof Uint8Array) {
|
|
53
|
+
return Buffer.from(data);
|
|
54
|
+
}
|
|
55
|
+
if (data.type === "data") {
|
|
56
|
+
return fileBytes(data.data);
|
|
57
|
+
}
|
|
58
|
+
if (data.type === "url") {
|
|
59
|
+
return fileBytes(new URL(data.url));
|
|
60
|
+
}
|
|
61
|
+
if (data.type === "text" && typeof data.text === "string") {
|
|
62
|
+
return Buffer.from(data.text);
|
|
63
|
+
}
|
|
64
|
+
throw new Error("Input files need bytes, text, or a URL; provider-only file references cannot be downloaded by the eval app");
|
|
65
|
+
}
|
|
66
|
+
export class InputFiles {
|
|
67
|
+
files = new Map();
|
|
68
|
+
parts = new WeakMap();
|
|
69
|
+
paths = new Map();
|
|
70
|
+
server = createServer((request, response) => {
|
|
71
|
+
const file = this.files.get(request.url?.slice(1) ?? "");
|
|
72
|
+
if (!file) {
|
|
73
|
+
response.writeHead(404).end();
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
response.writeHead(200, { "Content-Type": file.reference.mime_type });
|
|
77
|
+
response.end(file.bytes);
|
|
78
|
+
});
|
|
79
|
+
async prepare(content) {
|
|
80
|
+
if (typeof content === "string") {
|
|
81
|
+
return content;
|
|
82
|
+
}
|
|
83
|
+
const prepared = [];
|
|
84
|
+
for (const part of content) {
|
|
85
|
+
if (part.type !== "file") {
|
|
86
|
+
prepared.push(part);
|
|
87
|
+
continue;
|
|
88
|
+
}
|
|
89
|
+
let attachment = this.parts.get(part);
|
|
90
|
+
if (!attachment) {
|
|
91
|
+
const bytes = await fileBytes(part.data);
|
|
92
|
+
if (!this.server.listening) {
|
|
93
|
+
await new Promise((resolve, reject) => {
|
|
94
|
+
this.server.once("error", reject);
|
|
95
|
+
this.server.listen(0, "127.0.0.1", () => {
|
|
96
|
+
this.server.off("error", reject);
|
|
97
|
+
this.server.unref();
|
|
98
|
+
resolve();
|
|
99
|
+
});
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
const address = this.server.address();
|
|
103
|
+
if (!address || typeof address === "string") {
|
|
104
|
+
throw new Error("Input file server did not open a local port");
|
|
105
|
+
}
|
|
106
|
+
const id = randomUUID();
|
|
107
|
+
const file_id = `file_${id}`;
|
|
108
|
+
const reference = {
|
|
109
|
+
file_id,
|
|
110
|
+
download_url: `http://127.0.0.1:${address.port}/${file_id}`,
|
|
111
|
+
mime_type: part.mediaType,
|
|
112
|
+
...(part.filename === undefined ? {} : { file_name: part.filename }),
|
|
113
|
+
};
|
|
114
|
+
const filename = (part.filename ?? "input").split(/[/\\]/).pop() || "input";
|
|
115
|
+
const workspace_path = `/workspace/scratch/${id}/upload/${filename}`;
|
|
116
|
+
attachment = { reference, workspace_path };
|
|
117
|
+
this.files.set(file_id, { reference, bytes });
|
|
118
|
+
this.paths.set(workspace_path, reference);
|
|
119
|
+
this.parts.set(part, attachment);
|
|
120
|
+
const known = references.get(part) ?? new Set();
|
|
121
|
+
known.add(attachment);
|
|
122
|
+
references.set(part, known);
|
|
123
|
+
}
|
|
124
|
+
prepared.push({
|
|
125
|
+
type: "text",
|
|
126
|
+
text: `Attached file: ${JSON.stringify({ uploaded_name: part.filename ?? "input", workspace_path: attachment.workspace_path })}`,
|
|
127
|
+
}, part);
|
|
128
|
+
}
|
|
129
|
+
return prepared;
|
|
130
|
+
}
|
|
131
|
+
rewrite(input, fields) {
|
|
132
|
+
const rewritten = { ...input };
|
|
133
|
+
const resolve = (value) => {
|
|
134
|
+
if (typeof value !== "string") {
|
|
135
|
+
return value;
|
|
136
|
+
}
|
|
137
|
+
const reference = this.paths.get(value);
|
|
138
|
+
if (!reference) {
|
|
139
|
+
throw new Error(`Unknown input file path: ${value}`);
|
|
140
|
+
}
|
|
141
|
+
return reference;
|
|
142
|
+
};
|
|
143
|
+
for (const field of fields) {
|
|
144
|
+
if (field in rewritten) {
|
|
145
|
+
const value = rewritten[field];
|
|
146
|
+
rewritten[field] = Array.isArray(value)
|
|
147
|
+
? value.map(resolve)
|
|
148
|
+
: resolve(value);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
return rewritten;
|
|
152
|
+
}
|
|
153
|
+
async close() {
|
|
154
|
+
if (this.server.listening) {
|
|
155
|
+
await new Promise((resolve, reject) => {
|
|
156
|
+
this.server.close((error) => (error ? reject(error) : resolve()));
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
this.files.clear();
|
|
160
|
+
this.paths.clear();
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
//# sourceMappingURL=input-files.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"input-files.js","sourceRoot":"","sources":["../src/input-files.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAezC,MAAM,UAAU,GAAG,IAAI,OAAO,EAA6B,CAAC;AAC5D,MAAM,YAAY,GAAG,IAAI,OAAO,EAAU,CAAC;AAQ3C,MAAM,UAAU,SAAS,CAAC,IAAc;IACtC,SAAS,OAAO,CAAC,KAAc;QAC7B,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC7B,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAC3C,CAAC,EAAE,SAAS,EAAE,cAAc,EAAE,EAAE,EAAE;YAChC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,OAAO,CACL,KAAK,KAAK,cAAc;oBACxB,KAAK,KAAK,SAAS,CAAC,OAAO;oBAC3B,KAAK,KAAK,SAAS,CAAC,YAAY,CACjC,CAAC;YACJ,CAAC;YACD,OAAO,CACL,KAAK,KAAK,IAAI;gBACd,OAAO,KAAK,KAAK,QAAQ;gBACzB,SAAS,IAAI,KAAK;gBAClB,cAAc,IAAI,KAAK;gBACvB,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,OAAO;gBACnC,KAAK,CAAC,YAAY,KAAK,SAAS,CAAC,YAAY,CAC9C,CAAC;QACJ,CAAC,CACF,CAAC;IACJ,CAAC;IACD,MAAM,KAAK,GAAG,GAAG,EAAE,CACjB,aAAa,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,SAAS,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,UAAU,GAAG,CAAC;IAC3J,MAAM,OAAO,GAAG,EAAE,eAAe,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IAC7E,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC1B,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAc;IAC9C,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAChD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,CACL,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CACxE,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,IAAsB;IAC7C,IAAI,IAAI,YAAY,GAAG,EAAE,CAAC;QACxB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,oCAAoC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;QAC1E,CAAC;QACD,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;IACnD,CAAC;IACD,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC7B,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC7B,OAAO,SAAS,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;QAClC,CAAC;QACD,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACrC,CAAC;IACD,IAAI,IAAI,YAAY,WAAW,EAAE,CAAC;QAChC,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3B,CAAC;IACD,IAAI,IAAI,YAAY,UAAU,EAAE,CAAC;QAC/B,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3B,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QACzB,OAAO,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;QACxB,OAAO,SAAS,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC1D,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IACD,MAAM,IAAI,KAAK,CACb,4GAA4G,CAC7G,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,UAAU;IACJ,KAAK,GAAG,IAAI,GAAG,EAG7B,CAAC;IACa,KAAK,GAAG,IAAI,OAAO,EAAwB,CAAC;IAC5C,KAAK,GAAG,IAAI,GAAG,EAAyB,CAAC;IACzC,MAAM,GAAG,YAAY,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE;QAC3D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACzD,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;YAC9B,OAAO;QACT,CAAC;QACD,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC,CAAC;QACtE,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,KAAK,CAAC,OAAO,CAAC,OAAoB;QAChC,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YAChC,OAAO,OAAO,CAAC;QACjB,CAAC;QACD,MAAM,QAAQ,GAAiC,EAAE,CAAC;QAClD,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YAC3B,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACzB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACpB,SAAS;YACX,CAAC;YACD,IAAI,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACtC,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChB,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACzC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;oBAC3B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;wBAC1C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;wBAClC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,WAAW,EAAE,GAAG,EAAE;4BACtC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;4BACjC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;4BACpB,OAAO,EAAE,CAAC;wBACZ,CAAC,CAAC,CAAC;oBACL,CAAC,CAAC,CAAC;gBACL,CAAC;gBACD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBACtC,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;oBAC5C,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;gBACjE,CAAC;gBACD,MAAM,EAAE,GAAG,UAAU,EAAE,CAAC;gBACxB,MAAM,OAAO,GAAG,QAAQ,EAAE,EAAE,CAAC;gBAC7B,MAAM,SAAS,GAAG;oBAChB,OAAO;oBACP,YAAY,EAAE,oBAAoB,OAAO,CAAC,IAAI,IAAI,OAAO,EAAE;oBAC3D,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,GAAG,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;iBACrE,CAAC;gBACF,MAAM,QAAQ,GACZ,CAAC,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,IAAI,OAAO,CAAC;gBAC7D,MAAM,cAAc,GAAG,sBAAsB,EAAE,WAAW,QAAQ,EAAE,CAAC;gBACrE,UAAU,GAAG,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC;gBAC3C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC9C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;gBAC1C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;gBACjC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,EAAc,CAAC;gBAC5D,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBACtB,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAC9B,CAAC;YACD,QAAQ,CAAC,IAAI,CACX;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,kBAAkB,IAAI,CAAC,SAAS,CAAC,EAAE,aAAa,EAAE,IAAI,CAAC,QAAQ,IAAI,OAAO,EAAE,cAAc,EAAE,UAAU,CAAC,cAAc,EAAE,CAAC,EAAE;aACjI,EACD,IAAI,CACL,CAAC;QACJ,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,OAAO,CAAC,KAA8B,EAAE,MAAgB;QACtD,MAAM,SAAS,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;QAC/B,MAAM,OAAO,GAAG,CAAC,KAAc,EAAW,EAAE;YAC1C,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,OAAO,KAAK,CAAC;YACf,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACxC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,4BAA4B,KAAK,EAAE,CAAC,CAAC;YACvD,CAAC;YACD,OAAO,SAAS,CAAC;QACnB,CAAC,CAAC;QACF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,KAAK,IAAI,SAAS,EAAE,CAAC;gBACvB,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;gBAC/B,SAAS,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;oBACrC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC;oBACpB,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YAC1B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBAC1C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACpE,CAAC,CAAC,CAAC;QACL,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;CACF"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import "vitest";
|
|
2
|
-
import type { LanguageModel } from "ai";
|
|
2
|
+
import type { FilePart, LanguageModel } from "ai";
|
|
3
3
|
import type { ToolInput, ToolNames } from "skybridge/server";
|
|
4
|
+
import { type InputFileMatcher } from "../input-files.js";
|
|
4
5
|
import type { ChatLike } from "./types.js";
|
|
5
6
|
/** What a judge decided about a conversation. */
|
|
6
7
|
export interface Verdict {
|
|
@@ -25,14 +26,20 @@ export interface JudgmentOptions {
|
|
|
25
26
|
*/
|
|
26
27
|
judge?: (judgment: Judgment) => Verdict | Promise<Verdict>;
|
|
27
28
|
}
|
|
29
|
+
type ArgumentMatcher<T> = T | Pick<InputFileMatcher, "asymmetricMatch"> | (T extends readonly (infer Item)[] ? ArgumentMatcher<Item>[] : T extends object ? {
|
|
30
|
+
[Key in keyof T]: ArgumentMatcher<T[Key]>;
|
|
31
|
+
} : never);
|
|
32
|
+
type MatchingArguments<T> = {
|
|
33
|
+
[Key in keyof T]?: ArgumentMatcher<T[Key]>;
|
|
34
|
+
};
|
|
28
35
|
/**
|
|
29
36
|
* Assertions about the tool calls a conversation produced. Kept off vitest's
|
|
30
37
|
* global `Matchers` so they never appear on unrelated `expect()` calls, and so
|
|
31
38
|
* this stays the one place later tool-call helpers are added.
|
|
32
39
|
*/
|
|
33
40
|
export interface ChatMatchers<App> {
|
|
34
|
-
toHaveCalledToolOnce<Name extends ToolNames<App>>(name: Name, args?:
|
|
35
|
-
toHaveCalledToolWith<Name extends ToolNames<App>>(name: Name, args:
|
|
41
|
+
toHaveCalledToolOnce<Name extends ToolNames<App>>(name: Name, args?: MatchingArguments<ToolInput<App, Name>>): void;
|
|
42
|
+
toHaveCalledToolWith<Name extends ToolNames<App>>(name: Name, args: MatchingArguments<ToolInput<App, Name>>): void;
|
|
36
43
|
toNeverHaveCalledTool<Name extends ToolNames<App>>(name: Name): void;
|
|
37
44
|
toHaveFailedToolCall<Name extends ToolNames<App>>(name: Name): void;
|
|
38
45
|
toHaveSaid(text: string | RegExp): void;
|
|
@@ -45,7 +52,9 @@ export interface ChatAssertion<App> extends ChatMatchers<App> {
|
|
|
45
52
|
}
|
|
46
53
|
declare module "vitest" {
|
|
47
54
|
interface ExpectStatic {
|
|
55
|
+
inputFile(part: FilePart): InputFileMatcher;
|
|
48
56
|
chat<App>(chat: ChatLike<App>): ChatAssertion<App>;
|
|
49
57
|
}
|
|
50
58
|
}
|
|
59
|
+
export {};
|
|
51
60
|
//# sourceMappingURL=matchers.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"matchers.d.ts","sourceRoot":"","sources":["../../src/matchers/matchers.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,CAAC;AAChB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"matchers.d.ts","sourceRoot":"","sources":["../../src/matchers/matchers.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,CAAC;AAChB,OAAO,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAClD,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7D,OAAO,EAEL,KAAK,gBAAgB,EAEtB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,QAAQ,EAAkB,MAAM,YAAY,CAAC;AAE3D,iDAAiD;AACjD,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,OAAO,CAAC;IACd,iFAAiF;IACjF,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,kEAAkE;AAClE,MAAM,WAAW,QAAQ;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,2EAA2E;IAC3E,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,uDAAuD;AACvD,MAAM,WAAW,eAAe;IAC9B,4EAA4E;IAC5E,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB;;;;OAIG;IACH,KAAK,CAAC,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAC5D;AAED,KAAK,eAAe,CAAC,CAAC,IAClB,CAAC,GACD,IAAI,CAAC,gBAAgB,EAAE,iBAAiB,CAAC,GACzC,CAAC,CAAC,SAAS,SAAS,CAAC,MAAM,IAAI,CAAC,EAAE,GAC9B,eAAe,CAAC,IAAI,CAAC,EAAE,GACvB,CAAC,SAAS,MAAM,GACd;KAAG,GAAG,IAAI,MAAM,CAAC,GAAG,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;CAAE,GAC7C,KAAK,CAAC,CAAC;AAEjB,KAAK,iBAAiB,CAAC,CAAC,IAAI;KAAG,GAAG,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;CAAE,CAAC;AAE3E;;;;GAIG;AACH,MAAM,WAAW,YAAY,CAAC,GAAG;IAC/B,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAC9C,IAAI,EAAE,IAAI,EACV,IAAI,CAAC,EAAE,iBAAiB,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GAC7C,IAAI,CAAC;IACR,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAC9C,IAAI,EAAE,IAAI,EACV,IAAI,EAAE,iBAAiB,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GAC5C,IAAI,CAAC;IACR,qBAAqB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,CAAC;IACrE,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,CAAC;IACpE,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IACxC,wBAAwB,CACtB,GAAG,KAAK,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,GAAG,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,GAC9C,IAAI,CAAC;IACR,mBAAmB,IAAI,IAAI,CAAC;IAC5B,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5E;AAED,MAAM,WAAW,aAAa,CAAC,GAAG,CAAE,SAAQ,YAAY,CAAC,GAAG,CAAC;IAC3D,GAAG,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC;CACxB;AAED,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,YAAY;QACpB,SAAS,CAAC,IAAI,EAAE,QAAQ,GAAG,gBAAgB,CAAC;QAC5C,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;KACpD;CACF"}
|
|
@@ -1,10 +1,19 @@
|
|
|
1
1
|
import "vitest";
|
|
2
2
|
import { expect } from "vitest";
|
|
3
|
+
import { containsInputFile, inputFile, } from "../input-files.js";
|
|
3
4
|
function calls(chat) {
|
|
4
5
|
return chat.turns
|
|
5
6
|
.filter((turn) => turn.role === "tool")
|
|
6
7
|
.map((turn) => turn.call);
|
|
7
8
|
}
|
|
9
|
+
function assertFileParameters(chat, name, args = {}) {
|
|
10
|
+
const fields = chat.fileParams?.get(name) ?? [];
|
|
11
|
+
for (const [field, value] of Object.entries(args)) {
|
|
12
|
+
if (containsInputFile(value) && !fields.includes(field)) {
|
|
13
|
+
throw new Error(`expect.inputFile cannot be used for "${name}.${field}": "${field}" is not listed in openai/fileParams`);
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
}
|
|
8
17
|
function spokenTurns(chat) {
|
|
9
18
|
return chat.turns
|
|
10
19
|
.filter((turn) => turn.role === "assistant")
|
|
@@ -104,6 +113,7 @@ async function modelVerdict(chat, criteria, model) {
|
|
|
104
113
|
}
|
|
105
114
|
expect.extend({
|
|
106
115
|
toHaveCalledToolOnce(received, name, args) {
|
|
116
|
+
assertFileParameters(received, name, args);
|
|
107
117
|
const matching = acceptedCalls(received, name);
|
|
108
118
|
const argsMatch = args === undefined ||
|
|
109
119
|
(matching.length === 1 &&
|
|
@@ -119,6 +129,7 @@ expect.extend({
|
|
|
119
129
|
};
|
|
120
130
|
},
|
|
121
131
|
toHaveCalledToolWith(received, name, args) {
|
|
132
|
+
assertFileParameters(received, name, args);
|
|
122
133
|
const pass = acceptedCalls(received, name).some((call) => this.equals(call.arguments, expect.objectContaining(args)));
|
|
123
134
|
return {
|
|
124
135
|
pass,
|
|
@@ -215,4 +226,5 @@ judge: ${verdict.pass ? "PASS" : "FAIL"}${verdict.reasoning === undefined ? "" :
|
|
|
215
226
|
},
|
|
216
227
|
});
|
|
217
228
|
expect.chat = (chat) => expect(chat);
|
|
229
|
+
expect.inputFile = inputFile;
|
|
218
230
|
//# sourceMappingURL=matchers.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"matchers.js","sourceRoot":"","sources":["../../src/matchers/matchers.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,CAAC;AAGhB,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;
|
|
1
|
+
{"version":3,"file":"matchers.js","sourceRoot":"","sources":["../../src/matchers/matchers.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,CAAC;AAGhB,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EACL,iBAAiB,EAEjB,SAAS,GACV,MAAM,mBAAmB,CAAC;AA2E3B,SAAS,KAAK,CAAC,IAAuB;IACpC,OAAO,IAAI,CAAC,KAAK;SACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC;SACtC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC;AAED,SAAS,oBAAoB,CAC3B,IAAuB,EACvB,IAAY,EACZ,OAAgC,EAAE;IAElC,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IAChD,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QAClD,IAAI,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACxD,MAAM,IAAI,KAAK,CACb,wCAAwC,IAAI,IAAI,KAAK,OAAO,KAAK,sCAAsC,CACxG,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,IAAuB;IAC1C,OAAO,IAAI,CAAC,KAAK;SACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,WAAW,CAAC;SAC3C,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC;AAED,SAAS,cAAc,CACrB,IAAuB,EACvB,IAAY;IAEZ,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,WAAW,CAAC,IAAuB;IAC1C,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;AACjE,CAAC;AAED,SAAS,aAAa,CACpB,IAAuB,EACvB,IAAY;IAEZ,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;AACxE,CAAC;AAED,SAAS,QAAQ,CAAC,IAAuB;IACvC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IACzB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,wBAAwB,CAAC;IAClC,CAAC;IACD,OAAO,IAAI;SACR,GAAG,CACF,CAAC,IAAuB,EAAE,KAAa,EAAE,EAAE,CACzC,KAAK,KAAK,GAAG,CAAC,KAAK,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,cAAc,IAAI,CAAC,MAAM,GAAG,EAAE,CACnI;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AAC1C,CAAC;AAED,SAAS,IAAI,CAAC,IAAuB,EAAE,IAAqB;IAC1D,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CACrC,OAAO,IAAI,KAAK,QAAQ;QACtB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;QACrE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CACpB,CAAC;AACJ,CAAC;AAED,SAAS,MAAM,CAAC,IAAuB;IACrC,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,gCAAgC,CAAC;IAC1C,CAAC;IACD,OAAO,KAAK;SACT,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,GAAG,CAAC,KAAK,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;SACzD,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,SAAS,KAAK,CAAC,IAAY;IACzB,OAAO,IAAI,CAAC,OAAO,CAAC,uBAAuB,EAAE,QAAQ,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,YAAY,GAAG;;;6KAGwJ,CAAC;AAE9K,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAE9B,SAAS,QAAQ,CAAC,IAAmB;IACnC,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QACzB,OAAO,GAAG,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;IAC7C,CAAC;IACD,MAAM,OAAO,GACX,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS;QAC5B,CAAC,CAAC,OAAO,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,EAAE;QACxD,CAAC,CAAC,aAAa,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;IACvC,OAAO,SAAS,KAAK,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,OAAO,EAAE,CAAC,EAAE,CAAC;AACxG,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,IAAI,CAAC,MAAM,GAAG,gBAAgB;QACnC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC,eAAe;QACnD,CAAC,CAAC,IAAI,CAAC;AACX,CAAC;AAED,SAAS,aAAa,CAAC,IAAuB;IAC5C,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,6BAA6B,CAAC;IACvC,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC7C,CAAC;AAED,SAAS,MAAM,CAAC,IAAuB,EAAE,OAAe;IACtD,OAAO,GAAG,OAAO;;;EAGjB,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;AACnB,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,IAAuB,EACvB,QAAgB,EAChB,KAAqB;IAErB,MAAM,EAAE,cAAc,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;IAC1D,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,cAAc,CAAC;QACtC,KAAK,EAAE,KAAK,IAAI,IAAI,CAAC,KAAK;QAC1B,WAAW,EAAE,CAAC;QACd,MAAM,EAAE,YAAY;QACpB,MAAM,EAAE,cAAc,QAAQ,uBAAuB,aAAa,CAAC,IAAI,CAAC,mBAAmB;QAC3F,MAAM,EAAE,UAAU,CAAkC;YAClD,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;gBACzB,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;aAC9B;YACD,QAAQ,EAAE,CAAC,MAAM,EAAE,WAAW,CAAC;YAC/B,oBAAoB,EAAE,KAAK;SAC5B,CAAC;KACH,CAAC,CAAC;IACH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,MAAM,CAAC;IACZ,oBAAoB,CAClB,QAA2B,EAC3B,IAAY,EACZ,IAA8B;QAE9B,oBAAoB,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;QAC3C,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC/C,MAAM,SAAS,GACb,IAAI,KAAK,SAAS;YAClB,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACpB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxE,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC;QAEhD,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,aAAa,IAAI,kCAAkC;gBACrD,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;oBACrB,CAAC,CAAC,aAAa,IAAI,4BAA4B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,4BAA4B,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG;oBACxI,CAAC,CAAC,aAAa,IAAI,6DAA6D,QAAQ,CAAC,MAAM,wBAAwB,CAC5H;SACJ,CAAC;IACJ,CAAC;IAED,oBAAoB,CAClB,QAA2B,EAC3B,IAAY,EACZ,IAA6B;QAE7B,oBAAoB,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;QAC3C,MAAM,IAAI,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CACvD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAC3D,CAAC;QAEF,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,wBAAwB,IAAI,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG;gBAC/D,CAAC,CAAC,aAAa,IAAI,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CACpE;SACJ,CAAC;IACJ,CAAC;IAED,oBAAoB,CAAC,QAA2B,EAAE,IAAY;QAC5D,MAAM,SAAS,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;QAErE,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC;YACvB,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,MAAM,CAAC,MAAM,GAAG,CAAC;gBACf,CAAC,CAAC,2BAA2B,IAAI,eAAe;gBAChD,CAAC,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;oBACtB,CAAC,CAAC,aAAa,IAAI,mDAAmD;oBACtE,CAAC,CAAC,aAAa,IAAI,sBAAsB,SAAS,CAAC,MAAM,mBAAmB,CACjF;SACJ,CAAC;IACJ,CAAC;IAED,UAAU,CAAC,QAA2B,EAAE,IAAqB;QAC3D,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAClC,MAAM,MAAM,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAErE,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,GACE,IAAI;gBACF,CAAC,CAAC,qCAAqC,MAAM,GAAG;gBAChD,CAAC,CAAC,iCAAiC,MAAM,GAC7C;;;EAGN,MAAM,CAAC,QAAQ,CAAC,EAAE;SACf,CAAC;IACJ,CAAC;IAED,mBAAmB,CAAC,QAA2B;QAC7C,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;QAErC,OAAO;YACL,IAAI,EAAE,KAAK,KAAK,CAAC;YACjB,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,KAAK,KAAK,CAAC;gBACT,CAAC,CAAC,+CAA+C;gBACjD,CAAC,CAAC,wDAAwD,KAAK,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,GAAG,CACvG;SACJ,CAAC;IACJ,CAAC;IAED,wBAAwB,CAAC,QAA2B,EAAE,GAAG,KAAe;QACtE,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;YACzC,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;gBACzC,OAAO,IAAI,CAAC,CAAC;YACf,CAAC;YACD,IAAI,OAAO,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;gBAC7B,MAAM;YACR,CAAC;QACH,CAAC;QACD,MAAM,IAAI,GAAG,OAAO,KAAK,KAAK,CAAC,MAAM,CAAC;QACtC,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAE/D,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,YAAY,MAAM,kCAAkC;gBACtD,CAAC,CAAC,YAAY,MAAM,wBAAwB,KAAK,CAAC,OAAO,CAAC,eAAe,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CACpI;SACJ,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,cAAc,CAClB,QAA2B,EAC3B,QAAgB,EAChB,OAAyB;QAEzB,IAAI,OAAgB,CAAC;QACrB,IAAI,CAAC;YACH,OAAO,GAAG,OAAO,EAAE,KAAK;gBACtB,CAAC,CAAC,MAAM,OAAO,CAAC,KAAK,CAAC;oBAClB,QAAQ;oBACR,UAAU,EAAE,aAAa,CAAC,QAAQ,CAAC;iBACpC,CAAC;gBACJ,CAAC,CAAC,MAAM,YAAY,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;YAE3D,IAAI,OAAO,OAAO,EAAE,IAAI,KAAK,SAAS,EAAE,CAAC;gBACvC,MAAM,IAAI,KAAK,CACb,sBAAsB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,6CAA6C,CAC3F,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,sBAAsB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC/E,CAAC;QACJ,CAAC;QAED,OAAO;YACL,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,OAAO,EAAE,GAAG,EAAE,CACZ,6BAA6B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;KAC1D,QAAQ;;SAEJ,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,OAAO,CAAC,SAAS,EAAE,EAAE;SACxG,CAAC;IACJ,CAAC;IAED,qBAAqB,CAAC,QAA2B,EAAE,IAAY;QAC7D,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAEhD,OAAO;YACL,IAAI,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC;YAC3B,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACnB,CAAC,CAAC,aAAa,IAAI,+BAA+B;gBAClD,CAAC,CAAC,aAAa,IAAI,2CAA2C,QAAQ,CAAC,MAAM,SAAS,CACzF;SACJ,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,IAAI,GAAG,CAAM,IAAmB,EAAsB,EAAE,CAC7D,MAAM,CAAC,IAAI,CAAkC,CAAC;AAChD,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC"}
|
package/dist/matchers/types.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/matchers/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACxC,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7D;;;;GAIG;AACH,MAAM,MAAM,QAAQ,CAAC,GAAG,IAAI;KACzB,IAAI,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG;QACxB,IAAI,EAAE,IAAI,CAAC;QACX,SAAS,EAAE,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAChC,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB;CACF,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;AAElB;;;;GAIG;AACH,MAAM,MAAM,IAAI,CAAC,GAAG,IAChB;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAC9B;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACnC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;IAAC,MAAM,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC;AAE5D,yDAAyD;AACzD,MAAM,WAAW,QAAQ,CAAC,GAAG;IAC3B,4EAA4E;IAC5E,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/matchers/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACxC,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7D;;;;GAIG;AACH,MAAM,MAAM,QAAQ,CAAC,GAAG,IAAI;KACzB,IAAI,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG;QACxB,IAAI,EAAE,IAAI,CAAC;QACX,SAAS,EAAE,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAChC,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB;CACF,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;AAElB;;;;GAIG;AACH,MAAM,MAAM,IAAI,CAAC,GAAG,IAChB;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAC9B;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACnC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;IAAC,MAAM,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC;AAE5D,yDAAyD;AACzD,MAAM,WAAW,QAAQ,CAAC,GAAG;IAC3B,4EAA4E;IAC5E,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;IAC9B,QAAQ,CAAC,UAAU,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC;CAC9D;AAED,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,eAAe;QACvB,cAAc,EAAE,YAAY,GAAG,SAAS,CAAC;KAC1C;CACF"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@skybridge/test",
|
|
3
|
-
"version": "0.0.0-dev.
|
|
4
|
-
"description": "Eval runner for Skybridge MCP
|
|
3
|
+
"version": "0.0.0-dev.6e30880",
|
|
4
|
+
"description": "Eval runner for Skybridge MCP Apps: does a prompt reach the right tool with the right arguments",
|
|
5
5
|
"license": "ISC",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"repository": {
|
|
@@ -37,8 +37,8 @@
|
|
|
37
37
|
"typescript": "^5.9.3",
|
|
38
38
|
"vitest": "^4.1.10",
|
|
39
39
|
"zod": "^4.4.3",
|
|
40
|
-
"@skybridge/vite-plugin": "0.0.0-dev.
|
|
41
|
-
"skybridge": "0.0.0-dev.
|
|
40
|
+
"@skybridge/vite-plugin": "0.0.0-dev.6e30880",
|
|
41
|
+
"skybridge": "0.0.0-dev.6e30880"
|
|
42
42
|
},
|
|
43
43
|
"engines": {
|
|
44
44
|
"node": ">=22.23.1"
|