@skybridge/test 0.0.0-dev.b4419f2 → 0.0.0-dev.b5c271e
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -1
- package/dist/chat.d.ts +5 -2
- package/dist/chat.d.ts.map +1 -1
- package/dist/chat.js +103 -16
- package/dist/chat.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/input-files.d.ts +18 -0
- package/dist/input-files.d.ts.map +1 -0
- package/dist/input-files.js +145 -0
- package/dist/input-files.js.map +1 -0
- package/dist/matchers/index.d.ts +1 -1
- package/dist/matchers/index.d.ts.map +1 -1
- package/dist/matchers/matchers.d.ts +30 -3
- package/dist/matchers/matchers.d.ts.map +1 -1
- package/dist/matchers/matchers.js +31 -19
- package/dist/matchers/matchers.js.map +1 -1
- package/package.json +4 -4
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# @skybridge/test
|
|
2
2
|
|
|
3
|
-
Evals for [Skybridge](https://skybridge.tech) MCP
|
|
3
|
+
Evals for [Skybridge](https://skybridge.tech) MCP Apps: they answer whether a
|
|
4
4
|
prompt reaches the right tool with the right arguments, which no unit test can
|
|
5
5
|
check.
|
|
6
6
|
|
|
@@ -82,6 +82,41 @@ from, so a project can set them once instead of per scenario.
|
|
|
82
82
|
skybridge({ evals: { temperature: 0, maxSteps: 6 } });
|
|
83
83
|
```
|
|
84
84
|
|
|
85
|
+
## Input files
|
|
86
|
+
|
|
87
|
+
`chat.send` also accepts AI SDK `UserContent`. Attach a `FilePart` and use the
|
|
88
|
+
same object in `expect.inputFile` to check which file a tool received:
|
|
89
|
+
|
|
90
|
+
```ts
|
|
91
|
+
import { readFile } from "node:fs/promises";
|
|
92
|
+
import type { FilePart } from "ai";
|
|
93
|
+
|
|
94
|
+
const portrait: FilePart = {
|
|
95
|
+
type: "file",
|
|
96
|
+
mediaType: "image/png",
|
|
97
|
+
filename: "photo.png",
|
|
98
|
+
data: await readFile(new URL("./fixtures/photo.png", import.meta.url)),
|
|
99
|
+
};
|
|
100
|
+
|
|
101
|
+
await chat.send([{ type: "text", text: "Resize this photo" }, portrait]);
|
|
102
|
+
|
|
103
|
+
expect.chat(chat).toHaveCalledToolWith("image_crop_and_resize", {
|
|
104
|
+
imageURI: expect.inputFile(portrait),
|
|
105
|
+
});
|
|
106
|
+
expect.chat(chat).toHaveCalledToolOnce("asset_openai_file_upload", {
|
|
107
|
+
files: [expect.inputFile(portrait)],
|
|
108
|
+
});
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
The runner gives the model a file id and a temporary local download URL alongside
|
|
112
|
+
the content. For `_meta["openai/fileParams"]` fields, it turns ids into OpenAI
|
|
113
|
+
file objects before the handler or stub runs. Other fields keep the model's
|
|
114
|
+
original input. Downloads serve the attached bytes until the conversation closes.
|
|
115
|
+
|
|
116
|
+
`expect.inputFile(part)` accepts that file's id, URL, file object, or an array
|
|
117
|
+
containing one of them. Distinct file objects never match each other, even with
|
|
118
|
+
identical bytes. It works with both tool argument matchers and their `.not` forms.
|
|
119
|
+
|
|
85
120
|
## Matchers
|
|
86
121
|
|
|
87
122
|
Assertions on what the assistant said work the same way, though model prose
|
package/dist/chat.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type LanguageModel } from "ai";
|
|
1
|
+
import { type LanguageModel, type UserContent } from "ai";
|
|
2
2
|
import type { ToolCall, Turn } from "./matchers/types.js";
|
|
3
3
|
interface HostConfig {
|
|
4
4
|
model: LanguageModel;
|
|
@@ -33,6 +33,9 @@ export declare class Chat<App = unknown> {
|
|
|
33
33
|
get assistantTurns(): string[];
|
|
34
34
|
private readonly messages;
|
|
35
35
|
private readonly failures;
|
|
36
|
+
private readonly inputs;
|
|
37
|
+
private readonly inputFiles;
|
|
38
|
+
private readonly schemaValidator;
|
|
36
39
|
private readonly client;
|
|
37
40
|
private readonly transport;
|
|
38
41
|
private readonly host;
|
|
@@ -43,7 +46,7 @@ export declare class Chat<App = unknown> {
|
|
|
43
46
|
private constructor();
|
|
44
47
|
static open<App>(host: HostConfig, fetchImpl: TransportFetch, stubs: StubMap): Promise<Chat<App>>;
|
|
45
48
|
/** Takes a turn. Assert on the result with `expect.chat`. */
|
|
46
|
-
send(
|
|
49
|
+
send(content: UserContent): Promise<void>;
|
|
47
50
|
close(): Promise<void>;
|
|
48
51
|
private failureFor;
|
|
49
52
|
private toolSet;
|
package/dist/chat.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAMA,OAAO,EAKL,KAAK,aAAa,EAGlB,KAAK,WAAW,EACjB,MAAM,IAAI,CAAC;AAEZ,OAAO,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AAE1D,UAAU,UAAU;IAClB,KAAK,EAAE,aAAa,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;GAKG;AACH,MAAM,MAAM,OAAO,GAAG,MAAM,CAC1B,MAAM,EACN,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,OAAO,CAC3C,CAAC;AAsDF;;;GAGG;AACH,MAAM,MAAM,cAAc,GAAG,CAC3B,GAAG,EAAE,MAAM,GAAG,GAAG,EACjB,IAAI,CAAC,EAAE,WAAW,KACf,OAAO,CAAC,QAAQ,CAAC,CAAC;AAEvB;;;GAGG;AACH,qBAAa,IAAI,CAAC,GAAG,GAAG,OAAO;IAC7B,4EAA4E;IAC5E,SAAiB,IAAI,EAAE,GAAG,CAAC;IAE3B,0DAA0D;IAC1D,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAM;IAEjC,mEAAmE;IACnE,IAAI,SAAS,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE,CAI/B;IAED,+EAA+E;IAC/E,IAAI,cAAc,IAAI,MAAM,EAAE,CAI7B;IAED,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA6B;IACtD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA8C;IACrE,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAoB;IAC/C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAgC;IAChE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAa;IAClC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA+B;IACrD,OAAO,CAAC,KAAK,CAAc;IAE3B,mEAAmE;IACnE,IAAI,KAAK,IAAI,aAAa,CAEzB;IAED,OAAO;WAYM,IAAI,CAAC,GAAG,EACnB,IAAI,EAAE,UAAU,EAChB,SAAS,EAAE,cAAc,EACzB,KAAK,EAAE,OAAO,GACb,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IA0BrB,6DAA6D;IACvD,IAAI,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAiDzC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ5B,OAAO,CAAC,UAAU;IAalB,OAAO,CAAC,OAAO;CAoFhB"}
|
package/dist/chat.js
CHANGED
|
@@ -1,18 +1,47 @@
|
|
|
1
1
|
import { Client, StreamableHTTPClientTransport, } from "@modelcontextprotocol/client";
|
|
2
|
+
import { AjvJsonSchemaValidator } from "@modelcontextprotocol/client/validators/ajv";
|
|
2
3
|
import { dynamicTool, generateText, jsonSchema, stepCountIs, } from "ai";
|
|
4
|
+
import { InputFiles } from "./input-files.js";
|
|
3
5
|
const DEFAULT_MAX_STEPS = 8;
|
|
4
6
|
const IN_PROCESS_URL = "http://in-process.skybridge.test/mcp";
|
|
7
|
+
function fileInputSchema(schema) {
|
|
8
|
+
if (typeof schema === "boolean") {
|
|
9
|
+
return schema;
|
|
10
|
+
}
|
|
11
|
+
if (schema.anyOf) {
|
|
12
|
+
return { ...schema, anyOf: schema.anyOf.map(fileInputSchema) };
|
|
13
|
+
}
|
|
14
|
+
if (schema.oneOf) {
|
|
15
|
+
return { ...schema, oneOf: schema.oneOf.map(fileInputSchema) };
|
|
16
|
+
}
|
|
17
|
+
if (schema.type === "array") {
|
|
18
|
+
return {
|
|
19
|
+
...schema,
|
|
20
|
+
items: Array.isArray(schema.items)
|
|
21
|
+
? schema.items.map(fileInputSchema)
|
|
22
|
+
: schema.items === undefined
|
|
23
|
+
? undefined
|
|
24
|
+
: fileInputSchema(schema.items),
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
if (schema.type === "object" || schema.$ref) {
|
|
28
|
+
return { anyOf: [schema, { type: "string" }] };
|
|
29
|
+
}
|
|
30
|
+
return schema;
|
|
31
|
+
}
|
|
5
32
|
/**
|
|
6
33
|
* Dress a stubbed value the way the server dresses a real one, so the model
|
|
7
34
|
* reads the same envelope under a stub as it does live.
|
|
8
35
|
*/
|
|
9
36
|
function asToolResult(value) {
|
|
10
|
-
return
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
37
|
+
return {
|
|
38
|
+
content: [
|
|
39
|
+
{
|
|
40
|
+
type: "text",
|
|
41
|
+
text: typeof value === "string" ? value : JSON.stringify(value),
|
|
42
|
+
},
|
|
43
|
+
],
|
|
44
|
+
};
|
|
16
45
|
}
|
|
17
46
|
function outputOf(results, toolCallId) {
|
|
18
47
|
return results.find((result) => result.toolCallId === toolCallId)?.output;
|
|
@@ -38,6 +67,9 @@ export class Chat {
|
|
|
38
67
|
}
|
|
39
68
|
messages = [];
|
|
40
69
|
failures = new Map();
|
|
70
|
+
inputs = new Map();
|
|
71
|
+
inputFiles = new InputFiles();
|
|
72
|
+
schemaValidator = new AjvJsonSchemaValidator();
|
|
41
73
|
client;
|
|
42
74
|
transport;
|
|
43
75
|
host;
|
|
@@ -73,9 +105,20 @@ export class Chat {
|
|
|
73
105
|
return chat;
|
|
74
106
|
}
|
|
75
107
|
/** Takes a turn. Assert on the result with `expect.chat`. */
|
|
76
|
-
async send(
|
|
77
|
-
this.
|
|
78
|
-
this.
|
|
108
|
+
async send(content) {
|
|
109
|
+
const prepared = await this.inputFiles.prepare(content);
|
|
110
|
+
this.inputs.clear();
|
|
111
|
+
this.failures.clear();
|
|
112
|
+
this.messages.push({ role: "user", content: prepared });
|
|
113
|
+
this.turns.push({
|
|
114
|
+
role: "user",
|
|
115
|
+
text: typeof prepared === "string"
|
|
116
|
+
? prepared
|
|
117
|
+
: prepared
|
|
118
|
+
.filter((part) => part.type === "text")
|
|
119
|
+
.map((part) => part.text)
|
|
120
|
+
.join("\n"),
|
|
121
|
+
});
|
|
79
122
|
const result = await generateText({
|
|
80
123
|
model: this.host.model,
|
|
81
124
|
temperature: this.host.temperature ?? 0,
|
|
@@ -95,7 +138,8 @@ export class Chat {
|
|
|
95
138
|
role: "tool",
|
|
96
139
|
call: {
|
|
97
140
|
name: call.toolName,
|
|
98
|
-
arguments: (call.
|
|
141
|
+
arguments: this.inputs.get(call.toolCallId) ??
|
|
142
|
+
(call.input ?? {}),
|
|
99
143
|
...this.failureFor(call),
|
|
100
144
|
},
|
|
101
145
|
result: outputOf(step.toolResults, call.toolCallId),
|
|
@@ -105,7 +149,12 @@ export class Chat {
|
|
|
105
149
|
this.messages.push(...result.response.messages);
|
|
106
150
|
}
|
|
107
151
|
async close() {
|
|
108
|
-
|
|
152
|
+
try {
|
|
153
|
+
await this.transport.close();
|
|
154
|
+
}
|
|
155
|
+
finally {
|
|
156
|
+
await this.inputFiles.close();
|
|
157
|
+
}
|
|
109
158
|
}
|
|
110
159
|
failureFor(call) {
|
|
111
160
|
const reported = this.failures.get(call.toolCallId);
|
|
@@ -120,13 +169,48 @@ export class Chat {
|
|
|
120
169
|
toolSet() {
|
|
121
170
|
const entries = this.tools.map((definition) => {
|
|
122
171
|
const { $schema: _ignored, ...parameters } = definition.inputSchema;
|
|
172
|
+
const fileParams = definition._meta?.["openai/fileParams"];
|
|
173
|
+
const fields = Array.isArray(fileParams)
|
|
174
|
+
? fileParams.filter((field) => typeof field === "string")
|
|
175
|
+
: [];
|
|
176
|
+
const properties = { ...parameters.properties };
|
|
177
|
+
for (const field of fields) {
|
|
178
|
+
if (field in properties) {
|
|
179
|
+
properties[field] = fileInputSchema(properties[field]);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
const inputSchema = { ...parameters, properties };
|
|
183
|
+
const validate = fields.length === 0
|
|
184
|
+
? undefined
|
|
185
|
+
: this.schemaValidator.getValidator({
|
|
186
|
+
...definition.inputSchema,
|
|
187
|
+
properties,
|
|
188
|
+
});
|
|
123
189
|
return [
|
|
124
190
|
definition.name,
|
|
125
191
|
dynamicTool({
|
|
126
|
-
description: definition.description ?? "",
|
|
127
|
-
inputSchema: jsonSchema(
|
|
192
|
+
description: `${definition.description ?? ""}${fields.length === 0 ? "" : `\nFor openai/fileParams fields (${fields.join(", ")}), pass attached file_ids in place of file objects, keeping each field's single-file or array shape; the host supplies the file objects.`}`,
|
|
193
|
+
inputSchema: jsonSchema(inputSchema, {
|
|
194
|
+
validate: validate
|
|
195
|
+
? (input) => {
|
|
196
|
+
const result = validate(input);
|
|
197
|
+
return result.valid
|
|
198
|
+
? { success: true, value: result.data }
|
|
199
|
+
: { success: false, error: new Error(result.errorMessage) };
|
|
200
|
+
}
|
|
201
|
+
: undefined,
|
|
202
|
+
}),
|
|
128
203
|
execute: async (input, { toolCallId }) => {
|
|
129
|
-
|
|
204
|
+
let args;
|
|
205
|
+
try {
|
|
206
|
+
args = this.inputFiles.rewrite((input ?? {}), fields);
|
|
207
|
+
}
|
|
208
|
+
catch (error) {
|
|
209
|
+
this.failures.set(toolCallId, String(error));
|
|
210
|
+
throw error;
|
|
211
|
+
}
|
|
212
|
+
this.inputs.set(toolCallId, args);
|
|
213
|
+
const stubbed = await this.stubs.get(definition.name)?.(args);
|
|
130
214
|
if (stubbed !== undefined) {
|
|
131
215
|
return asToolResult(stubbed);
|
|
132
216
|
}
|
|
@@ -134,7 +218,7 @@ export class Chat {
|
|
|
134
218
|
try {
|
|
135
219
|
result = await this.client.callTool({
|
|
136
220
|
name: definition.name,
|
|
137
|
-
arguments:
|
|
221
|
+
arguments: args,
|
|
138
222
|
});
|
|
139
223
|
}
|
|
140
224
|
catch (error) {
|
|
@@ -144,7 +228,10 @@ export class Chat {
|
|
|
144
228
|
if (result.isError === true) {
|
|
145
229
|
this.failures.set(toolCallId, `the tool returned an error: ${JSON.stringify(result.content)}`);
|
|
146
230
|
}
|
|
147
|
-
return
|
|
231
|
+
return {
|
|
232
|
+
content: result.content,
|
|
233
|
+
structuredContent: result.structuredContent,
|
|
234
|
+
};
|
|
148
235
|
},
|
|
149
236
|
}),
|
|
150
237
|
];
|
package/dist/chat.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,MAAM,EACN,6BAA6B,GAE9B,MAAM,8BAA8B,CAAC;AACtC,OAAO,EACL,WAAW,EACX,YAAY,
|
|
1
|
+
{"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,MAAM,EACN,6BAA6B,GAE9B,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAAE,sBAAsB,EAAE,MAAM,6CAA6C,CAAC;AACrF,OAAO,EACL,WAAW,EACX,YAAY,EAEZ,UAAU,EAGV,WAAW,GAEZ,MAAM,IAAI,CAAC;AACZ,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAqB9C,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B,MAAM,cAAc,GAAG,sCAAsC,CAAC;AAE9D,SAAS,eAAe,CAAC,MAA6B;IACpD,IAAI,OAAO,MAAM,KAAK,SAAS,EAAE,CAAC;QAChC,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,OAAO,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,CAAC;IACjE,CAAC;IACD,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,OAAO,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,CAAC;IACjE,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC5B,OAAO;YACL,GAAG,MAAM;YACT,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;gBAChC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,eAAe,CAAC;gBACnC,CAAC,CAAC,MAAM,CAAC,KAAK,KAAK,SAAS;oBAC1B,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC;SACpC,CAAC;IACJ,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QAC5C,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;IACjD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO;QACL,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;aAChE;SACF;KACF,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CACf,OAAmD,EACnD,UAAkB;IAElB,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,KAAK,UAAU,CAAC,EAAE,MAAM,CAAC;AAC5E,CAAC;AAWD;;;GAGG;AACH,MAAM,OAAO,IAAI;IAIf,0DAA0D;IACjD,KAAK,GAAgB,EAAE,CAAC;IAEjC,mEAAmE;IACnE,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,KAAK;aACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC;aACtC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAED,+EAA+E;IAC/E,IAAI,cAAc;QAChB,OAAO,IAAI,CAAC,KAAK;aACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,WAAW,CAAC;aAC3C,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAEgB,QAAQ,GAAmB,EAAE,CAAC;IAC9B,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrC,MAAM,GAAG,IAAI,GAAG,EAAmC,CAAC;IACpD,UAAU,GAAG,IAAI,UAAU,EAAE,CAAC;IAC9B,eAAe,GAAG,IAAI,sBAAsB,EAAE,CAAC;IAC/C,MAAM,CAAS;IACf,SAAS,CAAgC;IACzC,IAAI,CAAa;IACjB,KAAK,CAA+B;IAC7C,KAAK,GAAW,EAAE,CAAC;IAE3B,mEAAmE;IACnE,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;IACzB,CAAC;IAED,YACE,MAAc,EACd,SAAwC,EACxC,IAAgB,EAChB,KAAc;QAEd,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,IAAI,CACf,IAAgB,EAChB,SAAyB,EACzB,KAAc;QAEd,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,IAAI,6BAA6B,CACjD,IAAI,GAAG,CAAC,cAAc,CAAC,EACvB,EAAE,KAAK,EAAE,SAAS,EAAE,CACrB,CAAC;QACF,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAM,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;QAC3D,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;YACnB,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAC5C,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,CACpD,CAAC;YACF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,IAAI,KAAK,CACb,iCAAiC,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,+BAA+B,CACnH,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YACxB,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,6DAA6D;IAC7D,KAAK,CAAC,IAAI,CAAC,OAAoB;QAC7B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACxD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;QACxD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;YACd,IAAI,EAAE,MAAM;YACZ,IAAI,EACF,OAAO,QAAQ,KAAK,QAAQ;gBAC1B,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,QAAQ;qBACL,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC;qBACtC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;qBACxB,IAAI,CAAC,IAAI,CAAC;SACpB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;YACtB,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC;YACvC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,KAAK,SAAS;gBACtC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE;YACrB,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;SAC/D,CAAC,CAAC;QAEH,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,IAAI,CAAC,IAAI,KAAK,EAAE,EAAE,CAAC;gBACrB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YAC1D,CAAC;YACD,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAClC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;oBACd,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE;wBACJ,IAAI,EAAE,IAAI,CAAC,QAAQ;wBACnB,SAAS,EACP,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC;4BAC/B,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAA6B;wBACjD,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;qBACR;oBAClB,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,UAAU,CAAC;iBACpD,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QAC/B,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QAChC,CAAC;IACH,CAAC;IAEO,UAAU,CAAC,IAA+C;QAGhE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QAC9B,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC1B,OAAO,EAAE,MAAM,EAAE,2CAA2C,EAAE,CAAC;QACjE,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAEO,OAAO;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;YAC5C,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,UAAU,EAAE,GAAG,UAAU,CAAC,WAAW,CAAC;YACpE,MAAM,UAAU,GAAG,UAAU,CAAC,KAAK,EAAE,CAAC,mBAAmB,CAAC,CAAC;YAC3D,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC;gBACtC,CAAC,CAAC,UAAU,CAAC,MAAM,CACf,CAAC,KAAK,EAAmB,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CACtD;gBACH,CAAC,CAAC,EAAE,CAAC;YACP,MAAM,UAAU,GAAG,EAAE,GAAG,UAAU,CAAC,UAAU,EAE5C,CAAC;YACF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,IAAI,KAAK,IAAI,UAAU,EAAE,CAAC;oBACxB,UAAU,CAAC,KAAK,CAAC,GAAG,eAAe,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;gBACzD,CAAC;YACH,CAAC;YACD,MAAM,WAAW,GAAG,EAAE,GAAG,UAAU,EAAE,UAAU,EAAE,CAAC;YAClD,MAAM,QAAQ,GACZ,MAAM,CAAC,MAAM,KAAK,CAAC;gBACjB,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,YAAY,CAAU;oBACzC,GAAG,UAAU,CAAC,WAAW;oBACzB,UAAU;iBAC8C,CAAC,CAAC;YAClE,OAAO;gBACL,UAAU,CAAC,IAAI;gBACf,WAAW,CAAC;oBACV,WAAW,EAAE,GAAG,UAAU,CAAC,WAAW,IAAI,EAAE,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,mCAAmC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,0IAA0I,EAAE;oBAC1Q,WAAW,EAAE,UAAU,CAAC,WAA0B,EAAE;wBAClD,QAAQ,EAAE,QAAQ;4BAChB,CAAC,CAAC,CAAC,KAAK,EAAE,EAAE;gCACR,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;gCAC/B,OAAO,MAAM,CAAC,KAAK;oCACjB,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,EAAE;oCACvC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,KAAK,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC;4BAChE,CAAC;4BACH,CAAC,CAAC,SAAS;qBACd,CAAC;oBACF,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE;wBACvC,IAAI,IAA6B,CAAC;wBAClC,IAAI,CAAC;4BACH,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAC5B,CAAC,KAAK,IAAI,EAAE,CAA4B,EACxC,MAAM,CACP,CAAC;wBACJ,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BACf,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;4BAC7C,MAAM,KAAK,CAAC;wBACd,CAAC;wBACD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;wBAClC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;wBAC9D,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;4BAC1B,OAAO,YAAY,CAAC,OAAO,CAAC,CAAC;wBAC/B,CAAC;wBACD,IAAI,MAA+C,CAAC;wBACpD,IAAI,CAAC;4BACH,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;gCAClC,IAAI,EAAE,UAAU,CAAC,IAAI;gCACrB,SAAS,EAAE,IAAI;6BAChB,CAAC,CAAC;wBACL,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BACf,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,sCAAsC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC/F,CAAC;4BACF,MAAM,KAAK,CAAC;wBACd,CAAC;wBACD,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;4BAC5B,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,+BAA+B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAChE,CAAC;wBACJ,CAAC;wBACD,OAAO;4BACL,OAAO,EAAE,MAAM,CAAC,OAAO;4BACvB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;yBAC5C,CAAC;oBACJ,CAAC;iBACF,CAAC;aACM,CAAC;QACb,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;CACF"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export type { Chat } from "./chat.js";
|
|
2
|
-
export type { ChatAssertion, ChatMatchers, JudgmentOptions, } from "./matchers/index.js";
|
|
2
|
+
export type { ChatAssertion, ChatMatchers, Judgment, JudgmentOptions, Verdict, } from "./matchers/index.js";
|
|
3
3
|
export type { EvalApp, EvalIdentity, ToolStubs, } from "./session-registry.js";
|
|
4
4
|
export { start } from "./session-registry.js";
|
|
5
5
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACtC,YAAY,EACV,aAAa,EACb,YAAY,EACZ,eAAe,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACtC,YAAY,EACV,aAAa,EACb,YAAY,EACZ,QAAQ,EACR,eAAe,EACf,OAAO,GACR,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EACV,OAAO,EACP,YAAY,EACZ,SAAS,GACV,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAaA,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { FilePart, UserContent } from "ai";
|
|
2
|
+
export interface InputFileMatcher {
|
|
3
|
+
asymmetricMatch(value: unknown): boolean;
|
|
4
|
+
toString(): string;
|
|
5
|
+
toJSON(): string;
|
|
6
|
+
}
|
|
7
|
+
export declare function inputFile(part: FilePart): InputFileMatcher;
|
|
8
|
+
export declare class InputFiles {
|
|
9
|
+
private readonly files;
|
|
10
|
+
private readonly parts;
|
|
11
|
+
private readonly server;
|
|
12
|
+
prepare(content: UserContent): Promise<UserContent>;
|
|
13
|
+
rewrite(input: Record<string, unknown>, fields: string[]): {
|
|
14
|
+
[x: string]: unknown;
|
|
15
|
+
};
|
|
16
|
+
close(): Promise<void>;
|
|
17
|
+
}
|
|
18
|
+
//# sourceMappingURL=input-files.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"input-files.d.ts","sourceRoot":"","sources":["../src/input-files.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,IAAI,CAAC;AAWhD,MAAM,WAAW,gBAAgB;IAC/B,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC;IACzC,QAAQ,IAAI,MAAM,CAAC;IACnB,MAAM,IAAI,MAAM,CAAC;CAClB;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,QAAQ,GAAG,gBAAgB,CAsB1D;AAoCD,qBAAa,UAAU;IACrB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAGlB;IACJ,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA0C;IAChE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAQpB;IAEG,OAAO,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAmDzD,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE;;;IAuBlD,KAAK;CAQZ"}
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { createServer } from "node:http";
|
|
3
|
+
const references = new WeakMap();
|
|
4
|
+
export function inputFile(part) {
|
|
5
|
+
function matches(value) {
|
|
6
|
+
if (Array.isArray(value)) {
|
|
7
|
+
return value.some(matches);
|
|
8
|
+
}
|
|
9
|
+
return [...(references.get(part) ?? [])].some((reference) => {
|
|
10
|
+
if (typeof value === "string") {
|
|
11
|
+
return value === reference.file_id || value === reference.download_url;
|
|
12
|
+
}
|
|
13
|
+
return (value !== null &&
|
|
14
|
+
typeof value === "object" &&
|
|
15
|
+
"file_id" in value &&
|
|
16
|
+
"download_url" in value &&
|
|
17
|
+
value.file_id === reference.file_id &&
|
|
18
|
+
value.download_url === reference.download_url);
|
|
19
|
+
});
|
|
20
|
+
}
|
|
21
|
+
const label = () => `inputFile(${part.filename ?? part.mediaType}: ${[...(references.get(part) ?? [])].map((reference) => reference.file_id).join(", ") || "not sent"})`;
|
|
22
|
+
return { asymmetricMatch: matches, toString: label, toJSON: label };
|
|
23
|
+
}
|
|
24
|
+
async function fileBytes(data) {
|
|
25
|
+
if (data instanceof URL) {
|
|
26
|
+
const response = await fetch(data);
|
|
27
|
+
if (!response.ok) {
|
|
28
|
+
throw new Error(`Input file download failed (HTTP ${response.status})`);
|
|
29
|
+
}
|
|
30
|
+
return Buffer.from(await response.arrayBuffer());
|
|
31
|
+
}
|
|
32
|
+
if (typeof data === "string") {
|
|
33
|
+
if (data.startsWith("data:")) {
|
|
34
|
+
return fileBytes(new URL(data));
|
|
35
|
+
}
|
|
36
|
+
return Buffer.from(data, "base64");
|
|
37
|
+
}
|
|
38
|
+
if (data instanceof ArrayBuffer) {
|
|
39
|
+
return Buffer.from(data);
|
|
40
|
+
}
|
|
41
|
+
if (data instanceof Uint8Array) {
|
|
42
|
+
return Buffer.from(data);
|
|
43
|
+
}
|
|
44
|
+
if (data.type === "data") {
|
|
45
|
+
return fileBytes(data.data);
|
|
46
|
+
}
|
|
47
|
+
if (data.type === "url") {
|
|
48
|
+
return fileBytes(new URL(data.url));
|
|
49
|
+
}
|
|
50
|
+
if (data.type === "text" && typeof data.text === "string") {
|
|
51
|
+
return Buffer.from(data.text);
|
|
52
|
+
}
|
|
53
|
+
throw new Error("Input files need bytes, text, or a URL; provider-only file references cannot be downloaded by the eval app");
|
|
54
|
+
}
|
|
55
|
+
export class InputFiles {
|
|
56
|
+
files = new Map();
|
|
57
|
+
parts = new WeakMap();
|
|
58
|
+
server = createServer((request, response) => {
|
|
59
|
+
const file = this.files.get(request.url?.slice(1) ?? "");
|
|
60
|
+
if (!file) {
|
|
61
|
+
response.writeHead(404).end();
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
response.writeHead(200, { "Content-Type": file.reference.mime_type });
|
|
65
|
+
response.end(file.bytes);
|
|
66
|
+
});
|
|
67
|
+
async prepare(content) {
|
|
68
|
+
if (typeof content === "string") {
|
|
69
|
+
return content;
|
|
70
|
+
}
|
|
71
|
+
const prepared = [];
|
|
72
|
+
for (const part of content) {
|
|
73
|
+
if (part.type !== "file") {
|
|
74
|
+
prepared.push(part);
|
|
75
|
+
continue;
|
|
76
|
+
}
|
|
77
|
+
let reference = this.parts.get(part);
|
|
78
|
+
if (!reference) {
|
|
79
|
+
const bytes = await fileBytes(part.data);
|
|
80
|
+
if (!this.server.listening) {
|
|
81
|
+
await new Promise((resolve, reject) => {
|
|
82
|
+
this.server.once("error", reject);
|
|
83
|
+
this.server.listen(0, "127.0.0.1", () => {
|
|
84
|
+
this.server.off("error", reject);
|
|
85
|
+
this.server.unref();
|
|
86
|
+
resolve();
|
|
87
|
+
});
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
const address = this.server.address();
|
|
91
|
+
if (!address || typeof address === "string") {
|
|
92
|
+
throw new Error("Input file server did not open a local port");
|
|
93
|
+
}
|
|
94
|
+
const file_id = `file_${randomUUID()}`;
|
|
95
|
+
reference = {
|
|
96
|
+
file_id,
|
|
97
|
+
download_url: `http://127.0.0.1:${address.port}/${file_id}`,
|
|
98
|
+
mime_type: part.mediaType,
|
|
99
|
+
...(part.filename === undefined ? {} : { file_name: part.filename }),
|
|
100
|
+
};
|
|
101
|
+
this.files.set(file_id, { reference, bytes });
|
|
102
|
+
this.parts.set(part, reference);
|
|
103
|
+
const known = references.get(part) ?? new Set();
|
|
104
|
+
known.add(reference);
|
|
105
|
+
references.set(part, known);
|
|
106
|
+
}
|
|
107
|
+
prepared.push({
|
|
108
|
+
type: "text",
|
|
109
|
+
text: `Attached file: ${JSON.stringify(reference)}. For openai/fileParams fields, pass its file_id; for plain URL fields, pass its download_url.`,
|
|
110
|
+
}, part);
|
|
111
|
+
}
|
|
112
|
+
return prepared;
|
|
113
|
+
}
|
|
114
|
+
rewrite(input, fields) {
|
|
115
|
+
const rewritten = { ...input };
|
|
116
|
+
const resolve = (value) => {
|
|
117
|
+
if (typeof value !== "string") {
|
|
118
|
+
return value;
|
|
119
|
+
}
|
|
120
|
+
const file = this.files.get(value);
|
|
121
|
+
if (!file) {
|
|
122
|
+
throw new Error(`Unknown input file id: ${value}`);
|
|
123
|
+
}
|
|
124
|
+
return file.reference;
|
|
125
|
+
};
|
|
126
|
+
for (const field of fields) {
|
|
127
|
+
if (field in rewritten) {
|
|
128
|
+
const value = rewritten[field];
|
|
129
|
+
rewritten[field] = Array.isArray(value)
|
|
130
|
+
? value.map(resolve)
|
|
131
|
+
: resolve(value);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
return rewritten;
|
|
135
|
+
}
|
|
136
|
+
async close() {
|
|
137
|
+
if (this.server.listening) {
|
|
138
|
+
await new Promise((resolve, reject) => {
|
|
139
|
+
this.server.close((error) => (error ? reject(error) : resolve()));
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
this.files.clear();
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
//# sourceMappingURL=input-files.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"input-files.js","sourceRoot":"","sources":["../src/input-files.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAUzC,MAAM,UAAU,GAAG,IAAI,OAAO,EAAgC,CAAC;AAQ/D,MAAM,UAAU,SAAS,CAAC,IAAc;IACtC,SAAS,OAAO,CAAC,KAAc;QAC7B,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC7B,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE;YAC1D,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,OAAO,KAAK,KAAK,SAAS,CAAC,OAAO,IAAI,KAAK,KAAK,SAAS,CAAC,YAAY,CAAC;YACzE,CAAC;YACD,OAAO,CACL,KAAK,KAAK,IAAI;gBACd,OAAO,KAAK,KAAK,QAAQ;gBACzB,SAAS,IAAI,KAAK;gBAClB,cAAc,IAAI,KAAK;gBACvB,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,OAAO;gBACnC,KAAK,CAAC,YAAY,KAAK,SAAS,CAAC,YAAY,CAC9C,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IACD,MAAM,KAAK,GAAG,GAAG,EAAE,CACjB,aAAa,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,SAAS,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,UAAU,GAAG,CAAC;IACvJ,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AACtE,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,IAAsB;IAC7C,IAAI,IAAI,YAAY,GAAG,EAAE,CAAC;QACxB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,oCAAoC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;QAC1E,CAAC;QACD,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;IACnD,CAAC;IACD,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC7B,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC7B,OAAO,SAAS,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;QAClC,CAAC;QACD,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACrC,CAAC;IACD,IAAI,IAAI,YAAY,WAAW,EAAE,CAAC;QAChC,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3B,CAAC;IACD,IAAI,IAAI,YAAY,UAAU,EAAE,CAAC;QAC/B,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3B,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QACzB,OAAO,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;QACxB,OAAO,SAAS,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC1D,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IACD,MAAM,IAAI,KAAK,CACb,4GAA4G,CAC7G,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,UAAU;IACJ,KAAK,GAAG,IAAI,GAAG,EAG7B,CAAC;IACa,KAAK,GAAG,IAAI,OAAO,EAA2B,CAAC;IAC/C,MAAM,GAAG,YAAY,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE;QAC3D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACzD,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;YAC9B,OAAO;QACT,CAAC;QACD,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC,CAAC;QACtE,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,KAAK,CAAC,OAAO,CAAC,OAAoB;QAChC,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YAChC,OAAO,OAAO,CAAC;QACjB,CAAC;QACD,MAAM,QAAQ,GAAiC,EAAE,CAAC;QAClD,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YAC3B,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACzB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACpB,SAAS;YACX,CAAC;YACD,IAAI,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACrC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACzC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;oBAC3B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;wBAC1C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;wBAClC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,WAAW,EAAE,GAAG,EAAE;4BACtC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;4BACjC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;4BACpB,OAAO,EAAE,CAAC;wBACZ,CAAC,CAAC,CAAC;oBACL,CAAC,CAAC,CAAC;gBACL,CAAC;gBACD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBACtC,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;oBAC5C,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;gBACjE,CAAC;gBACD,MAAM,OAAO,GAAG,QAAQ,UAAU,EAAE,EAAE,CAAC;gBACvC,SAAS,GAAG;oBACV,OAAO;oBACP,YAAY,EAAE,oBAAoB,OAAO,CAAC,IAAI,IAAI,OAAO,EAAE;oBAC3D,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,GAAG,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;iBACrE,CAAC;gBACF,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC9C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;gBAChC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,EAAiB,CAAC;gBAC/D,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBACrB,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAC9B,CAAC;YACD,QAAQ,CAAC,IAAI,CACX;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,kBAAkB,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,gGAAgG;aAClJ,EACD,IAAI,CACL,CAAC;QACJ,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,OAAO,CAAC,KAA8B,EAAE,MAAgB;QACtD,MAAM,SAAS,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;QAC/B,MAAM,OAAO,GAAG,CAAC,KAAc,EAAW,EAAE;YAC1C,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,OAAO,KAAK,CAAC;YACf,CAAC;YACD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACnC,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,MAAM,IAAI,KAAK,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;YACrD,CAAC;YACD,OAAO,IAAI,CAAC,SAAS,CAAC;QACxB,CAAC,CAAC;QACF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,KAAK,IAAI,SAAS,EAAE,CAAC;gBACvB,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;gBAC/B,SAAS,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;oBACrC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC;oBACpB,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YAC1B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBAC1C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACpE,CAAC,CAAC,CAAC;QACL,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;CACF"}
|
package/dist/matchers/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
import "./matchers.js";
|
|
2
|
-
export type { ChatAssertion, ChatMatchers, JudgmentOptions, } from "./matchers.js";
|
|
2
|
+
export type { ChatAssertion, ChatMatchers, Judgment, JudgmentOptions, Verdict, } from "./matchers.js";
|
|
3
3
|
export type { ChatLike, ToolCall, Turn } from "./types.js";
|
|
4
4
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/matchers/index.ts"],"names":[],"mappings":"AAAA,OAAO,eAAe,CAAC;AAEvB,YAAY,EACV,aAAa,EACb,YAAY,EACZ,eAAe,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/matchers/index.ts"],"names":[],"mappings":"AAAA,OAAO,eAAe,CAAC;AAEvB,YAAY,EACV,aAAa,EACb,YAAY,EACZ,QAAQ,EACR,eAAe,EACf,OAAO,GACR,MAAM,eAAe,CAAC;AACvB,YAAY,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC"}
|
|
@@ -1,20 +1,45 @@
|
|
|
1
1
|
import "vitest";
|
|
2
|
-
import type { LanguageModel } from "ai";
|
|
2
|
+
import type { FilePart, LanguageModel } from "ai";
|
|
3
3
|
import type { ToolInput, ToolNames } from "skybridge/server";
|
|
4
|
+
import { type InputFileMatcher } from "../input-files.js";
|
|
4
5
|
import type { ChatLike } from "./types.js";
|
|
6
|
+
/** What a judge decided about a conversation. */
|
|
7
|
+
export interface Verdict {
|
|
8
|
+
pass: boolean;
|
|
9
|
+
/** Why, quoted in the failure message. Omit it when the judge gives no prose. */
|
|
10
|
+
reasoning?: string;
|
|
11
|
+
}
|
|
12
|
+
/** The conversation handed to a {@link JudgmentOptions.judge}. */
|
|
13
|
+
export interface Judgment {
|
|
14
|
+
criteria: string;
|
|
15
|
+
/** The rendered conversation, one turn per line, tool results included. */
|
|
16
|
+
transcript: string;
|
|
17
|
+
}
|
|
5
18
|
/** Options for {@link ChatMatchers.toPassJudgment}. */
|
|
6
19
|
export interface JudgmentOptions {
|
|
7
20
|
/** Overrides the judge for this assertion. Defaults to the chat's model. */
|
|
8
21
|
model?: LanguageModel;
|
|
22
|
+
/**
|
|
23
|
+
* Grades the conversation instead of the default model call, so an
|
|
24
|
+
* evaluation model or a scoring service can stand in for the judge. Takes
|
|
25
|
+
* precedence over {@link JudgmentOptions.model}.
|
|
26
|
+
*/
|
|
27
|
+
judge?: (judgment: Judgment) => Verdict | Promise<Verdict>;
|
|
9
28
|
}
|
|
29
|
+
type ArgumentMatcher<T> = T | Pick<InputFileMatcher, "asymmetricMatch"> | (T extends readonly (infer Item)[] ? ArgumentMatcher<Item>[] : T extends object ? {
|
|
30
|
+
[Key in keyof T]: ArgumentMatcher<T[Key]>;
|
|
31
|
+
} : never);
|
|
32
|
+
type MatchingArguments<T> = {
|
|
33
|
+
[Key in keyof T]?: ArgumentMatcher<T[Key]>;
|
|
34
|
+
};
|
|
10
35
|
/**
|
|
11
36
|
* Assertions about the tool calls a conversation produced. Kept off vitest's
|
|
12
37
|
* global `Matchers` so they never appear on unrelated `expect()` calls, and so
|
|
13
38
|
* this stays the one place later tool-call helpers are added.
|
|
14
39
|
*/
|
|
15
40
|
export interface ChatMatchers<App> {
|
|
16
|
-
toHaveCalledToolOnce<Name extends ToolNames<App>>(name: Name, args?:
|
|
17
|
-
toHaveCalledToolWith<Name extends ToolNames<App>>(name: Name, args:
|
|
41
|
+
toHaveCalledToolOnce<Name extends ToolNames<App>>(name: Name, args?: MatchingArguments<ToolInput<App, Name>>): void;
|
|
42
|
+
toHaveCalledToolWith<Name extends ToolNames<App>>(name: Name, args: MatchingArguments<ToolInput<App, Name>>): void;
|
|
18
43
|
toNeverHaveCalledTool<Name extends ToolNames<App>>(name: Name): void;
|
|
19
44
|
toHaveFailedToolCall<Name extends ToolNames<App>>(name: Name): void;
|
|
20
45
|
toHaveSaid(text: string | RegExp): void;
|
|
@@ -27,7 +52,9 @@ export interface ChatAssertion<App> extends ChatMatchers<App> {
|
|
|
27
52
|
}
|
|
28
53
|
declare module "vitest" {
|
|
29
54
|
interface ExpectStatic {
|
|
55
|
+
inputFile(part: FilePart): InputFileMatcher;
|
|
30
56
|
chat<App>(chat: ChatLike<App>): ChatAssertion<App>;
|
|
31
57
|
}
|
|
32
58
|
}
|
|
59
|
+
export {};
|
|
33
60
|
//# sourceMappingURL=matchers.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"matchers.d.ts","sourceRoot":"","sources":["../../src/matchers/matchers.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,CAAC;AAChB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"matchers.d.ts","sourceRoot":"","sources":["../../src/matchers/matchers.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,CAAC;AAChB,OAAO,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAClD,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7D,OAAO,EAAE,KAAK,gBAAgB,EAAa,MAAM,mBAAmB,CAAC;AACrE,OAAO,KAAK,EAAE,QAAQ,EAAkB,MAAM,YAAY,CAAC;AAE3D,iDAAiD;AACjD,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,OAAO,CAAC;IACd,iFAAiF;IACjF,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,kEAAkE;AAClE,MAAM,WAAW,QAAQ;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,2EAA2E;IAC3E,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,uDAAuD;AACvD,MAAM,WAAW,eAAe;IAC9B,4EAA4E;IAC5E,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB;;;;OAIG;IACH,KAAK,CAAC,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAC5D;AAED,KAAK,eAAe,CAAC,CAAC,IAClB,CAAC,GACD,IAAI,CAAC,gBAAgB,EAAE,iBAAiB,CAAC,GACzC,CAAC,CAAC,SAAS,SAAS,CAAC,MAAM,IAAI,CAAC,EAAE,GAC9B,eAAe,CAAC,IAAI,CAAC,EAAE,GACvB,CAAC,SAAS,MAAM,GACd;KAAG,GAAG,IAAI,MAAM,CAAC,GAAG,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;CAAE,GAC7C,KAAK,CAAC,CAAC;AAEjB,KAAK,iBAAiB,CAAC,CAAC,IAAI;KAAG,GAAG,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;CAAE,CAAC;AAE3E;;;;GAIG;AACH,MAAM,WAAW,YAAY,CAAC,GAAG;IAC/B,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAC9C,IAAI,EAAE,IAAI,EACV,IAAI,CAAC,EAAE,iBAAiB,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GAC7C,IAAI,CAAC;IACR,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAC9C,IAAI,EAAE,IAAI,EACV,IAAI,EAAE,iBAAiB,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GAC5C,IAAI,CAAC;IACR,qBAAqB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,CAAC;IACrE,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,CAAC;IACpE,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IACxC,wBAAwB,CACtB,GAAG,KAAK,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,GAAG,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,GAC9C,IAAI,CAAC;IACR,mBAAmB,IAAI,IAAI,CAAC;IAC5B,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5E;AAED,MAAM,WAAW,aAAa,CAAC,GAAG,CAAE,SAAQ,YAAY,CAAC,GAAG,CAAC;IAC3D,GAAG,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC;CACxB;AAED,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,YAAY;QACpB,SAAS,CAAC,IAAI,EAAE,QAAQ,GAAG,gBAAgB,CAAC;QAC5C,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;KACpD;CACF"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import "vitest";
|
|
2
2
|
import { expect } from "vitest";
|
|
3
|
+
import { inputFile } from "../input-files.js";
|
|
3
4
|
function calls(chat) {
|
|
4
5
|
return chat.turns
|
|
5
6
|
.filter((turn) => turn.role === "tool")
|
|
@@ -83,6 +84,25 @@ function report(chat, summary) {
|
|
|
83
84
|
Calls the model actually made:
|
|
84
85
|
${observed(chat)}`;
|
|
85
86
|
}
|
|
87
|
+
async function modelVerdict(chat, criteria, model) {
|
|
88
|
+
const { generateObject, jsonSchema } = await import("ai");
|
|
89
|
+
const { object } = await generateObject({
|
|
90
|
+
model: model ?? chat.model,
|
|
91
|
+
temperature: 0,
|
|
92
|
+
system: JUDGE_SYSTEM,
|
|
93
|
+
prompt: `Criteria:\n${criteria}\n\n<conversation>\n${transcriptFor(chat)}\n</conversation>`,
|
|
94
|
+
schema: jsonSchema({
|
|
95
|
+
type: "object",
|
|
96
|
+
properties: {
|
|
97
|
+
pass: { type: "boolean" },
|
|
98
|
+
reasoning: { type: "string" },
|
|
99
|
+
},
|
|
100
|
+
required: ["pass", "reasoning"],
|
|
101
|
+
additionalProperties: false,
|
|
102
|
+
}),
|
|
103
|
+
});
|
|
104
|
+
return object;
|
|
105
|
+
}
|
|
86
106
|
expect.extend({
|
|
87
107
|
toHaveCalledToolOnce(received, name, args) {
|
|
88
108
|
const matching = acceptedCalls(received, name);
|
|
@@ -164,23 +184,15 @@ ${spoken(received)}`,
|
|
|
164
184
|
async toPassJudgment(received, criteria, options) {
|
|
165
185
|
let verdict;
|
|
166
186
|
try {
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
pass: { type: "boolean" },
|
|
177
|
-
reasoning: { type: "string" },
|
|
178
|
-
},
|
|
179
|
-
required: ["pass", "reasoning"],
|
|
180
|
-
additionalProperties: false,
|
|
181
|
-
}),
|
|
182
|
-
});
|
|
183
|
-
verdict = object;
|
|
187
|
+
verdict = options?.judge
|
|
188
|
+
? await options.judge({
|
|
189
|
+
criteria,
|
|
190
|
+
transcript: transcriptFor(received),
|
|
191
|
+
})
|
|
192
|
+
: await modelVerdict(received, criteria, options?.model);
|
|
193
|
+
if (typeof verdict?.pass !== "boolean") {
|
|
194
|
+
throw new Error(`the judge returned ${JSON.stringify(verdict)} instead of a verdict with a boolean "pass"`);
|
|
195
|
+
}
|
|
184
196
|
}
|
|
185
197
|
catch (error) {
|
|
186
198
|
throw new Error(`judge unavailable: ${error instanceof Error ? error.message : String(error)}`);
|
|
@@ -190,8 +202,7 @@ ${spoken(received)}`,
|
|
|
190
202
|
message: () => `expected the conversation ${verdict.pass ? "not " : ""}to pass judgment:
|
|
191
203
|
"${criteria}"
|
|
192
204
|
|
|
193
|
-
judge: ${verdict.pass ? "PASS" : "FAIL"}
|
|
194
|
-
${verdict.reasoning}`,
|
|
205
|
+
judge: ${verdict.pass ? "PASS" : "FAIL"}${verdict.reasoning === undefined ? "" : `\n ${verdict.reasoning}`}`,
|
|
195
206
|
};
|
|
196
207
|
},
|
|
197
208
|
toNeverHaveCalledTool(received, name) {
|
|
@@ -205,4 +216,5 @@ judge: ${verdict.pass ? "PASS" : "FAIL"}
|
|
|
205
216
|
},
|
|
206
217
|
});
|
|
207
218
|
expect.chat = (chat) => expect(chat);
|
|
219
|
+
expect.inputFile = inputFile;
|
|
208
220
|
//# sourceMappingURL=matchers.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"matchers.js","sourceRoot":"","sources":["../../src/matchers/matchers.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,CAAC;AAGhB,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;
|
|
1
|
+
{"version":3,"file":"matchers.js","sourceRoot":"","sources":["../../src/matchers/matchers.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,CAAC;AAGhB,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EAAyB,SAAS,EAAE,MAAM,mBAAmB,CAAC;AA2ErE,SAAS,KAAK,CAAC,IAAuB;IACpC,OAAO,IAAI,CAAC,KAAK;SACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC;SACtC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC;AAED,SAAS,WAAW,CAAC,IAAuB;IAC1C,OAAO,IAAI,CAAC,KAAK;SACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,WAAW,CAAC;SAC3C,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC;AAED,SAAS,cAAc,CACrB,IAAuB,EACvB,IAAY;IAEZ,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,WAAW,CAAC,IAAuB;IAC1C,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;AACjE,CAAC;AAED,SAAS,aAAa,CACpB,IAAuB,EACvB,IAAY;IAEZ,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;AACxE,CAAC;AAED,SAAS,QAAQ,CAAC,IAAuB;IACvC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IACzB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,wBAAwB,CAAC;IAClC,CAAC;IACD,OAAO,IAAI;SACR,GAAG,CACF,CAAC,IAAuB,EAAE,KAAa,EAAE,EAAE,CACzC,KAAK,KAAK,GAAG,CAAC,KAAK,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,cAAc,IAAI,CAAC,MAAM,GAAG,EAAE,CACnI;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AAC1C,CAAC;AAED,SAAS,IAAI,CAAC,IAAuB,EAAE,IAAqB;IAC1D,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CACrC,OAAO,IAAI,KAAK,QAAQ;QACtB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;QACrE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CACpB,CAAC;AACJ,CAAC;AAED,SAAS,MAAM,CAAC,IAAuB;IACrC,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,gCAAgC,CAAC;IAC1C,CAAC;IACD,OAAO,KAAK;SACT,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,GAAG,CAAC,KAAK,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;SACzD,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,SAAS,KAAK,CAAC,IAAY;IACzB,OAAO,IAAI,CAAC,OAAO,CAAC,uBAAuB,EAAE,QAAQ,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,YAAY,GAAG;;;6KAGwJ,CAAC;AAE9K,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAE9B,SAAS,QAAQ,CAAC,IAAmB;IACnC,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QACzB,OAAO,GAAG,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;IAC7C,CAAC;IACD,MAAM,OAAO,GACX,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS;QAC5B,CAAC,CAAC,OAAO,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,EAAE;QACxD,CAAC,CAAC,aAAa,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;IACvC,OAAO,SAAS,KAAK,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,OAAO,EAAE,CAAC,EAAE,CAAC;AACxG,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,IAAI,CAAC,MAAM,GAAG,gBAAgB;QACnC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC,eAAe;QACnD,CAAC,CAAC,IAAI,CAAC;AACX,CAAC;AAED,SAAS,aAAa,CAAC,IAAuB;IAC5C,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,6BAA6B,CAAC;IACvC,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC7C,CAAC;AAED,SAAS,MAAM,CAAC,IAAuB,EAAE,OAAe;IACtD,OAAO,GAAG,OAAO;;;EAGjB,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;AACnB,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,IAAuB,EACvB,QAAgB,EAChB,KAAqB;IAErB,MAAM,EAAE,cAAc,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;IAC1D,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,cAAc,CAAC;QACtC,KAAK,EAAE,KAAK,IAAI,IAAI,CAAC,KAAK;QAC1B,WAAW,EAAE,CAAC;QACd,MAAM,EAAE,YAAY;QACpB,MAAM,EAAE,cAAc,QAAQ,uBAAuB,aAAa,CAAC,IAAI,CAAC,mBAAmB;QAC3F,MAAM,EAAE,UAAU,CAAkC;YAClD,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;gBACzB,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;aAC9B;YACD,QAAQ,EAAE,CAAC,MAAM,EAAE,WAAW,CAAC;YAC/B,oBAAoB,EAAE,KAAK;SAC5B,CAAC;KACH,CAAC,CAAC;IACH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,MAAM,CAAC;IACZ,oBAAoB,CAClB,QAA2B,EAC3B,IAAY,EACZ,IAA8B;QAE9B,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC/C,MAAM,SAAS,GACb,IAAI,KAAK,SAAS;YAClB,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACpB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxE,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC;QAEhD,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,aAAa,IAAI,kCAAkC;gBACrD,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;oBACrB,CAAC,CAAC,aAAa,IAAI,4BAA4B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,4BAA4B,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG;oBACxI,CAAC,CAAC,aAAa,IAAI,6DAA6D,QAAQ,CAAC,MAAM,wBAAwB,CAC5H;SACJ,CAAC;IACJ,CAAC;IAED,oBAAoB,CAClB,QAA2B,EAC3B,IAAY,EACZ,IAA6B;QAE7B,MAAM,IAAI,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CACvD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAC3D,CAAC;QAEF,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,wBAAwB,IAAI,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG;gBAC/D,CAAC,CAAC,aAAa,IAAI,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CACpE;SACJ,CAAC;IACJ,CAAC;IAED,oBAAoB,CAAC,QAA2B,EAAE,IAAY;QAC5D,MAAM,SAAS,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;QAErE,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC;YACvB,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,MAAM,CAAC,MAAM,GAAG,CAAC;gBACf,CAAC,CAAC,2BAA2B,IAAI,eAAe;gBAChD,CAAC,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;oBACtB,CAAC,CAAC,aAAa,IAAI,mDAAmD;oBACtE,CAAC,CAAC,aAAa,IAAI,sBAAsB,SAAS,CAAC,MAAM,mBAAmB,CACjF;SACJ,CAAC;IACJ,CAAC;IAED,UAAU,CAAC,QAA2B,EAAE,IAAqB;QAC3D,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAClC,MAAM,MAAM,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAErE,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,GACE,IAAI;gBACF,CAAC,CAAC,qCAAqC,MAAM,GAAG;gBAChD,CAAC,CAAC,iCAAiC,MAAM,GAC7C;;;EAGN,MAAM,CAAC,QAAQ,CAAC,EAAE;SACf,CAAC;IACJ,CAAC;IAED,mBAAmB,CAAC,QAA2B;QAC7C,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;QAErC,OAAO;YACL,IAAI,EAAE,KAAK,KAAK,CAAC;YACjB,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,KAAK,KAAK,CAAC;gBACT,CAAC,CAAC,+CAA+C;gBACjD,CAAC,CAAC,wDAAwD,KAAK,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,GAAG,CACvG;SACJ,CAAC;IACJ,CAAC;IAED,wBAAwB,CAAC,QAA2B,EAAE,GAAG,KAAe;QACtE,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;YACzC,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;gBACzC,OAAO,IAAI,CAAC,CAAC;YACf,CAAC;YACD,IAAI,OAAO,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;gBAC7B,MAAM;YACR,CAAC;QACH,CAAC;QACD,MAAM,IAAI,GAAG,OAAO,KAAK,KAAK,CAAC,MAAM,CAAC;QACtC,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAE/D,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,YAAY,MAAM,kCAAkC;gBACtD,CAAC,CAAC,YAAY,MAAM,wBAAwB,KAAK,CAAC,OAAO,CAAC,eAAe,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CACpI;SACJ,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,cAAc,CAClB,QAA2B,EAC3B,QAAgB,EAChB,OAAyB;QAEzB,IAAI,OAAgB,CAAC;QACrB,IAAI,CAAC;YACH,OAAO,GAAG,OAAO,EAAE,KAAK;gBACtB,CAAC,CAAC,MAAM,OAAO,CAAC,KAAK,CAAC;oBAClB,QAAQ;oBACR,UAAU,EAAE,aAAa,CAAC,QAAQ,CAAC;iBACpC,CAAC;gBACJ,CAAC,CAAC,MAAM,YAAY,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;YAE3D,IAAI,OAAO,OAAO,EAAE,IAAI,KAAK,SAAS,EAAE,CAAC;gBACvC,MAAM,IAAI,KAAK,CACb,sBAAsB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,6CAA6C,CAC3F,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,sBAAsB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC/E,CAAC;QACJ,CAAC;QAED,OAAO;YACL,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,OAAO,EAAE,GAAG,EAAE,CACZ,6BAA6B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;KAC1D,QAAQ;;SAEJ,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,OAAO,CAAC,SAAS,EAAE,EAAE;SACxG,CAAC;IACJ,CAAC;IAED,qBAAqB,CAAC,QAA2B,EAAE,IAAY;QAC7D,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAEhD,OAAO;YACL,IAAI,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC;YAC3B,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACnB,CAAC,CAAC,aAAa,IAAI,+BAA+B;gBAClD,CAAC,CAAC,aAAa,IAAI,2CAA2C,QAAQ,CAAC,MAAM,SAAS,CACzF;SACJ,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,IAAI,GAAG,CAAM,IAAmB,EAAsB,EAAE,CAC7D,MAAM,CAAC,IAAI,CAAkC,CAAC;AAChD,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@skybridge/test",
|
|
3
|
-
"version": "0.0.0-dev.
|
|
4
|
-
"description": "Eval runner for Skybridge MCP
|
|
3
|
+
"version": "0.0.0-dev.b5c271e",
|
|
4
|
+
"description": "Eval runner for Skybridge MCP Apps: does a prompt reach the right tool with the right arguments",
|
|
5
5
|
"license": "ISC",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"repository": {
|
|
@@ -37,8 +37,8 @@
|
|
|
37
37
|
"typescript": "^5.9.3",
|
|
38
38
|
"vitest": "^4.1.10",
|
|
39
39
|
"zod": "^4.4.3",
|
|
40
|
-
"@skybridge/vite-plugin": "0.0.0-dev.
|
|
41
|
-
"skybridge": "0.0.0-dev.
|
|
40
|
+
"@skybridge/vite-plugin": "0.0.0-dev.b5c271e",
|
|
41
|
+
"skybridge": "0.0.0-dev.b5c271e"
|
|
42
42
|
},
|
|
43
43
|
"engines": {
|
|
44
44
|
"node": ">=22.23.1"
|