@skybridge/test 0.0.0-dev.fd4b722 → 0.0.0-dev.fec0253
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -1
- package/dist/chat.d.ts +5 -2
- package/dist/chat.d.ts.map +1 -1
- package/dist/chat.js +69 -8
- package/dist/chat.js.map +1 -1
- package/dist/input-files.d.ts +15 -0
- package/dist/input-files.d.ts.map +1 -0
- package/dist/input-files.js +125 -0
- package/dist/input-files.js.map +1 -0
- package/dist/matchers/matchers.d.ts +3 -2
- package/dist/matchers/matchers.d.ts.map +1 -1
- package/dist/matchers/matchers.js +12 -1
- package/dist/matchers/matchers.js.map +1 -1
- package/dist/matchers/types.d.ts +1 -0
- package/dist/matchers/types.d.ts.map +1 -1
- package/package.json +4 -4
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# @skybridge/test
|
|
2
2
|
|
|
3
|
-
Evals for [Skybridge](https://skybridge.tech) MCP
|
|
3
|
+
Evals for [Skybridge](https://skybridge.tech) MCP Apps: they answer whether a
|
|
4
4
|
prompt reaches the right tool with the right arguments, which no unit test can
|
|
5
5
|
check.
|
|
6
6
|
|
|
@@ -82,6 +82,37 @@ from, so a project can set them once instead of per scenario.
|
|
|
82
82
|
skybridge({ evals: { temperature: 0, maxSteps: 6 } });
|
|
83
83
|
```
|
|
84
84
|
|
|
85
|
+
## Input files
|
|
86
|
+
|
|
87
|
+
`chat.send` also accepts AI SDK `UserContent`. Attach a `FilePart` and use the
|
|
88
|
+
same object in `expect.inputFile` to check which file a tool received:
|
|
89
|
+
|
|
90
|
+
```ts
|
|
91
|
+
import { readFile } from "node:fs/promises";
|
|
92
|
+
import type { FilePart } from "ai";
|
|
93
|
+
|
|
94
|
+
const portrait: FilePart = {
|
|
95
|
+
type: "file",
|
|
96
|
+
mediaType: "image/png",
|
|
97
|
+
filename: "photo.png",
|
|
98
|
+
data: await readFile(new URL("./fixtures/photo.png", import.meta.url)),
|
|
99
|
+
};
|
|
100
|
+
|
|
101
|
+
await chat.send([{ type: "text", text: "Resize this photo" }, portrait]);
|
|
102
|
+
|
|
103
|
+
expect.chat(chat).toHaveCalledToolWith("image_crop_and_resize", {
|
|
104
|
+
file: expect.inputFile(portrait),
|
|
105
|
+
});
|
|
106
|
+
expect.chat(chat).toHaveCalledToolOnce("asset_openai_file_upload", {
|
|
107
|
+
files: [expect.inputFile(portrait)],
|
|
108
|
+
});
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
`file` and `files` must be listed in their tools' `_meta["openai/fileParams"]`.
|
|
112
|
+
Misplaced file matchers throw immediately, including nested matchers and `.not`.
|
|
113
|
+
See [Evals](https://docs.skybridge.tech/test/evals) for how files reach the model
|
|
114
|
+
and the tool.
|
|
115
|
+
|
|
85
116
|
## Matchers
|
|
86
117
|
|
|
87
118
|
Assertions on what the assistant said work the same way, though model prose
|
package/dist/chat.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type LanguageModel } from "ai";
|
|
1
|
+
import { type LanguageModel, type UserContent } from "ai";
|
|
2
2
|
import type { ToolCall, Turn } from "./matchers/types.js";
|
|
3
3
|
interface HostConfig {
|
|
4
4
|
model: LanguageModel;
|
|
@@ -33,6 +33,8 @@ export declare class Chat<App = unknown> {
|
|
|
33
33
|
get assistantTurns(): string[];
|
|
34
34
|
private readonly messages;
|
|
35
35
|
private readonly failures;
|
|
36
|
+
private readonly inputs;
|
|
37
|
+
private readonly inputFiles;
|
|
36
38
|
private readonly client;
|
|
37
39
|
private readonly transport;
|
|
38
40
|
private readonly host;
|
|
@@ -40,10 +42,11 @@ export declare class Chat<App = unknown> {
|
|
|
40
42
|
private tools;
|
|
41
43
|
/** The model the conversation runs on, and the judge's default. */
|
|
42
44
|
get model(): LanguageModel;
|
|
45
|
+
get fileParams(): Map<string, string[]>;
|
|
43
46
|
private constructor();
|
|
44
47
|
static open<App>(host: HostConfig, fetchImpl: TransportFetch, stubs: StubMap): Promise<Chat<App>>;
|
|
45
48
|
/** Takes a turn. Assert on the result with `expect.chat`. */
|
|
46
|
-
send(
|
|
49
|
+
send(content: UserContent): Promise<void>;
|
|
47
50
|
close(): Promise<void>;
|
|
48
51
|
private failureFor;
|
|
49
52
|
private toolSet;
|
package/dist/chat.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAKA,OAAO,
|
|
1
|
+
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAKA,OAAO,EAKL,KAAK,aAAa,EAGlB,KAAK,WAAW,EACjB,MAAM,IAAI,CAAC;AAEZ,OAAO,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AAE1D,UAAU,UAAU;IAClB,KAAK,EAAE,aAAa,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;GAKG;AACH,MAAM,MAAM,OAAO,GAAG,MAAM,CAC1B,MAAM,EACN,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,OAAO,CAC3C,CAAC;AAkDF;;;GAGG;AACH,MAAM,MAAM,cAAc,GAAG,CAC3B,GAAG,EAAE,MAAM,GAAG,GAAG,EACjB,IAAI,CAAC,EAAE,WAAW,KACf,OAAO,CAAC,QAAQ,CAAC,CAAC;AAEvB;;;GAGG;AACH,qBAAa,IAAI,CAAC,GAAG,GAAG,OAAO;IAC7B,4EAA4E;IAC5E,SAAiB,IAAI,EAAE,GAAG,CAAC;IAE3B,0DAA0D;IAC1D,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAM;IAEjC,mEAAmE;IACnE,IAAI,SAAS,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE,CAI/B;IAED,+EAA+E;IAC/E,IAAI,cAAc,IAAI,MAAM,EAAE,CAI7B;IAED,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA6B;IACtD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA8C;IACrE,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAoB;IAC/C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAa;IAClC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA+B;IACrD,OAAO,CAAC,KAAK,CAAc;IAE3B,mEAAmE;IACnE,IAAI,KAAK,IAAI,aAAa,CAEzB;IAED,IAAI,UAAU,0BAOb;IAED,OAAO;WAYM,IAAI,CAAC,GAAG,EACnB,IAAI,EAAE,UAAU,EAChB,SAAS,EAAE,cAAc,EACzB,KAAK,EAAE,OAAO,GACb,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IA0BrB,6DAA6D;IACvD,IAAI,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAmDzC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAK5B,OAAO,CAAC,UAAU;IAalB,OAAO,CAAC,OAAO;CA+DhB"}
|
package/dist/chat.js
CHANGED
|
@@ -1,7 +1,29 @@
|
|
|
1
1
|
import { Client, StreamableHTTPClientTransport, } from "@modelcontextprotocol/client";
|
|
2
2
|
import { dynamicTool, generateText, jsonSchema, stepCountIs, } from "ai";
|
|
3
|
+
import { InputFiles } from "./input-files.js";
|
|
3
4
|
const DEFAULT_MAX_STEPS = 8;
|
|
4
5
|
const IN_PROCESS_URL = "http://in-process.skybridge.test/mcp";
|
|
6
|
+
function fileInputSchema(schema) {
|
|
7
|
+
if (typeof schema === "boolean") {
|
|
8
|
+
return schema;
|
|
9
|
+
}
|
|
10
|
+
if (schema.anyOf) {
|
|
11
|
+
return { ...schema, anyOf: schema.anyOf.map(fileInputSchema) };
|
|
12
|
+
}
|
|
13
|
+
if (schema.type === "array") {
|
|
14
|
+
return {
|
|
15
|
+
...schema,
|
|
16
|
+
items: fileInputSchema(schema.items),
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
if (schema.type === "object") {
|
|
20
|
+
return {
|
|
21
|
+
type: "string",
|
|
22
|
+
description: "This parameter expects an absolute local file path.",
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
return schema;
|
|
26
|
+
}
|
|
5
27
|
/**
|
|
6
28
|
* Dress a stubbed value the way the server dresses a real one, so the model
|
|
7
29
|
* reads the same envelope under a stub as it does live.
|
|
@@ -40,6 +62,8 @@ export class Chat {
|
|
|
40
62
|
}
|
|
41
63
|
messages = [];
|
|
42
64
|
failures = new Map();
|
|
65
|
+
inputs = new Map();
|
|
66
|
+
inputFiles = new InputFiles();
|
|
43
67
|
client;
|
|
44
68
|
transport;
|
|
45
69
|
host;
|
|
@@ -49,6 +73,12 @@ export class Chat {
|
|
|
49
73
|
get model() {
|
|
50
74
|
return this.host.model;
|
|
51
75
|
}
|
|
76
|
+
get fileParams() {
|
|
77
|
+
return new Map(this.tools.map((tool) => [
|
|
78
|
+
tool.name,
|
|
79
|
+
tool._meta?.["openai/fileParams"] ?? [],
|
|
80
|
+
]));
|
|
81
|
+
}
|
|
52
82
|
constructor(client, transport, host, stubs) {
|
|
53
83
|
this.client = client;
|
|
54
84
|
this.transport = transport;
|
|
@@ -75,12 +105,25 @@ export class Chat {
|
|
|
75
105
|
return chat;
|
|
76
106
|
}
|
|
77
107
|
/** Takes a turn. Assert on the result with `expect.chat`. */
|
|
78
|
-
async send(
|
|
79
|
-
this.
|
|
80
|
-
this.
|
|
108
|
+
async send(content) {
|
|
109
|
+
const prepared = await this.inputFiles.prepare(content);
|
|
110
|
+
this.inputs.clear();
|
|
111
|
+
this.failures.clear();
|
|
112
|
+
this.messages.push({ role: "user", content: prepared });
|
|
113
|
+
this.turns.push({
|
|
114
|
+
role: "user",
|
|
115
|
+
text: typeof prepared === "string"
|
|
116
|
+
? prepared
|
|
117
|
+
: prepared
|
|
118
|
+
.filter((part) => part.type === "text")
|
|
119
|
+
.map((part) => part.text)
|
|
120
|
+
.join("\n"),
|
|
121
|
+
});
|
|
81
122
|
const result = await generateText({
|
|
82
123
|
model: this.host.model,
|
|
83
|
-
|
|
124
|
+
...(this.host.temperature === undefined
|
|
125
|
+
? {}
|
|
126
|
+
: { temperature: this.host.temperature }),
|
|
84
127
|
...(this.host.systemPrompt === undefined
|
|
85
128
|
? {}
|
|
86
129
|
: { system: this.host.systemPrompt }),
|
|
@@ -97,7 +140,8 @@ export class Chat {
|
|
|
97
140
|
role: "tool",
|
|
98
141
|
call: {
|
|
99
142
|
name: call.toolName,
|
|
100
|
-
arguments: (call.
|
|
143
|
+
arguments: this.inputs.get(call.toolCallId) ??
|
|
144
|
+
(call.input ?? {}),
|
|
101
145
|
...this.failureFor(call),
|
|
102
146
|
},
|
|
103
147
|
result: outputOf(step.toolResults, call.toolCallId),
|
|
@@ -107,6 +151,7 @@ export class Chat {
|
|
|
107
151
|
this.messages.push(...result.response.messages);
|
|
108
152
|
}
|
|
109
153
|
async close() {
|
|
154
|
+
await this.inputFiles.close();
|
|
110
155
|
await this.transport.close();
|
|
111
156
|
}
|
|
112
157
|
failureFor(call) {
|
|
@@ -122,13 +167,29 @@ export class Chat {
|
|
|
122
167
|
toolSet() {
|
|
123
168
|
const entries = this.tools.map((definition) => {
|
|
124
169
|
const { $schema: _ignored, ...parameters } = definition.inputSchema;
|
|
170
|
+
const fields = definition._meta?.["openai/fileParams"] ?? [];
|
|
171
|
+
const properties = { ...parameters.properties };
|
|
172
|
+
for (const field of fields) {
|
|
173
|
+
if (field in properties) {
|
|
174
|
+
properties[field] = fileInputSchema(properties[field]);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
125
177
|
return [
|
|
126
178
|
definition.name,
|
|
127
179
|
dynamicTool({
|
|
128
180
|
description: definition.description ?? "",
|
|
129
|
-
inputSchema: jsonSchema(parameters),
|
|
181
|
+
inputSchema: jsonSchema({ ...parameters, properties }),
|
|
130
182
|
execute: async (input, { toolCallId }) => {
|
|
131
|
-
|
|
183
|
+
let args;
|
|
184
|
+
try {
|
|
185
|
+
args = this.inputFiles.rewrite((input ?? {}), fields);
|
|
186
|
+
}
|
|
187
|
+
catch (error) {
|
|
188
|
+
this.failures.set(toolCallId, String(error));
|
|
189
|
+
throw error;
|
|
190
|
+
}
|
|
191
|
+
this.inputs.set(toolCallId, args);
|
|
192
|
+
const stubbed = await this.stubs.get(definition.name)?.(args);
|
|
132
193
|
if (stubbed !== undefined) {
|
|
133
194
|
return asToolResult(stubbed);
|
|
134
195
|
}
|
|
@@ -136,7 +197,7 @@ export class Chat {
|
|
|
136
197
|
try {
|
|
137
198
|
result = await this.client.callTool({
|
|
138
199
|
name: definition.name,
|
|
139
|
-
arguments:
|
|
200
|
+
arguments: args,
|
|
140
201
|
});
|
|
141
202
|
}
|
|
142
203
|
catch (error) {
|
package/dist/chat.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,MAAM,EACN,6BAA6B,GAE9B,MAAM,8BAA8B,CAAC;AACtC,OAAO,EACL,WAAW,EACX,YAAY,
|
|
1
|
+
{"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,MAAM,EACN,6BAA6B,GAE9B,MAAM,8BAA8B,CAAC;AACtC,OAAO,EACL,WAAW,EACX,YAAY,EAEZ,UAAU,EAGV,WAAW,GAEZ,MAAM,IAAI,CAAC;AACZ,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAqB9C,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B,MAAM,cAAc,GAAG,sCAAsC,CAAC;AAE9D,SAAS,eAAe,CAAC,MAA6B;IACpD,IAAI,OAAO,MAAM,KAAK,SAAS,EAAE,CAAC;QAChC,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,OAAO,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,CAAC;IACjE,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC5B,OAAO;YACL,GAAG,MAAM;YACT,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,KAAoB,CAAC;SACpD,CAAC;IACJ,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC7B,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,qDAAqD;SACnE,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO;QACL,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;aAChE;SACF;KACF,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CACf,OAAmD,EACnD,UAAkB;IAElB,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,KAAK,UAAU,CAAC,EAAE,MAAM,CAAC;AAC5E,CAAC;AAWD;;;GAGG;AACH,MAAM,OAAO,IAAI;IAIf,0DAA0D;IACjD,KAAK,GAAgB,EAAE,CAAC;IAEjC,mEAAmE;IACnE,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,KAAK;aACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC;aACtC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAED,+EAA+E;IAC/E,IAAI,cAAc;QAChB,OAAO,IAAI,CAAC,KAAK;aACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,WAAW,CAAC;aAC3C,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAEgB,QAAQ,GAAmB,EAAE,CAAC;IAC9B,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrC,MAAM,GAAG,IAAI,GAAG,EAAmC,CAAC;IACpD,UAAU,GAAG,IAAI,UAAU,EAAE,CAAC;IAC9B,MAAM,CAAS;IACf,SAAS,CAAgC;IACzC,IAAI,CAAa;IACjB,KAAK,CAA+B;IAC7C,KAAK,GAAW,EAAE,CAAC;IAE3B,mEAAmE;IACnE,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;IACzB,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,GAAG,CACZ,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YACvB,IAAI,CAAC,IAAI;YACR,IAAI,CAAC,KAAK,EAAE,CAAC,mBAAmB,CAA0B,IAAI,EAAE;SAClE,CAAC,CACH,CAAC;IACJ,CAAC;IAED,YACE,MAAc,EACd,SAAwC,EACxC,IAAgB,EAChB,KAAc;QAEd,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,IAAI,CACf,IAAgB,EAChB,SAAyB,EACzB,KAAc;QAEd,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,IAAI,6BAA6B,CACjD,IAAI,GAAG,CAAC,cAAc,CAAC,EACvB,EAAE,KAAK,EAAE,SAAS,EAAE,CACrB,CAAC;QACF,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAM,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;QAC3D,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;YACnB,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAC5C,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,CACpD,CAAC;YACF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,IAAI,KAAK,CACb,iCAAiC,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,+BAA+B,CACnH,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YACxB,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,6DAA6D;IAC7D,KAAK,CAAC,IAAI,CAAC,OAAoB;QAC7B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACxD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;QACxD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;YACd,IAAI,EAAE,MAAM;YACZ,IAAI,EACF,OAAO,QAAQ,KAAK,QAAQ;gBAC1B,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,QAAQ;qBACL,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC;qBACtC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;qBACxB,IAAI,CAAC,IAAI,CAAC;SACpB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;YACtB,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,KAAK,SAAS;gBACrC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YAC3C,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,KAAK,SAAS;gBACtC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE;YACrB,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;SAC/D,CAAC,CAAC;QAEH,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,IAAI,CAAC,IAAI,KAAK,EAAE,EAAE,CAAC;gBACrB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YAC1D,CAAC;YACD,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAClC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;oBACd,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE;wBACJ,IAAI,EAAE,IAAI,CAAC,QAAQ;wBACnB,SAAS,EACP,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC;4BAC/B,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAA6B;wBACjD,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;qBACR;oBAClB,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,UAAU,CAAC;iBACpD,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QAC9B,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;IAEO,UAAU,CAAC,IAA+C;QAGhE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QAC9B,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC1B,OAAO,EAAE,MAAM,EAAE,2CAA2C,EAAE,CAAC;QACjE,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAEO,OAAO;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;YAC5C,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,UAAU,EAAE,GAAG,UAAU,CAAC,WAAW,CAAC;YACpE,MAAM,MAAM,GACT,UAAU,CAAC,KAAK,EAAE,CAAC,mBAAmB,CAA0B,IAAI,EAAE,CAAC;YAC1E,MAAM,UAAU,GAAG,EAAE,GAAG,UAAU,CAAC,UAAU,EAE5C,CAAC;YACF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,IAAI,KAAK,IAAI,UAAU,EAAE,CAAC;oBACxB,UAAU,CAAC,KAAK,CAAC,GAAG,eAAe,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;gBACzD,CAAC;YACH,CAAC;YACD,OAAO;gBACL,UAAU,CAAC,IAAI;gBACf,WAAW,CAAC;oBACV,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,EAAE;oBACzC,WAAW,EAAE,UAAU,CAAC,EAAE,GAAG,UAAU,EAAE,UAAU,EAAiB,CAAC;oBACrE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE;wBACvC,IAAI,IAA6B,CAAC;wBAClC,IAAI,CAAC;4BACH,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAC5B,CAAC,KAAK,IAAI,EAAE,CAA4B,EACxC,MAAM,CACP,CAAC;wBACJ,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BACf,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;4BAC7C,MAAM,KAAK,CAAC;wBACd,CAAC;wBACD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;wBAClC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;wBAC9D,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;4BAC1B,OAAO,YAAY,CAAC,OAAO,CAAC,CAAC;wBAC/B,CAAC;wBACD,IAAI,MAA+C,CAAC;wBACpD,IAAI,CAAC;4BACH,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;gCAClC,IAAI,EAAE,UAAU,CAAC,IAAI;gCACrB,SAAS,EAAE,IAAI;6BAChB,CAAC,CAAC;wBACL,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BACf,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,sCAAsC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC/F,CAAC;4BACF,MAAM,KAAK,CAAC;wBACd,CAAC;wBACD,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;4BAC5B,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,+BAA+B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAChE,CAAC;wBACJ,CAAC;wBACD,OAAO;4BACL,OAAO,EAAE,MAAM,CAAC,OAAO;4BACvB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;yBAC5C,CAAC;oBACJ,CAAC;iBACF,CAAC;aACM,CAAC;QACb,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;CACF"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { FilePart, UserContent } from "ai";
|
|
2
|
+
import type { FileRef } from "skybridge/server";
|
|
3
|
+
export declare function inputFile(part: FilePart): FileRef;
|
|
4
|
+
export declare function containsInputFile(value: unknown): boolean;
|
|
5
|
+
export declare class InputFiles {
|
|
6
|
+
private readonly files;
|
|
7
|
+
private readonly paths;
|
|
8
|
+
private readonly server;
|
|
9
|
+
prepare(content: UserContent): Promise<UserContent>;
|
|
10
|
+
rewrite(input: Record<string, unknown>, fields: string[]): {
|
|
11
|
+
[x: string]: unknown;
|
|
12
|
+
};
|
|
13
|
+
close(): Promise<void>;
|
|
14
|
+
}
|
|
15
|
+
//# sourceMappingURL=input-files.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"input-files.d.ts","sourceRoot":"","sources":["../src/input-files.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,IAAI,CAAC;AAChD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAKhD,wBAAgB,SAAS,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAUjD;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAMzD;AAgCD,qBAAa,UAAU;IACrB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAGlB;IACJ,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA8B;IACpD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAQpB;IAEG,OAAO,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAwCzD,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE;;;IAuBlD,KAAK;CAMZ"}
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { once } from "node:events";
|
|
3
|
+
import { createServer } from "node:http";
|
|
4
|
+
const owners = new WeakMap();
|
|
5
|
+
const fileMatchers = new WeakSet();
|
|
6
|
+
export function inputFile(part) {
|
|
7
|
+
const label = () => `inputFile(${part.filename ?? part.mediaType})`;
|
|
8
|
+
const matcher = {
|
|
9
|
+
asymmetricMatch: (value) => typeof value === "object" && value !== null && owners.get(value) === part,
|
|
10
|
+
toString: label,
|
|
11
|
+
toJSON: label,
|
|
12
|
+
};
|
|
13
|
+
fileMatchers.add(matcher);
|
|
14
|
+
return matcher;
|
|
15
|
+
}
|
|
16
|
+
export function containsInputFile(value) {
|
|
17
|
+
return (typeof value === "object" &&
|
|
18
|
+
value !== null &&
|
|
19
|
+
(fileMatchers.has(value) || Object.values(value).some(containsInputFile)));
|
|
20
|
+
}
|
|
21
|
+
async function fileBytes(data) {
|
|
22
|
+
if (data instanceof URL) {
|
|
23
|
+
const response = await fetch(data);
|
|
24
|
+
if (!response.ok) {
|
|
25
|
+
throw new Error(`Input file download failed (HTTP ${response.status})`);
|
|
26
|
+
}
|
|
27
|
+
return Buffer.from(await response.arrayBuffer());
|
|
28
|
+
}
|
|
29
|
+
if (typeof data === "string") {
|
|
30
|
+
return data.startsWith("data:")
|
|
31
|
+
? fileBytes(new URL(data))
|
|
32
|
+
: Buffer.from(data, "base64");
|
|
33
|
+
}
|
|
34
|
+
if (data instanceof Uint8Array || data instanceof ArrayBuffer) {
|
|
35
|
+
return Buffer.from(new Uint8Array(data));
|
|
36
|
+
}
|
|
37
|
+
if (data.type === "data") {
|
|
38
|
+
return fileBytes(data.data);
|
|
39
|
+
}
|
|
40
|
+
if (data.type === "url") {
|
|
41
|
+
return fileBytes(new URL(data.url));
|
|
42
|
+
}
|
|
43
|
+
if (data.type === "text") {
|
|
44
|
+
return Buffer.from(data.text);
|
|
45
|
+
}
|
|
46
|
+
throw new Error("Input files need bytes, text, or a URL; provider-only file references cannot be downloaded by the eval app");
|
|
47
|
+
}
|
|
48
|
+
export class InputFiles {
|
|
49
|
+
files = new Map();
|
|
50
|
+
paths = new Map();
|
|
51
|
+
server = createServer((request, response) => {
|
|
52
|
+
const file = this.files.get(request.url?.slice(1) ?? "");
|
|
53
|
+
if (!file) {
|
|
54
|
+
response.writeHead(404).end();
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
response.writeHead(200, { "Content-Type": file.reference.mime_type });
|
|
58
|
+
response.end(file.bytes);
|
|
59
|
+
});
|
|
60
|
+
async prepare(content) {
|
|
61
|
+
if (typeof content === "string") {
|
|
62
|
+
return content;
|
|
63
|
+
}
|
|
64
|
+
const prepared = [];
|
|
65
|
+
for (const part of content) {
|
|
66
|
+
if (part.type !== "file") {
|
|
67
|
+
prepared.push(part);
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
const bytes = await fileBytes(part.data);
|
|
71
|
+
if (!this.server.listening) {
|
|
72
|
+
this.server.listen(0, "127.0.0.1").unref();
|
|
73
|
+
await once(this.server, "listening");
|
|
74
|
+
}
|
|
75
|
+
const { port } = this.server.address();
|
|
76
|
+
const id = randomUUID();
|
|
77
|
+
const file_id = `file_${id}`;
|
|
78
|
+
const reference = {
|
|
79
|
+
file_id,
|
|
80
|
+
download_url: `http://127.0.0.1:${port}/${file_id}`,
|
|
81
|
+
mime_type: part.mediaType,
|
|
82
|
+
...(part.filename === undefined ? {} : { file_name: part.filename }),
|
|
83
|
+
};
|
|
84
|
+
const uploaded_name = part.filename ?? "input";
|
|
85
|
+
const workspace_path = `/workspace/scratch/${id}/upload/${uploaded_name}`;
|
|
86
|
+
this.files.set(file_id, { reference, bytes });
|
|
87
|
+
this.paths.set(workspace_path, reference);
|
|
88
|
+
owners.set(reference, part);
|
|
89
|
+
prepared.push({
|
|
90
|
+
type: "text",
|
|
91
|
+
text: `Attached file: ${JSON.stringify({ uploaded_name, workspace_path })}`,
|
|
92
|
+
}, part);
|
|
93
|
+
}
|
|
94
|
+
return prepared;
|
|
95
|
+
}
|
|
96
|
+
rewrite(input, fields) {
|
|
97
|
+
const rewritten = { ...input };
|
|
98
|
+
const resolve = (value) => {
|
|
99
|
+
if (typeof value !== "string") {
|
|
100
|
+
return value;
|
|
101
|
+
}
|
|
102
|
+
const reference = this.paths.get(value);
|
|
103
|
+
if (!reference) {
|
|
104
|
+
throw new Error(`Unknown input file path: ${value}`);
|
|
105
|
+
}
|
|
106
|
+
return reference;
|
|
107
|
+
};
|
|
108
|
+
for (const field of fields) {
|
|
109
|
+
if (field in rewritten) {
|
|
110
|
+
const value = rewritten[field];
|
|
111
|
+
rewritten[field] = Array.isArray(value)
|
|
112
|
+
? value.map(resolve)
|
|
113
|
+
: resolve(value);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
return rewritten;
|
|
117
|
+
}
|
|
118
|
+
async close() {
|
|
119
|
+
if (this.server.listening) {
|
|
120
|
+
this.server.close();
|
|
121
|
+
await once(this.server, "close");
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
//# sourceMappingURL=input-files.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"input-files.js","sourceRoot":"","sources":["../src/input-files.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AACnC,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAKzC,MAAM,MAAM,GAAG,IAAI,OAAO,EAAoB,CAAC;AAC/C,MAAM,YAAY,GAAG,IAAI,OAAO,EAAU,CAAC;AAE3C,MAAM,UAAU,SAAS,CAAC,IAAc;IACtC,MAAM,KAAK,GAAG,GAAG,EAAE,CAAC,aAAa,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC;IACpE,MAAM,OAAO,GAAG;QACd,eAAe,EAAE,CAAC,KAAc,EAAE,EAAE,CAClC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI;QAC3E,QAAQ,EAAE,KAAK;QACf,MAAM,EAAE,KAAK;KACd,CAAC;IACF,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC1B,OAAO,OAA6B,CAAC;AACvC,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAc;IAC9C,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAC1E,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,IAAsB;IAC7C,IAAI,IAAI,YAAY,GAAG,EAAE,CAAC;QACxB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,oCAAoC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;QAC1E,CAAC;QACD,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;IACnD,CAAC;IACD,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;YAC7B,CAAC,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;YAC1B,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;IACD,IAAI,IAAI,YAAY,UAAU,IAAI,IAAI,YAAY,WAAW,EAAE,CAAC;QAC9D,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3C,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QACzB,OAAO,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;QACxB,OAAO,SAAS,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QACzB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IACD,MAAM,IAAI,KAAK,CACb,4GAA4G,CAC7G,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,UAAU;IACJ,KAAK,GAAG,IAAI,GAAG,EAG7B,CAAC;IACa,KAAK,GAAG,IAAI,GAAG,EAAmB,CAAC;IACnC,MAAM,GAAG,YAAY,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE;QAC3D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACzD,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;YAC9B,OAAO;QACT,CAAC;QACD,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC,CAAC;QACtE,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,KAAK,CAAC,OAAO,CAAC,OAAoB;QAChC,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YAChC,OAAO,OAAO,CAAC;QACjB,CAAC;QACD,MAAM,QAAQ,GAAiC,EAAE,CAAC;QAClD,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YAC3B,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACzB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACpB,SAAS;YACX,CAAC;YACD,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;gBAC3B,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,KAAK,EAAE,CAAC;gBAC3C,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;YACvC,CAAC;YACD,MAAM,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,EAAiB,CAAC;YACtD,MAAM,EAAE,GAAG,UAAU,EAAE,CAAC;YACxB,MAAM,OAAO,GAAG,QAAQ,EAAE,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG;gBAChB,OAAO;gBACP,YAAY,EAAE,oBAAoB,IAAI,IAAI,OAAO,EAAE;gBACnD,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,GAAG,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;aACrE,CAAC;YACF,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC;YAC/C,MAAM,cAAc,GAAG,sBAAsB,EAAE,WAAW,aAAa,EAAE,CAAC;YAC1E,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;YAC9C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;YAC1C,MAAM,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;YAC5B,QAAQ,CAAC,IAAI,CACX;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,kBAAkB,IAAI,CAAC,SAAS,CAAC,EAAE,aAAa,EAAE,cAAc,EAAE,CAAC,EAAE;aAC5E,EACD,IAAI,CACL,CAAC;QACJ,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,OAAO,CAAC,KAA8B,EAAE,MAAgB;QACtD,MAAM,SAAS,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;QAC/B,MAAM,OAAO,GAAG,CAAC,KAAc,EAAW,EAAE;YAC1C,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,OAAO,KAAK,CAAC;YACf,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACxC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,4BAA4B,KAAK,EAAE,CAAC,CAAC;YACvD,CAAC;YACD,OAAO,SAAS,CAAC;QACnB,CAAC,CAAC;QACF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,KAAK,IAAI,SAAS,EAAE,CAAC;gBACvB,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;gBAC/B,SAAS,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;oBACrC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC;oBACpB,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YACpB,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;CACF"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import "vitest";
|
|
2
|
-
import type { LanguageModel } from "ai";
|
|
3
|
-
import type { ToolInput, ToolNames } from "skybridge/server";
|
|
2
|
+
import type { FilePart, LanguageModel } from "ai";
|
|
3
|
+
import type { FileRef, ToolInput, ToolNames } from "skybridge/server";
|
|
4
4
|
import type { ChatLike } from "./types.js";
|
|
5
5
|
/** What a judge decided about a conversation. */
|
|
6
6
|
export interface Verdict {
|
|
@@ -45,6 +45,7 @@ export interface ChatAssertion<App> extends ChatMatchers<App> {
|
|
|
45
45
|
}
|
|
46
46
|
declare module "vitest" {
|
|
47
47
|
interface ExpectStatic {
|
|
48
|
+
inputFile(part: FilePart): FileRef;
|
|
48
49
|
chat<App>(chat: ChatLike<App>): ChatAssertion<App>;
|
|
49
50
|
}
|
|
50
51
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"matchers.d.ts","sourceRoot":"","sources":["../../src/matchers/matchers.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,CAAC;AAChB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"matchers.d.ts","sourceRoot":"","sources":["../../src/matchers/matchers.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,CAAC;AAChB,OAAO,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAClD,OAAO,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAGtE,OAAO,KAAK,EAAE,QAAQ,EAAkB,MAAM,YAAY,CAAC;AAE3D,iDAAiD;AACjD,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,OAAO,CAAC;IACd,iFAAiF;IACjF,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,kEAAkE;AAClE,MAAM,WAAW,QAAQ;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,2EAA2E;IAC3E,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,uDAAuD;AACvD,MAAM,WAAW,eAAe;IAC9B,4EAA4E;IAC5E,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB;;;;OAIG;IACH,KAAK,CAAC,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAC5D;AAED;;;;GAIG;AACH,MAAM,WAAW,YAAY,CAAC,GAAG;IAC/B,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAC9C,IAAI,EAAE,IAAI,EACV,IAAI,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GACnC,IAAI,CAAC;IACR,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAC9C,IAAI,EAAE,IAAI,EACV,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GAClC,IAAI,CAAC;IACR,qBAAqB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,CAAC;IACrE,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,CAAC;IACpE,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IACxC,wBAAwB,CACtB,GAAG,KAAK,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,GAAG,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,GAC9C,IAAI,CAAC;IACR,mBAAmB,IAAI,IAAI,CAAC;IAC5B,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5E;AAED,MAAM,WAAW,aAAa,CAAC,GAAG,CAAE,SAAQ,YAAY,CAAC,GAAG,CAAC;IAC3D,GAAG,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC;CACxB;AAED,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,YAAY;QACpB,SAAS,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC;QACnC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;KACpD;CACF"}
|
|
@@ -1,10 +1,19 @@
|
|
|
1
1
|
import "vitest";
|
|
2
2
|
import { expect } from "vitest";
|
|
3
|
+
import { containsInputFile, inputFile } from "../input-files.js";
|
|
3
4
|
function calls(chat) {
|
|
4
5
|
return chat.turns
|
|
5
6
|
.filter((turn) => turn.role === "tool")
|
|
6
7
|
.map((turn) => turn.call);
|
|
7
8
|
}
|
|
9
|
+
function assertFileParameters(chat, name, args = {}) {
|
|
10
|
+
const fields = chat.fileParams?.get(name) ?? [];
|
|
11
|
+
for (const [field, value] of Object.entries(args)) {
|
|
12
|
+
if (containsInputFile(value) && !fields.includes(field)) {
|
|
13
|
+
throw new Error(`expect.inputFile cannot be used for "${name}.${field}": "${field}" is not listed in openai/fileParams`);
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
}
|
|
8
17
|
function spokenTurns(chat) {
|
|
9
18
|
return chat.turns
|
|
10
19
|
.filter((turn) => turn.role === "assistant")
|
|
@@ -87,7 +96,6 @@ async function modelVerdict(chat, criteria, model) {
|
|
|
87
96
|
const { generateObject, jsonSchema } = await import("ai");
|
|
88
97
|
const { object } = await generateObject({
|
|
89
98
|
model: model ?? chat.model,
|
|
90
|
-
temperature: 0,
|
|
91
99
|
system: JUDGE_SYSTEM,
|
|
92
100
|
prompt: `Criteria:\n${criteria}\n\n<conversation>\n${transcriptFor(chat)}\n</conversation>`,
|
|
93
101
|
schema: jsonSchema({
|
|
@@ -104,6 +112,7 @@ async function modelVerdict(chat, criteria, model) {
|
|
|
104
112
|
}
|
|
105
113
|
expect.extend({
|
|
106
114
|
toHaveCalledToolOnce(received, name, args) {
|
|
115
|
+
assertFileParameters(received, name, args);
|
|
107
116
|
const matching = acceptedCalls(received, name);
|
|
108
117
|
const argsMatch = args === undefined ||
|
|
109
118
|
(matching.length === 1 &&
|
|
@@ -119,6 +128,7 @@ expect.extend({
|
|
|
119
128
|
};
|
|
120
129
|
},
|
|
121
130
|
toHaveCalledToolWith(received, name, args) {
|
|
131
|
+
assertFileParameters(received, name, args);
|
|
122
132
|
const pass = acceptedCalls(received, name).some((call) => this.equals(call.arguments, expect.objectContaining(args)));
|
|
123
133
|
return {
|
|
124
134
|
pass,
|
|
@@ -215,4 +225,5 @@ judge: ${verdict.pass ? "PASS" : "FAIL"}${verdict.reasoning === undefined ? "" :
|
|
|
215
225
|
},
|
|
216
226
|
});
|
|
217
227
|
expect.chat = (chat) => expect(chat);
|
|
228
|
+
expect.inputFile = inputFile;
|
|
218
229
|
//# sourceMappingURL=matchers.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"matchers.js","sourceRoot":"","sources":["../../src/matchers/matchers.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,CAAC;AAGhB,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;
|
|
1
|
+
{"version":3,"file":"matchers.js","sourceRoot":"","sources":["../../src/matchers/matchers.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,CAAC;AAGhB,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAgEjE,SAAS,KAAK,CAAC,IAAuB;IACpC,OAAO,IAAI,CAAC,KAAK;SACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC;SACtC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC;AAED,SAAS,oBAAoB,CAC3B,IAAuB,EACvB,IAAY,EACZ,OAAgC,EAAE;IAElC,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IAChD,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QAClD,IAAI,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACxD,MAAM,IAAI,KAAK,CACb,wCAAwC,IAAI,IAAI,KAAK,OAAO,KAAK,sCAAsC,CACxG,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,IAAuB;IAC1C,OAAO,IAAI,CAAC,KAAK;SACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,WAAW,CAAC;SAC3C,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC;AAED,SAAS,cAAc,CACrB,IAAuB,EACvB,IAAY;IAEZ,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,WAAW,CAAC,IAAuB;IAC1C,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;AACjE,CAAC;AAED,SAAS,aAAa,CACpB,IAAuB,EACvB,IAAY;IAEZ,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;AACxE,CAAC;AAED,SAAS,QAAQ,CAAC,IAAuB;IACvC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IACzB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,wBAAwB,CAAC;IAClC,CAAC;IACD,OAAO,IAAI;SACR,GAAG,CACF,CAAC,IAAuB,EAAE,KAAa,EAAE,EAAE,CACzC,KAAK,KAAK,GAAG,CAAC,KAAK,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,cAAc,IAAI,CAAC,MAAM,GAAG,EAAE,CACnI;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AAC1C,CAAC;AAED,SAAS,IAAI,CAAC,IAAuB,EAAE,IAAqB;IAC1D,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CACrC,OAAO,IAAI,KAAK,QAAQ;QACtB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;QACrE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CACpB,CAAC;AACJ,CAAC;AAED,SAAS,MAAM,CAAC,IAAuB;IACrC,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,gCAAgC,CAAC;IAC1C,CAAC;IACD,OAAO,KAAK;SACT,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,GAAG,CAAC,KAAK,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;SACzD,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,SAAS,KAAK,CAAC,IAAY;IACzB,OAAO,IAAI,CAAC,OAAO,CAAC,uBAAuB,EAAE,QAAQ,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,YAAY,GAAG;;;6KAGwJ,CAAC;AAE9K,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAE9B,SAAS,QAAQ,CAAC,IAAmB;IACnC,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QACzB,OAAO,GAAG,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;IAC7C,CAAC;IACD,MAAM,OAAO,GACX,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS;QAC5B,CAAC,CAAC,OAAO,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,EAAE;QACxD,CAAC,CAAC,aAAa,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;IACvC,OAAO,SAAS,KAAK,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,OAAO,EAAE,CAAC,EAAE,CAAC;AACxG,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,IAAI,CAAC,MAAM,GAAG,gBAAgB;QACnC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC,eAAe;QACnD,CAAC,CAAC,IAAI,CAAC;AACX,CAAC;AAED,SAAS,aAAa,CAAC,IAAuB;IAC5C,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,6BAA6B,CAAC;IACvC,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC7C,CAAC;AAED,SAAS,MAAM,CAAC,IAAuB,EAAE,OAAe;IACtD,OAAO,GAAG,OAAO;;;EAGjB,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;AACnB,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,IAAuB,EACvB,QAAgB,EAChB,KAAqB;IAErB,MAAM,EAAE,cAAc,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;IAC1D,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,cAAc,CAAC;QACtC,KAAK,EAAE,KAAK,IAAI,IAAI,CAAC,KAAK;QAC1B,MAAM,EAAE,YAAY;QACpB,MAAM,EAAE,cAAc,QAAQ,uBAAuB,aAAa,CAAC,IAAI,CAAC,mBAAmB;QAC3F,MAAM,EAAE,UAAU,CAAkC;YAClD,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;gBACzB,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;aAC9B;YACD,QAAQ,EAAE,CAAC,MAAM,EAAE,WAAW,CAAC;YAC/B,oBAAoB,EAAE,KAAK;SAC5B,CAAC;KACH,CAAC,CAAC;IACH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,MAAM,CAAC;IACZ,oBAAoB,CAClB,QAA2B,EAC3B,IAAY,EACZ,IAA8B;QAE9B,oBAAoB,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;QAC3C,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC/C,MAAM,SAAS,GACb,IAAI,KAAK,SAAS;YAClB,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACpB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxE,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC;QAEhD,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,aAAa,IAAI,kCAAkC;gBACrD,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;oBACrB,CAAC,CAAC,aAAa,IAAI,4BAA4B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,4BAA4B,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG;oBACxI,CAAC,CAAC,aAAa,IAAI,6DAA6D,QAAQ,CAAC,MAAM,wBAAwB,CAC5H;SACJ,CAAC;IACJ,CAAC;IAED,oBAAoB,CAClB,QAA2B,EAC3B,IAAY,EACZ,IAA6B;QAE7B,oBAAoB,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;QAC3C,MAAM,IAAI,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CACvD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAC3D,CAAC;QAEF,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,wBAAwB,IAAI,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG;gBAC/D,CAAC,CAAC,aAAa,IAAI,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CACpE;SACJ,CAAC;IACJ,CAAC;IAED,oBAAoB,CAAC,QAA2B,EAAE,IAAY;QAC5D,MAAM,SAAS,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;QAErE,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC;YACvB,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,MAAM,CAAC,MAAM,GAAG,CAAC;gBACf,CAAC,CAAC,2BAA2B,IAAI,eAAe;gBAChD,CAAC,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;oBACtB,CAAC,CAAC,aAAa,IAAI,mDAAmD;oBACtE,CAAC,CAAC,aAAa,IAAI,sBAAsB,SAAS,CAAC,MAAM,mBAAmB,CACjF;SACJ,CAAC;IACJ,CAAC;IAED,UAAU,CAAC,QAA2B,EAAE,IAAqB;QAC3D,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAClC,MAAM,MAAM,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAErE,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,GACE,IAAI;gBACF,CAAC,CAAC,qCAAqC,MAAM,GAAG;gBAChD,CAAC,CAAC,iCAAiC,MAAM,GAC7C;;;EAGN,MAAM,CAAC,QAAQ,CAAC,EAAE;SACf,CAAC;IACJ,CAAC;IAED,mBAAmB,CAAC,QAA2B;QAC7C,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;QAErC,OAAO;YACL,IAAI,EAAE,KAAK,KAAK,CAAC;YACjB,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,KAAK,KAAK,CAAC;gBACT,CAAC,CAAC,+CAA+C;gBACjD,CAAC,CAAC,wDAAwD,KAAK,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,GAAG,CACvG;SACJ,CAAC;IACJ,CAAC;IAED,wBAAwB,CAAC,QAA2B,EAAE,GAAG,KAAe;QACtE,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;YACzC,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;gBACzC,OAAO,IAAI,CAAC,CAAC;YACf,CAAC;YACD,IAAI,OAAO,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;gBAC7B,MAAM;YACR,CAAC;QACH,CAAC;QACD,MAAM,IAAI,GAAG,OAAO,KAAK,KAAK,CAAC,MAAM,CAAC;QACtC,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAE/D,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,YAAY,MAAM,kCAAkC;gBACtD,CAAC,CAAC,YAAY,MAAM,wBAAwB,KAAK,CAAC,OAAO,CAAC,eAAe,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CACpI;SACJ,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,cAAc,CAClB,QAA2B,EAC3B,QAAgB,EAChB,OAAyB;QAEzB,IAAI,OAAgB,CAAC;QACrB,IAAI,CAAC;YACH,OAAO,GAAG,OAAO,EAAE,KAAK;gBACtB,CAAC,CAAC,MAAM,OAAO,CAAC,KAAK,CAAC;oBAClB,QAAQ;oBACR,UAAU,EAAE,aAAa,CAAC,QAAQ,CAAC;iBACpC,CAAC;gBACJ,CAAC,CAAC,MAAM,YAAY,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;YAE3D,IAAI,OAAO,OAAO,EAAE,IAAI,KAAK,SAAS,EAAE,CAAC;gBACvC,MAAM,IAAI,KAAK,CACb,sBAAsB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,6CAA6C,CAC3F,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,sBAAsB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC/E,CAAC;QACJ,CAAC;QAED,OAAO;YACL,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,OAAO,EAAE,GAAG,EAAE,CACZ,6BAA6B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;KAC1D,QAAQ;;SAEJ,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,OAAO,CAAC,SAAS,EAAE,EAAE;SACxG,CAAC;IACJ,CAAC;IAED,qBAAqB,CAAC,QAA2B,EAAE,IAAY;QAC7D,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAEhD,OAAO;YACL,IAAI,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC;YAC3B,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACnB,CAAC,CAAC,aAAa,IAAI,+BAA+B;gBAClD,CAAC,CAAC,aAAa,IAAI,2CAA2C,QAAQ,CAAC,MAAM,SAAS,CACzF;SACJ,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,IAAI,GAAG,CAAM,IAAmB,EAAsB,EAAE,CAC7D,MAAM,CAAC,IAAI,CAAkC,CAAC;AAChD,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC"}
|
package/dist/matchers/types.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/matchers/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACxC,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7D;;;;GAIG;AACH,MAAM,MAAM,QAAQ,CAAC,GAAG,IAAI;KACzB,IAAI,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG;QACxB,IAAI,EAAE,IAAI,CAAC;QACX,SAAS,EAAE,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAChC,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB;CACF,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;AAElB;;;;GAIG;AACH,MAAM,MAAM,IAAI,CAAC,GAAG,IAChB;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAC9B;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACnC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;IAAC,MAAM,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC;AAE5D,yDAAyD;AACzD,MAAM,WAAW,QAAQ,CAAC,GAAG;IAC3B,4EAA4E;IAC5E,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/matchers/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACxC,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7D;;;;GAIG;AACH,MAAM,MAAM,QAAQ,CAAC,GAAG,IAAI;KACzB,IAAI,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG;QACxB,IAAI,EAAE,IAAI,CAAC;QACX,SAAS,EAAE,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAChC,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB;CACF,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;AAElB;;;;GAIG;AACH,MAAM,MAAM,IAAI,CAAC,GAAG,IAChB;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAC9B;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACnC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;IAAC,MAAM,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC;AAE5D,yDAAyD;AACzD,MAAM,WAAW,QAAQ,CAAC,GAAG;IAC3B,4EAA4E;IAC5E,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;IAC9B,QAAQ,CAAC,UAAU,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC;CAC9D;AAED,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,eAAe;QACvB,cAAc,EAAE,YAAY,GAAG,SAAS,CAAC;KAC1C;CACF"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@skybridge/test",
|
|
3
|
-
"version": "0.0.0-dev.
|
|
4
|
-
"description": "Eval runner for Skybridge MCP
|
|
3
|
+
"version": "0.0.0-dev.fec0253",
|
|
4
|
+
"description": "Eval runner for Skybridge MCP Apps: does a prompt reach the right tool with the right arguments",
|
|
5
5
|
"license": "ISC",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"repository": {
|
|
@@ -37,8 +37,8 @@
|
|
|
37
37
|
"typescript": "^5.9.3",
|
|
38
38
|
"vitest": "^4.1.10",
|
|
39
39
|
"zod": "^4.4.3",
|
|
40
|
-
"@skybridge/vite-plugin": "0.0.0-dev.
|
|
41
|
-
"skybridge": "0.0.0-dev.
|
|
40
|
+
"@skybridge/vite-plugin": "0.0.0-dev.fec0253",
|
|
41
|
+
"skybridge": "0.0.0-dev.fec0253"
|
|
42
42
|
},
|
|
43
43
|
"engines": {
|
|
44
44
|
"node": ">=22.23.1"
|