@runtypelabs/cli 2.59.0 → 2.59.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-0-T9NWjH.js +3899 -0
- package/dist/chunk-0x7UKC1U.js +17 -0
- package/dist/chunk-2ySxXtcg.js +418 -0
- package/dist/chunk-3An_BRHv.js +501 -0
- package/dist/chunk-6uTxgdiY.js +16218 -0
- package/dist/chunk-87nRnNRG.js +1363 -0
- package/dist/chunk-B3wo8uRx.js +322 -0
- package/dist/chunk-BGp3ZdBy.js +560 -0
- package/dist/chunk-BJQv9Elp.js +107 -0
- package/dist/chunk-BOpO6xE-.js +6053 -0
- package/dist/chunk-BQXXCL8B.js +102 -0
- package/dist/chunk-BV-8isOd.js +34 -0
- package/dist/chunk-BVMhjGVe.js +826 -0
- package/dist/chunk-BW2fg46T.js +141 -0
- package/dist/chunk-BWJdY5Ag.js +36 -0
- package/dist/chunk-BYJOrLhh.js +302 -0
- package/dist/chunk-BYab8EVL.js +127 -0
- package/dist/chunk-BfEKvp7h.js +82 -0
- package/dist/chunk-BhgLSfH6.js +1123 -0
- package/dist/chunk-BirUxTtH.js +296 -0
- package/dist/chunk-BmB6UVZ9.js +1559 -0
- package/dist/chunk-BnWhIqW1.js +109 -0
- package/dist/chunk-Bpfnfk0t.js +123 -0
- package/dist/chunk-BrhgdnM3.js +108 -0
- package/dist/chunk-Bt0XAgX5.js +327 -0
- package/dist/chunk-Bu0FJr6S.js +84 -0
- package/dist/chunk-BvEoKpN3.js +255 -0
- package/dist/chunk-BxNJgYbE.js +28 -0
- package/dist/chunk-C1OFhd7F.js +6131 -0
- package/dist/chunk-C7e7WwbU.js +331 -0
- package/dist/chunk-CDTYP6bN.js +558 -0
- package/dist/chunk-CGDQoqDC.js +24 -0
- package/dist/chunk-CJraEBVd.js +127 -0
- package/dist/chunk-CQrNhOtv.js +40 -0
- package/dist/chunk-CW6AzBH6.js +128 -0
- package/dist/chunk-CXgL4w-9.js +317 -0
- package/dist/chunk-CZrqoPyf.js +233 -0
- package/dist/chunk-CcFCyqbB.js +83 -0
- package/dist/chunk-CdaIvO-y.js +485 -0
- package/dist/chunk-CehSXtgK.js +624 -0
- package/dist/chunk-Ciqkop65.js +218 -0
- package/dist/chunk-CoAWdSiE.js +43 -0
- package/dist/chunk-Crb4y7t4.js +830 -0
- package/dist/chunk-CrlqIfmb.js +337 -0
- package/dist/chunk-Cs4pwCQ-.js +324 -0
- package/dist/chunk-CwM_9y01.js +519 -0
- package/dist/chunk-CyRbIJg8.js +440 -0
- package/dist/chunk-D0OHsyMy.js +212 -0
- package/dist/chunk-D3y2bURs.js +1347 -0
- package/dist/chunk-D4_mJk03.js +2338 -0
- package/dist/chunk-D6C8rned.js +578 -0
- package/dist/chunk-D9ZbUnks.js +11 -0
- package/dist/chunk-DEPmLshX.js +834 -0
- package/dist/chunk-DFj1UPB2.js +330 -0
- package/dist/chunk-DGdFdZVf.js +197 -0
- package/dist/chunk-DGv7X9S_.js +63 -0
- package/dist/chunk-DJQ47WYZ.js +664 -0
- package/dist/chunk-DJw3O61j.js +1974 -0
- package/dist/chunk-DK4FbWVK.js +100 -0
- package/dist/chunk-DSRYhUus.js +993 -0
- package/dist/chunk-DajZ9Eg4.js +299 -0
- package/dist/chunk-Df4dunvU.js +195 -0
- package/dist/chunk-DjNTPhfO.js +481 -0
- package/dist/chunk-Dn8eaJuu.js +1042 -0
- package/dist/chunk-DwDWqSxM.js +637 -0
- package/dist/chunk-Dz7cEtxl.js +84 -0
- package/dist/chunk-E_0bgjrS.js +359 -0
- package/dist/chunk-KPYN2vUp.js +776 -0
- package/dist/chunk-KcqPQJBi.js +86 -0
- package/dist/chunk-N9H0JnoG.js +349 -0
- package/dist/chunk-NONFcXkn.js +728 -0
- package/dist/chunk-PrFjcpmH.js +256 -0
- package/dist/chunk-QSfNYiw9.js +11 -0
- package/dist/chunk-R265bzj9.js +226 -0
- package/dist/chunk-X9V_r6jM.js +166 -0
- package/dist/chunk-ZBtPUtyA.js +55 -0
- package/dist/chunk-ZCTfmOs7.js +4939 -0
- package/dist/chunk-h6oavj0t.js +97 -0
- package/dist/chunk-i7DwkvvD.js +42 -0
- package/dist/chunk-mG-4eZ6S.js +203 -0
- package/dist/chunk-o0ygUbCf.js +313 -0
- package/dist/chunk-ucu5lrDZ.js +5715 -0
- package/dist/index.js +11367 -73764
- package/package.json +6 -6
|
@@ -0,0 +1,578 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { t as isTTY } from "./index.js";
|
|
3
|
+
import { DataList, EntityCard, MutationResult, React, Text, render, useEffect, useState } from "./chunk-CoAWdSiE.js";
|
|
4
|
+
import { r as createCliClient } from "./chunk-BW2fg46T.js";
|
|
5
|
+
import { n as ensureAuth } from "./chunk-CJraEBVd.js";
|
|
6
|
+
import { a as printJson, r as getTotalCount } from "./chunk-CcFCyqbB.js";
|
|
7
|
+
import { c as buildEntityPullCommand, s as buildEntityEnsureCommand, t as EVAL_ENSURE_SPEC } from "./chunk-BirUxTtH.js";
|
|
8
|
+
import { n as buildNestedAgentSelector } from "./chunk-BxNJgYbE.js";
|
|
9
|
+
import { Command } from "commander";
|
|
10
|
+
import chalk from "chalk";
|
|
11
|
+
import { mkdirSync, readFileSync, readdirSync, writeFileSync } from "fs";
|
|
12
|
+
import { dirname, join, relative, resolve, sep } from "path";
|
|
13
|
+
import { createJiti } from "jiti";
|
|
14
|
+
// src/lib/eval-discovery.ts
|
|
15
|
+
/** Suffixes that mark a file as an eval suite. */
|
|
16
|
+
const EVAL_FILE_SUFFIXES = [".eval.ts", ".eval.mts"];
|
|
17
|
+
/** Directories never worth walking into during discovery. */
|
|
18
|
+
const IGNORED_DIRS = /* @__PURE__ */ new Set([
|
|
19
|
+
"node_modules",
|
|
20
|
+
".git",
|
|
21
|
+
"dist",
|
|
22
|
+
"build",
|
|
23
|
+
".next",
|
|
24
|
+
".turbo",
|
|
25
|
+
"coverage",
|
|
26
|
+
".cache"
|
|
27
|
+
]);
|
|
28
|
+
function isEvalFile(name) {
|
|
29
|
+
return EVAL_FILE_SUFFIXES.some((suffix) => name.endsWith(suffix));
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Recursively find every `*.eval.ts` under `rootDir`, skipping vendored/build
|
|
33
|
+
* directories. Returns absolute paths, sorted for deterministic run order.
|
|
34
|
+
*/
|
|
35
|
+
function discoverEvalFiles(rootDir) {
|
|
36
|
+
const found = [];
|
|
37
|
+
const walk = (dir) => {
|
|
38
|
+
let entries;
|
|
39
|
+
try {
|
|
40
|
+
entries = readdirSync(dir, { withFileTypes: true });
|
|
41
|
+
} catch {
|
|
42
|
+
return;
|
|
43
|
+
}
|
|
44
|
+
for (const entry of entries) {
|
|
45
|
+
const name = entry.name.toString();
|
|
46
|
+
if (entry.isDirectory()) {
|
|
47
|
+
if (!IGNORED_DIRS.has(name) && !name.startsWith(".")) walk(join(dir, name));
|
|
48
|
+
} else if (entry.isFile() && isEvalFile(name)) found.push(join(dir, name));
|
|
49
|
+
}
|
|
50
|
+
};
|
|
51
|
+
walk(rootDir);
|
|
52
|
+
return found.sort();
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Narrow a discovered file list by an id-or-directory-prefix argument.
|
|
56
|
+
* Matches when the relative path starts with the prefix, or the basename contains it.
|
|
57
|
+
* Suite-name matching happens after loading, so this stays permissive: an unmatched prefix still loads everything.
|
|
58
|
+
*/
|
|
59
|
+
function filterEvalFilesByPrefix(files, rootDir, prefix) {
|
|
60
|
+
const normalized = prefix.replace(/\//g, sep);
|
|
61
|
+
return files.filter((file) => {
|
|
62
|
+
const rel = relative(rootDir, file);
|
|
63
|
+
const base = rel.split(sep).pop() ?? rel;
|
|
64
|
+
return rel.startsWith(normalized) || base.includes(prefix);
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
let jitiLoader;
|
|
68
|
+
/**
|
|
69
|
+
* Load a `*.eval.ts` module via jiti and return its default export, which must
|
|
70
|
+
* be the `defineEval(...)` result (an `EvalDefinition`). Validates the default
|
|
71
|
+
* export is present and structurally an eval definition; throws an actionable
|
|
72
|
+
* error otherwise (the CLI maps a load failure to exit code 2 — a config error).
|
|
73
|
+
*/
|
|
74
|
+
async function loadEvalDefinition(filePath) {
|
|
75
|
+
jitiLoader ??= createJiti(import.meta.url, { interopDefault: false });
|
|
76
|
+
let mod;
|
|
77
|
+
try {
|
|
78
|
+
mod = await jitiLoader.import(filePath);
|
|
79
|
+
} catch (error) {
|
|
80
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
81
|
+
throw new Error(`Failed to load eval file ${filePath}: ${message}`, { cause: error });
|
|
82
|
+
}
|
|
83
|
+
const exported = mod.default;
|
|
84
|
+
if (exported === void 0 || exported === null) throw new Error(`Eval file ${filePath} must have a default export: the result of defineEval({...}) from @runtypelabs/sdk.`);
|
|
85
|
+
if (typeof exported !== "object" || !("target" in exported) || !("cases" in exported) || !Array.isArray(exported.cases)) throw new Error(`Eval file ${filePath} default export is not an eval definition. Export defineEval({ target, cases, ... }).`);
|
|
86
|
+
return exported;
|
|
87
|
+
}
|
|
88
|
+
// src/lib/eval-junit.ts
|
|
89
|
+
/** XML-escape text/attribute content (the five predefined XML entities). */
|
|
90
|
+
function escapeXml(value) {
|
|
91
|
+
return value.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
92
|
+
}
|
|
93
|
+
/** Build a JUnit XML document string from per-suite case results. */
|
|
94
|
+
function buildJUnitXml(suites) {
|
|
95
|
+
const totalTests = suites.reduce((sum, s) => sum + s.cases.length, 0);
|
|
96
|
+
const totalFailures = suites.reduce((sum, s) => sum + s.cases.filter((c) => !c.passed).length, 0);
|
|
97
|
+
const lines = [];
|
|
98
|
+
lines.push("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
|
|
99
|
+
lines.push(`<testsuites name="runtype-eval" tests="${totalTests}" failures="${totalFailures}">`);
|
|
100
|
+
for (const suite of suites) {
|
|
101
|
+
const suiteFailures = suite.cases.filter((c) => !c.passed).length;
|
|
102
|
+
lines.push(` <testsuite name="${escapeXml(suite.name)}" tests="${suite.cases.length}" failures="${suiteFailures}">`);
|
|
103
|
+
for (const testCase of suite.cases) {
|
|
104
|
+
const classname = escapeXml(suite.name);
|
|
105
|
+
const caseName = escapeXml(testCase.name);
|
|
106
|
+
if (testCase.passed) lines.push(` <testcase name="${caseName}" classname="${classname}" />`);
|
|
107
|
+
else {
|
|
108
|
+
const reason = testCase.errored ? "Case errored while producing output" : "One or more graders failed";
|
|
109
|
+
const detailLines = [...testCase.errored && testCase.error ? [`error: ${testCase.error}`] : [], ...testCase.failedGraders];
|
|
110
|
+
const detail = detailLines.length ? detailLines.join("\n") : reason;
|
|
111
|
+
lines.push(` <testcase name="${caseName}" classname="${classname}">`);
|
|
112
|
+
lines.push(` <failure message="${escapeXml(reason)}">${escapeXml(detail)}</failure>`);
|
|
113
|
+
lines.push(" </testcase>");
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
lines.push(" </testsuite>");
|
|
117
|
+
}
|
|
118
|
+
lines.push("</testsuites>");
|
|
119
|
+
return lines.join("\n") + "\n";
|
|
120
|
+
}
|
|
121
|
+
// src/commands/eval.ts
|
|
122
|
+
const evalCommand = new Command("eval").description("Manage evaluations");
|
|
123
|
+
evalCommand.command("submit").description("Submit an eval batch").requiredOption("-f, --flow <id>", "Flow ID to evaluate").requiredOption("-r, --records <file>", "JSON file with record IDs").option("-n, --name <name>", "Eval batch name").option("--json", "Output as JSON").option("--tty", "Force TTY mode").option("--no-tty", "Force non-TTY mode").action(async (options) => {
|
|
124
|
+
const apiKey = await ensureAuth();
|
|
125
|
+
if (!apiKey) return;
|
|
126
|
+
let recordIds;
|
|
127
|
+
try {
|
|
128
|
+
const content = readFileSync(options.records, "utf-8");
|
|
129
|
+
const parsed = JSON.parse(content);
|
|
130
|
+
recordIds = Array.isArray(parsed) ? parsed : parsed.recordIds || parsed.records || [];
|
|
131
|
+
} catch (error) {
|
|
132
|
+
const message = error instanceof Error ? error.message : "Unknown error";
|
|
133
|
+
console.error(chalk.red(`Failed to read records file: ${message}`));
|
|
134
|
+
process.exit(1);
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
const client = createCliClient(apiKey);
|
|
138
|
+
if (!isTTY(options) || options.json) {
|
|
139
|
+
try {
|
|
140
|
+
const data = await client.post("/eval/submit", {
|
|
141
|
+
flowId: options.flow,
|
|
142
|
+
recordIds,
|
|
143
|
+
name: options.name
|
|
144
|
+
});
|
|
145
|
+
if (options.json) printJson(data);
|
|
146
|
+
else {
|
|
147
|
+
console.log(chalk.green("Eval submitted"));
|
|
148
|
+
console.log(` Batch ID: ${chalk.green(data.id)}`);
|
|
149
|
+
if (data.name) console.log(` Name: ${data.name}`);
|
|
150
|
+
console.log(` Status: ${data.status}`);
|
|
151
|
+
console.log(` Records: ${data.totalRecords}`);
|
|
152
|
+
if (data.groupId) console.log(` Group: ${data.groupId}`);
|
|
153
|
+
}
|
|
154
|
+
} catch (error) {
|
|
155
|
+
const message = error instanceof Error ? error.message : "Unknown error";
|
|
156
|
+
console.error(chalk.red("Failed to submit eval"));
|
|
157
|
+
console.error(chalk.red(message));
|
|
158
|
+
process.exit(1);
|
|
159
|
+
}
|
|
160
|
+
return;
|
|
161
|
+
}
|
|
162
|
+
const App = () => {
|
|
163
|
+
const [loading, setLoading] = useState(true);
|
|
164
|
+
const [success, setSuccess] = useState(null);
|
|
165
|
+
const [error, setError] = useState(null);
|
|
166
|
+
const [resultNode, setResultNode] = useState(void 0);
|
|
167
|
+
useEffect(() => {
|
|
168
|
+
const run = async () => {
|
|
169
|
+
try {
|
|
170
|
+
const data = await client.post("/eval/submit", {
|
|
171
|
+
flowId: options.flow,
|
|
172
|
+
recordIds,
|
|
173
|
+
name: options.name
|
|
174
|
+
});
|
|
175
|
+
const fields = [{
|
|
176
|
+
label: "Batch ID",
|
|
177
|
+
value: data.id,
|
|
178
|
+
color: "green"
|
|
179
|
+
}];
|
|
180
|
+
if (data.name) fields.push({
|
|
181
|
+
label: "Name",
|
|
182
|
+
value: data.name
|
|
183
|
+
});
|
|
184
|
+
fields.push({
|
|
185
|
+
label: "Status",
|
|
186
|
+
value: data.status
|
|
187
|
+
});
|
|
188
|
+
fields.push({
|
|
189
|
+
label: "Records",
|
|
190
|
+
value: data.totalRecords
|
|
191
|
+
});
|
|
192
|
+
if (data.groupId) fields.push({
|
|
193
|
+
label: "Group",
|
|
194
|
+
value: data.groupId
|
|
195
|
+
});
|
|
196
|
+
setResultNode(React.createElement(EntityCard, { fields }));
|
|
197
|
+
setSuccess(true);
|
|
198
|
+
setLoading(false);
|
|
199
|
+
} catch (err) {
|
|
200
|
+
setError(err instanceof Error ? err : new Error(String(err)));
|
|
201
|
+
setSuccess(false);
|
|
202
|
+
setLoading(false);
|
|
203
|
+
}
|
|
204
|
+
};
|
|
205
|
+
run();
|
|
206
|
+
}, []);
|
|
207
|
+
return React.createElement(MutationResult, {
|
|
208
|
+
loading,
|
|
209
|
+
loadingLabel: `Submitting eval with ${recordIds.length} records...`,
|
|
210
|
+
success,
|
|
211
|
+
successMessage: "Eval submitted",
|
|
212
|
+
error,
|
|
213
|
+
result: resultNode
|
|
214
|
+
});
|
|
215
|
+
};
|
|
216
|
+
const { waitUntilExit } = render(React.createElement(App));
|
|
217
|
+
await waitUntilExit();
|
|
218
|
+
});
|
|
219
|
+
evalCommand.command("list").description("List eval batches").option("--flow <id>", "Filter by flow ID").option("--limit <n>", "Limit results", "20").option("--json", "Output as JSON").option("--tty", "Force TTY mode").option("--no-tty", "Force non-TTY mode").action(async (options) => {
|
|
220
|
+
const apiKey = await ensureAuth();
|
|
221
|
+
if (!apiKey) return;
|
|
222
|
+
const client = createCliClient(apiKey);
|
|
223
|
+
const params = { limit: options.limit };
|
|
224
|
+
if (options.flow) params.flowId = options.flow;
|
|
225
|
+
if (!isTTY(options) || options.json) {
|
|
226
|
+
try {
|
|
227
|
+
const data = await client.get("/eval/batches", params);
|
|
228
|
+
if (options.json) printJson(data);
|
|
229
|
+
else {
|
|
230
|
+
const batches = data.data ?? [];
|
|
231
|
+
if (batches.length === 0) {
|
|
232
|
+
console.log(chalk.gray("No eval batches found"));
|
|
233
|
+
return;
|
|
234
|
+
}
|
|
235
|
+
console.log(chalk.cyan("Eval Batches:"));
|
|
236
|
+
for (const batch of batches) {
|
|
237
|
+
const name = batch.name || batch.id;
|
|
238
|
+
const progress = batch.totalRecords ? `${batch.completedRecords ?? 0}/${batch.totalRecords}` : "";
|
|
239
|
+
const statusColor = batch.status === "completed" ? "green" : "yellow";
|
|
240
|
+
console.log(` ${chalk.green(batch.id)} ${name} ${chalk[statusColor](`[${batch.status}]`)} ${chalk.gray(progress)}`);
|
|
241
|
+
}
|
|
242
|
+
const total = getTotalCount(data.pagination);
|
|
243
|
+
if (total !== void 0) console.log(chalk.dim(`\n Total: ${total} batches`));
|
|
244
|
+
}
|
|
245
|
+
} catch (error) {
|
|
246
|
+
const message = error instanceof Error ? error.message : "Unknown error";
|
|
247
|
+
console.error(chalk.red("Failed to fetch eval batches"));
|
|
248
|
+
console.error(chalk.red(message));
|
|
249
|
+
process.exit(1);
|
|
250
|
+
}
|
|
251
|
+
return;
|
|
252
|
+
}
|
|
253
|
+
const App = () => {
|
|
254
|
+
const [loading, setLoading] = useState(true);
|
|
255
|
+
const [items, setItems] = useState(null);
|
|
256
|
+
const [total, setTotal] = useState(void 0);
|
|
257
|
+
const [error, setError] = useState(null);
|
|
258
|
+
useEffect(() => {
|
|
259
|
+
const run = async () => {
|
|
260
|
+
try {
|
|
261
|
+
const data = await client.get("/eval/batches", params);
|
|
262
|
+
setItems(data.data ?? []);
|
|
263
|
+
setTotal(getTotalCount(data.pagination));
|
|
264
|
+
setLoading(false);
|
|
265
|
+
} catch (err) {
|
|
266
|
+
setError(err instanceof Error ? err : new Error(String(err)));
|
|
267
|
+
setLoading(false);
|
|
268
|
+
}
|
|
269
|
+
};
|
|
270
|
+
run();
|
|
271
|
+
}, []);
|
|
272
|
+
return React.createElement(DataList, {
|
|
273
|
+
title: "Eval Batches",
|
|
274
|
+
items,
|
|
275
|
+
error,
|
|
276
|
+
loading,
|
|
277
|
+
total,
|
|
278
|
+
emptyMessage: "No eval batches found",
|
|
279
|
+
renderCard: (item) => {
|
|
280
|
+
const b = item;
|
|
281
|
+
const name = b.name || b.id;
|
|
282
|
+
const progress = b.totalRecords ? `${b.completedRecords ?? 0}/${b.totalRecords}` : "";
|
|
283
|
+
const statusColor = b.status === "completed" ? "green" : "yellow";
|
|
284
|
+
return React.createElement(Text, { color: statusColor }, ` ${b.id} ${name} [${b.status}] ${progress}`);
|
|
285
|
+
}
|
|
286
|
+
});
|
|
287
|
+
};
|
|
288
|
+
const { waitUntilExit } = render(React.createElement(App));
|
|
289
|
+
await waitUntilExit();
|
|
290
|
+
});
|
|
291
|
+
evalCommand.command("results <id>").description("Get eval batch results").option("--json", "Output as JSON").option("--tty", "Force TTY mode").option("--no-tty", "Force non-TTY mode").action(async (id, options) => {
|
|
292
|
+
const apiKey = await ensureAuth();
|
|
293
|
+
if (!apiKey) return;
|
|
294
|
+
const client = createCliClient(apiKey);
|
|
295
|
+
if (!isTTY(options) || options.json) {
|
|
296
|
+
try {
|
|
297
|
+
const data = await client.get(`/eval/${id}/results`);
|
|
298
|
+
if (options.json) printJson(data);
|
|
299
|
+
else {
|
|
300
|
+
if (data.batch) {
|
|
301
|
+
console.log(chalk.cyan(`Eval: ${data.batch.name || data.batch.id}`));
|
|
302
|
+
console.log(` Status: ${data.batch.status}`);
|
|
303
|
+
console.log(` Progress: ${data.batch.completedRecords ?? 0}/${data.batch.totalRecords ?? 0}`);
|
|
304
|
+
console.log();
|
|
305
|
+
}
|
|
306
|
+
const results = data.data ?? [];
|
|
307
|
+
if (results.length === 0) {
|
|
308
|
+
console.log(chalk.gray("No results yet"));
|
|
309
|
+
return;
|
|
310
|
+
}
|
|
311
|
+
console.log(chalk.cyan("Results:"));
|
|
312
|
+
for (const result of results) {
|
|
313
|
+
const scoreStr = result.score !== void 0 ? chalk.blue(` score=${result.score}`) : "";
|
|
314
|
+
const statusColor = result.status === "completed" ? "green" : "red";
|
|
315
|
+
console.log(` ${chalk.gray(result.recordId)} ${chalk[statusColor](`[${result.status}]`)}${scoreStr}`);
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
} catch (error) {
|
|
319
|
+
const message = error instanceof Error ? error.message : "Unknown error";
|
|
320
|
+
console.error(chalk.red("Failed to fetch eval results"));
|
|
321
|
+
console.error(chalk.red(message));
|
|
322
|
+
process.exit(1);
|
|
323
|
+
}
|
|
324
|
+
return;
|
|
325
|
+
}
|
|
326
|
+
const App = () => {
|
|
327
|
+
const [loading, setLoading] = useState(true);
|
|
328
|
+
const [success, setSuccess] = useState(null);
|
|
329
|
+
const [error, setError] = useState(null);
|
|
330
|
+
const [resultNode, setResultNode] = useState(void 0);
|
|
331
|
+
useEffect(() => {
|
|
332
|
+
const run = async () => {
|
|
333
|
+
try {
|
|
334
|
+
const data = await client.get(`/eval/${id}/results`);
|
|
335
|
+
const results = data.data ?? [];
|
|
336
|
+
const fields = [];
|
|
337
|
+
if (data.batch) {
|
|
338
|
+
fields.push({
|
|
339
|
+
label: "Eval",
|
|
340
|
+
value: data.batch.name || data.batch.id
|
|
341
|
+
});
|
|
342
|
+
fields.push({
|
|
343
|
+
label: "Status",
|
|
344
|
+
value: data.batch.status
|
|
345
|
+
});
|
|
346
|
+
fields.push({
|
|
347
|
+
label: "Progress",
|
|
348
|
+
value: `${data.batch.completedRecords ?? 0}/${data.batch.totalRecords ?? 0}`
|
|
349
|
+
});
|
|
350
|
+
}
|
|
351
|
+
fields.push({
|
|
352
|
+
label: "Results",
|
|
353
|
+
value: results.length
|
|
354
|
+
});
|
|
355
|
+
if (results.length > 0) {
|
|
356
|
+
const completed = results.filter((r) => r.status === "completed").length;
|
|
357
|
+
const avgScore = results.filter((r) => r.score !== void 0).reduce((sum, r) => sum + (r.score ?? 0), 0) / (results.filter((r) => r.score !== void 0).length || 1);
|
|
358
|
+
fields.push({
|
|
359
|
+
label: "Completed",
|
|
360
|
+
value: `${completed}/${results.length}`,
|
|
361
|
+
color: "green"
|
|
362
|
+
});
|
|
363
|
+
if (results.some((r) => r.score !== void 0)) fields.push({
|
|
364
|
+
label: "Avg Score",
|
|
365
|
+
value: avgScore.toFixed(2)
|
|
366
|
+
});
|
|
367
|
+
}
|
|
368
|
+
setResultNode(React.createElement(EntityCard, { fields }));
|
|
369
|
+
setSuccess(true);
|
|
370
|
+
setLoading(false);
|
|
371
|
+
} catch (err) {
|
|
372
|
+
setError(err instanceof Error ? err : new Error(String(err)));
|
|
373
|
+
setSuccess(false);
|
|
374
|
+
setLoading(false);
|
|
375
|
+
}
|
|
376
|
+
};
|
|
377
|
+
run();
|
|
378
|
+
}, []);
|
|
379
|
+
return React.createElement(MutationResult, {
|
|
380
|
+
loading,
|
|
381
|
+
loadingLabel: "Fetching eval results...",
|
|
382
|
+
success,
|
|
383
|
+
successMessage: "Eval Results",
|
|
384
|
+
error,
|
|
385
|
+
result: resultNode
|
|
386
|
+
});
|
|
387
|
+
};
|
|
388
|
+
const { waitUntilExit } = render(React.createElement(App));
|
|
389
|
+
await waitUntilExit();
|
|
390
|
+
});
|
|
391
|
+
evalCommand.command("compare <groupId>").description("Compare evals in a group").option("--json", "Output as JSON").option("--tty", "Force TTY mode").option("--no-tty", "Force non-TTY mode").action(async (groupId, options) => {
|
|
392
|
+
const apiKey = await ensureAuth();
|
|
393
|
+
if (!apiKey) return;
|
|
394
|
+
const client = createCliClient(apiKey);
|
|
395
|
+
if (!isTTY(options) || options.json) {
|
|
396
|
+
try {
|
|
397
|
+
const data = await client.post("/eval/compare", { groupId });
|
|
398
|
+
printJson(data);
|
|
399
|
+
} catch (error) {
|
|
400
|
+
const message = error instanceof Error ? error.message : "Unknown error";
|
|
401
|
+
console.error(chalk.red("Failed to compare evals"));
|
|
402
|
+
console.error(chalk.red(message));
|
|
403
|
+
process.exit(1);
|
|
404
|
+
}
|
|
405
|
+
return;
|
|
406
|
+
}
|
|
407
|
+
const App = () => {
|
|
408
|
+
const [loading, setLoading] = useState(true);
|
|
409
|
+
const [success, setSuccess] = useState(null);
|
|
410
|
+
const [error, setError] = useState(null);
|
|
411
|
+
useEffect(() => {
|
|
412
|
+
const run = async () => {
|
|
413
|
+
try {
|
|
414
|
+
const data = await client.post("/eval/compare", { groupId });
|
|
415
|
+
printJson(data);
|
|
416
|
+
setSuccess(true);
|
|
417
|
+
setLoading(false);
|
|
418
|
+
} catch (err) {
|
|
419
|
+
setError(err instanceof Error ? err : new Error(String(err)));
|
|
420
|
+
setSuccess(false);
|
|
421
|
+
setLoading(false);
|
|
422
|
+
}
|
|
423
|
+
};
|
|
424
|
+
run();
|
|
425
|
+
}, []);
|
|
426
|
+
return React.createElement(MutationResult, {
|
|
427
|
+
loading,
|
|
428
|
+
loadingLabel: "Comparing evals...",
|
|
429
|
+
success,
|
|
430
|
+
successMessage: "Comparison complete",
|
|
431
|
+
error
|
|
432
|
+
});
|
|
433
|
+
};
|
|
434
|
+
const { waitUntilExit } = render(React.createElement(App));
|
|
435
|
+
await waitUntilExit();
|
|
436
|
+
});
|
|
437
|
+
/** Exit codes mirror eve: 0 = all pass, 1 = a gate failed, 2 = config error. */
|
|
438
|
+
const EVAL_RUN_EXIT = {
|
|
439
|
+
pass: 0,
|
|
440
|
+
fail: 1,
|
|
441
|
+
config: 2
|
|
442
|
+
};
|
|
443
|
+
/** Print a config error and exit 2 (no `.eval.ts`, load/ensure/run failure, auth). */
|
|
444
|
+
function failConfig(message) {
|
|
445
|
+
console.error(chalk.red(`Error: ${message}`));
|
|
446
|
+
process.exit(EVAL_RUN_EXIT.config);
|
|
447
|
+
}
|
|
448
|
+
function printSuiteResult(rootDir, outcome) {
|
|
449
|
+
const { definition, result } = outcome;
|
|
450
|
+
const rel = relative(rootDir, outcome.file) || outcome.file;
|
|
451
|
+
const badge = result.passed ? chalk.green("PASS") : chalk.red("FAIL");
|
|
452
|
+
const score = `${result.passedCases}/${result.totalCases}`;
|
|
453
|
+
const runRef = result.runId ? ` · run ${result.runId}` : "";
|
|
454
|
+
console.log(`${badge} ${chalk.bold(definition.name)} ${chalk.gray(`(${score} cases · ${rel}${runRef})`)}`);
|
|
455
|
+
for (const testCase of result.cases) {
|
|
456
|
+
const caseBadge = testCase.passed ? chalk.green(" ✓") : chalk.red(" ✗");
|
|
457
|
+
console.log(`${caseBadge} ${testCase.name}`);
|
|
458
|
+
if (!testCase.passed) {
|
|
459
|
+
for (const outcomeItem of testCase.outcomes.filter((o) => !o.passed)) {
|
|
460
|
+
const reason = outcomeItem.reasoning ? `: ${outcomeItem.reasoning}` : "";
|
|
461
|
+
console.log(chalk.red(` ✗ ${outcomeItem.kind}${reason}`));
|
|
462
|
+
}
|
|
463
|
+
if (testCase.errored) {
|
|
464
|
+
const cause = testCase.error ?? testCase.outputExcerpt;
|
|
465
|
+
console.log(chalk.red(` ✗ errored: ${cause.slice(0, 200)}`));
|
|
466
|
+
}
|
|
467
|
+
} else for (const outcomeItem of testCase.outcomes.filter((o) => !o.passed && o.severity === "soft")) {
|
|
468
|
+
const reason = outcomeItem.reasoning ? `: ${outcomeItem.reasoning}` : "";
|
|
469
|
+
console.log(chalk.yellow(` ⚠ soft ${outcomeItem.kind}${reason}`));
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
function toJUnitSuite(outcome) {
|
|
474
|
+
return {
|
|
475
|
+
name: outcome.definition.name,
|
|
476
|
+
cases: outcome.result.cases.map((c) => ({
|
|
477
|
+
name: c.name,
|
|
478
|
+
passed: c.passed,
|
|
479
|
+
errored: c.errored,
|
|
480
|
+
...c.error ? { error: c.error } : {},
|
|
481
|
+
failedGraders: c.outcomes.filter((o) => !o.passed).map((o) => `${o.kind}${o.reasoning ? `: ${o.reasoning}` : ""}`)
|
|
482
|
+
}))
|
|
483
|
+
};
|
|
484
|
+
}
|
|
485
|
+
evalCommand.command("run [idOrDirPrefix]").description("Run code-colocated eval suites (**/*.eval.ts) as a CI gate (exit 0 pass / 1 fail / 2 config)").option("--strict", "Fail the exit code on soft grader misses too (default: soft misses are reported but do not fail)").option("--virtual", "Run inline without persisting a suite/batch to the dashboard").option("--junit <path>", "Write JUnit XML results to <path>").option("--url <api>", "Override the API base URL (e.g. staging)").option("--cwd <dir>", "Directory to discover *.eval.ts under (default: current directory)").option("--alias <name>", "Run an Agent-target suite against whichever version this release alias points at").option("--version-id <id>", "Run an Agent-target suite against this exact version snapshot").option("--overwrite-managed", "Converge each suite even when its last write came from the dashboard or API (replaces edited ensure-owned cases)").action(async (idOrDirPrefix, options) => {
|
|
486
|
+
const apiKey = await ensureAuth();
|
|
487
|
+
if (!apiKey) failConfig("Not authenticated. Set RUNTYPE_API_KEY or run `runtype login`.");
|
|
488
|
+
let agentSelector = {};
|
|
489
|
+
try {
|
|
490
|
+
agentSelector = buildNestedAgentSelector(options);
|
|
491
|
+
} catch (error) {
|
|
492
|
+
failConfig(error instanceof Error ? error.message : String(error));
|
|
493
|
+
}
|
|
494
|
+
const rootDir = resolve(options.cwd ?? process.cwd());
|
|
495
|
+
const allFiles = discoverEvalFiles(rootDir);
|
|
496
|
+
if (allFiles.length === 0) failConfig(`No *.eval.ts files found under ${rootDir}.`);
|
|
497
|
+
const loadAll = async (files) => {
|
|
498
|
+
const loaded = [];
|
|
499
|
+
for (const file of files) try {
|
|
500
|
+
loaded.push({
|
|
501
|
+
file,
|
|
502
|
+
def: await loadEvalDefinition(file)
|
|
503
|
+
});
|
|
504
|
+
} catch (error) {
|
|
505
|
+
failConfig(error instanceof Error ? error.message : String(error));
|
|
506
|
+
}
|
|
507
|
+
return loaded;
|
|
508
|
+
};
|
|
509
|
+
let loaded;
|
|
510
|
+
if (idOrDirPrefix) {
|
|
511
|
+
const byPath = filterEvalFilesByPrefix(allFiles, rootDir, idOrDirPrefix);
|
|
512
|
+
loaded = byPath.length > 0 ? await loadAll(byPath) : [];
|
|
513
|
+
if (loaded.length === 0) loaded = (await loadAll(allFiles)).filter(({ def }) => def.name === idOrDirPrefix || def.name.includes(idOrDirPrefix));
|
|
514
|
+
if (loaded.length === 0) failConfig(`No eval suites matched "${idOrDirPrefix}".`);
|
|
515
|
+
} else loaded = await loadAll(allFiles);
|
|
516
|
+
const client = createCliClient(apiKey, options.url);
|
|
517
|
+
const outcomes = [];
|
|
518
|
+
for (const { file, def } of loaded) {
|
|
519
|
+
const runVirtual = options.virtual || def.virtual;
|
|
520
|
+
try {
|
|
521
|
+
let result;
|
|
522
|
+
if (runVirtual) result = await client.post("/eval/run", {
|
|
523
|
+
definition: def,
|
|
524
|
+
strict: options.strict ?? false,
|
|
525
|
+
...agentSelector
|
|
526
|
+
});
|
|
527
|
+
else {
|
|
528
|
+
const ensured = await client.post("/eval/ensure", {
|
|
529
|
+
name: def.name,
|
|
530
|
+
definition: def,
|
|
531
|
+
...options.overwriteManaged ? { onConflict: "overwrite" } : {}
|
|
532
|
+
});
|
|
533
|
+
if (!ensured.suiteId) throw new Error(`ensure did not return a suiteId (result: ${ensured.result})`);
|
|
534
|
+
result = await client.post("/eval/run", {
|
|
535
|
+
suiteId: ensured.suiteId,
|
|
536
|
+
strict: options.strict ?? false,
|
|
537
|
+
...agentSelector
|
|
538
|
+
});
|
|
539
|
+
}
|
|
540
|
+
outcomes.push({
|
|
541
|
+
file,
|
|
542
|
+
definition: def,
|
|
543
|
+
result
|
|
544
|
+
});
|
|
545
|
+
printSuiteResult(rootDir, {
|
|
546
|
+
file,
|
|
547
|
+
definition: def,
|
|
548
|
+
result
|
|
549
|
+
});
|
|
550
|
+
} catch (error) {
|
|
551
|
+
failConfig(`Suite "${def.name}" (${relative(rootDir, file) || file}): ` + (error instanceof Error ? error.message : String(error)));
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
if (options.junit) try {
|
|
555
|
+
const xml = buildJUnitXml(outcomes.map(toJUnitSuite));
|
|
556
|
+
const outPath = resolve(options.junit);
|
|
557
|
+
mkdirSync(dirname(outPath), { recursive: true });
|
|
558
|
+
writeFileSync(outPath, xml, "utf-8");
|
|
559
|
+
console.log(chalk.gray(`JUnit results written to ${options.junit}`));
|
|
560
|
+
} catch (error) {
|
|
561
|
+
failConfig(`Failed to write JUnit report: ${error instanceof Error ? error.message : String(error)}`);
|
|
562
|
+
}
|
|
563
|
+
const failedSuites = outcomes.filter((o) => !o.result.passed);
|
|
564
|
+
const totalCases = outcomes.reduce((sum, o) => sum + o.result.totalCases, 0);
|
|
565
|
+
const passedCases = outcomes.reduce((sum, o) => sum + o.result.passedCases, 0);
|
|
566
|
+
console.log();
|
|
567
|
+
if (failedSuites.length === 0) {
|
|
568
|
+
console.log(chalk.green(`All ${outcomes.length} suite(s) passed (${passedCases}/${totalCases} cases).`));
|
|
569
|
+
process.exit(EVAL_RUN_EXIT.pass);
|
|
570
|
+
} else {
|
|
571
|
+
console.log(chalk.red(`${failedSuites.length} of ${outcomes.length} suite(s) failed (${passedCases}/${totalCases} cases passed).`));
|
|
572
|
+
process.exit(EVAL_RUN_EXIT.fail);
|
|
573
|
+
}
|
|
574
|
+
});
|
|
575
|
+
evalCommand.addCommand(buildEntityEnsureCommand(EVAL_ENSURE_SPEC));
|
|
576
|
+
evalCommand.addCommand(buildEntityPullCommand(EVAL_ENSURE_SPEC));
|
|
577
|
+
export { evalCommand };
|
|
578
|
+
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { D as _isoDate, O as _isoDateTime, n as ZodISODateTime, t as ZodISODate } from "./chunk-BOpO6xE-.js";
|
|
3
|
+
// ../../node_modules/.pnpm/zod@4.6.7/node_modules/zod/v4/classic/iso.js
|
|
4
|
+
function datetime(params) {
|
|
5
|
+
return _isoDateTime(ZodISODateTime, params);
|
|
6
|
+
}
|
|
7
|
+
function date(params) {
|
|
8
|
+
return _isoDate(ZodISODate, params);
|
|
9
|
+
}
|
|
10
|
+
export { datetime as n, date as t };
|
|
11
|
+
|