@leostera/evalkit 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/cli.mjs +216 -106
- package/dist/cli.mjs.map +1 -1
- package/dist/dashboard/assets/{index-BYSEeNaZ.js → index-Z6raM-uW.js} +10 -10
- package/dist/dashboard/index.html +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -21,6 +21,7 @@ bun run check # type-check the project
|
|
|
21
21
|
bun run evals # both configured styles
|
|
22
22
|
bun run matrix:plan # inspect the matrix without executing it
|
|
23
23
|
bun run dashboard # in a second terminal
|
|
24
|
+
bun run dashboard:build # optional read-only snapshot to share
|
|
24
25
|
```
|
|
25
26
|
|
|
26
27
|
Already have a Bun project? From its root you can instead run:
|
package/dist/cli.mjs
CHANGED
|
@@ -3,7 +3,7 @@ import { F as defineEvalMatrix, I as canonicalParameters, m as registerEvals, n
|
|
|
3
3
|
import { a as readTrialEvents, c as readTrialSummary, i as readRunSummary, l as localReportStore, n as runMatrix, o as readTrialManifest, r as readRunManifest, t as runEval } from "./src-DluldGKn.mjs";
|
|
4
4
|
import * as Schema from "effect/Schema";
|
|
5
5
|
import { Effect, Option } from "effect";
|
|
6
|
-
import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises";
|
|
6
|
+
import { cp, mkdir, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
|
|
7
7
|
import * as Path from "node:path";
|
|
8
8
|
import { basename, dirname, extname, relative, resolve, sep } from "node:path";
|
|
9
9
|
import * as OS from "node:os";
|
|
@@ -7775,7 +7775,7 @@ const ProcessProto = {
|
|
|
7775
7775
|
};
|
|
7776
7776
|
}
|
|
7777
7777
|
};
|
|
7778
|
-
const runCommand = (fileSystem) => (command) => {
|
|
7778
|
+
const runCommand$1 = (fileSystem) => (command) => {
|
|
7779
7779
|
switch (command._tag) {
|
|
7780
7780
|
case "StandardCommand": {
|
|
7781
7781
|
const spawn = Effect$1.flatMap(Deferred.make(), (exitCode) => Effect$1.async((resume) => {
|
|
@@ -7851,13 +7851,13 @@ const runCommand = (fileSystem) => (command) => {
|
|
|
7851
7851
|
}
|
|
7852
7852
|
case "PipedCommand": {
|
|
7853
7853
|
const flattened = flatten(command);
|
|
7854
|
-
if (flattened.length === 1) return pipe(flattened[0], runCommand(fileSystem));
|
|
7854
|
+
if (flattened.length === 1) return pipe(flattened[0], runCommand$1(fileSystem));
|
|
7855
7855
|
const head = flattened[0];
|
|
7856
7856
|
const tail = flattened.slice(1);
|
|
7857
7857
|
const initial = tail.slice(0, tail.length - 1);
|
|
7858
7858
|
const last = tail[tail.length - 1];
|
|
7859
|
-
const stream = initial.reduce((stdin$2, command) => pipe(stdin(command, stdin$2), runCommand(fileSystem), Effect$1.map((process) => process.stdout), Stream.unwrapScoped), pipe(runCommand(fileSystem)(head), Effect$1.map((process) => process.stdout), Stream.unwrapScoped));
|
|
7860
|
-
return pipe(stdin(last, stream), runCommand(fileSystem));
|
|
7859
|
+
const stream = initial.reduce((stdin$2, command) => pipe(stdin(command, stdin$2), runCommand$1(fileSystem), Effect$1.map((process) => process.stdout), Stream.unwrapScoped), pipe(runCommand$1(fileSystem)(head), Effect$1.map((process) => process.stdout), Stream.unwrapScoped));
|
|
7860
|
+
return pipe(stdin(last, stream), runCommand$1(fileSystem));
|
|
7861
7861
|
}
|
|
7862
7862
|
}
|
|
7863
7863
|
};
|
|
@@ -7867,7 +7867,7 @@ const runCommand = (fileSystem) => (command) => {
|
|
|
7867
7867
|
* @since 1.0.0
|
|
7868
7868
|
* @category layer
|
|
7869
7869
|
*/
|
|
7870
|
-
const layer$5 = /* @__PURE__ */ Layer.effect(CommandExecutor, /*#__PURE__*/ pipe(FileSystem, /*#__PURE__*/ Effect$1.map((fileSystem) => makeExecutor(runCommand(fileSystem)))));
|
|
7870
|
+
const layer$5 = /* @__PURE__ */ Layer.effect(CommandExecutor, /*#__PURE__*/ pipe(FileSystem, /*#__PURE__*/ Effect$1.map((fileSystem) => makeExecutor(runCommand$1(fileSystem)))));
|
|
7871
7871
|
//#endregion
|
|
7872
7872
|
//#region ../../node_modules/.bun/@effect+platform-node-shared@0.61.1+1edbab697a2570fc/node_modules/@effect/platform-node-shared/dist/esm/internal/path.js
|
|
7873
7873
|
const fromFileUrl = (url) => Effect$1.try({
|
|
@@ -8048,63 +8048,6 @@ const layerManager = /* @__PURE__ */ Layer.provide(layerManager$2, layerWorker);
|
|
|
8048
8048
|
*/
|
|
8049
8049
|
const layer = /*#__PURE__*/ pipe(/*#__PURE__*/ Layer.mergeAll(layer$3, layer$5, layer$1, layerManager), /*#__PURE__*/ Layer.provideMerge(layer$7));
|
|
8050
8050
|
//#endregion
|
|
8051
|
-
//#region ../cli/src/dashboard-cells.ts
|
|
8052
|
-
const MAX_SCAN = 1e5;
|
|
8053
|
-
function display(value) {
|
|
8054
|
-
return typeof value === "string" ? value : JSON.stringify(value) ?? String(value);
|
|
8055
|
-
}
|
|
8056
|
-
/** Bounded report projection; both count and rows come from the core eligible-cell iterator. */
|
|
8057
|
-
function listMatrixCells(matrix, input) {
|
|
8058
|
-
if (!Number.isSafeInteger(input.offset) || input.offset < 0 || !Number.isSafeInteger(input.limit) || input.limit < 1 || input.limit > 100) throw new Error("Matrix page requires offset >= 0 and limit between 1 and 100");
|
|
8059
|
-
if (![
|
|
8060
|
-
"eval",
|
|
8061
|
-
"agent",
|
|
8062
|
-
"trials",
|
|
8063
|
-
...[...Object.keys(matrix.cases?.[0] ?? {}), ...Object.keys(matrix.parameters)].map((key) => `axis:${key}`)
|
|
8064
|
-
].includes(input.sort)) throw new Error(`Unknown matrix sort: ${input.sort}`);
|
|
8065
|
-
const selection = input.evalIds.length ? { evals: input.evalIds } : {};
|
|
8066
|
-
const planned = matrix.count(selection);
|
|
8067
|
-
if (planned > MAX_SCAN) throw new Error(`Matrix has ${planned} eligible cells; narrow the selection in the CLI (dashboard limit ${MAX_SCAN})`);
|
|
8068
|
-
const terms = input.query.trim().toLowerCase().split(/\s+/).filter(Boolean);
|
|
8069
|
-
const cells = [...matrix.cells(selection)].filter((cell) => {
|
|
8070
|
-
const haystack = [
|
|
8071
|
-
cell.eval.id,
|
|
8072
|
-
cell.eval.name,
|
|
8073
|
-
cell.eval.agent?.identity?.name,
|
|
8074
|
-
cell.eval.agent?.identity?.kind,
|
|
8075
|
-
...Object.entries(cell.parameters).flatMap(([key, value]) => [key, display(value)])
|
|
8076
|
-
].map(display).join(" ").toLowerCase();
|
|
8077
|
-
return terms.every((term) => haystack.includes(term));
|
|
8078
|
-
});
|
|
8079
|
-
function value(cell) {
|
|
8080
|
-
if (input.sort.startsWith("axis:")) {
|
|
8081
|
-
const parameter = cell.parameters[input.sort.slice(5)];
|
|
8082
|
-
return typeof parameter === "number" ? parameter : canonicalParameters(parameter);
|
|
8083
|
-
}
|
|
8084
|
-
if (input.sort === "eval") return cell.eval.name ?? cell.eval.id;
|
|
8085
|
-
if (input.sort === "agent") return cell.eval.agent?.identity?.name ?? cell.eval.agent?.identity?.kind ?? "";
|
|
8086
|
-
return cell.eval.policy?.trials ?? 1;
|
|
8087
|
-
}
|
|
8088
|
-
cells.sort((left, right) => {
|
|
8089
|
-
const a = value(left);
|
|
8090
|
-
const b = value(right);
|
|
8091
|
-
const comparison = typeof a === "number" && typeof b === "number" ? a - b : String(a).localeCompare(String(b), void 0, {
|
|
8092
|
-
numeric: true,
|
|
8093
|
-
sensitivity: "base"
|
|
8094
|
-
});
|
|
8095
|
-
return (input.direction === "asc" ? comparison : -comparison) || left.key.localeCompare(right.key);
|
|
8096
|
-
});
|
|
8097
|
-
return {
|
|
8098
|
-
total: cells.length,
|
|
8099
|
-
planned,
|
|
8100
|
-
cells: cells.slice(input.offset, input.offset + input.limit).map((cell) => ({
|
|
8101
|
-
evalId: cell.eval.id,
|
|
8102
|
-
key: cell.key,
|
|
8103
|
-
parameters: cell.parameters
|
|
8104
|
-
}))
|
|
8105
|
-
};
|
|
8106
|
-
}
|
|
8107
|
-
//#endregion
|
|
8108
8051
|
//#region ../cli/src/dashboard-reports.ts
|
|
8109
8052
|
const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
8110
8053
|
function isMissing(error) {
|
|
@@ -8259,6 +8202,147 @@ async function walkArtifacts(root) {
|
|
|
8259
8202
|
return result;
|
|
8260
8203
|
}
|
|
8261
8204
|
//#endregion
|
|
8205
|
+
//#region ../cli/src/dashboard-build.ts
|
|
8206
|
+
/** Marks a directory as EvalKit-owned so a rebuild may safely replace it. */
|
|
8207
|
+
const MARKER = ".evalkit-dashboard";
|
|
8208
|
+
/**
|
|
8209
|
+
* Write a read-only static dashboard: the bundled UI plus JSON snapshots of
|
|
8210
|
+
* the catalog and local reports, and each trial's candidate artifact files.
|
|
8211
|
+
* Evaluator workspaces are never copied; reports may still contain sensitive
|
|
8212
|
+
* messages, scores, or evidence, so review the output before sharing it.
|
|
8213
|
+
*/
|
|
8214
|
+
async function buildDashboard(options) {
|
|
8215
|
+
const { registry, reportRoot, dashboardRoot, outDir } = options;
|
|
8216
|
+
await prepareOutDir(outDir);
|
|
8217
|
+
await cp(dashboardRoot, outDir, { recursive: true });
|
|
8218
|
+
await writeFile(resolve(outDir, MARKER), "Generated by `evalkit build`.\n");
|
|
8219
|
+
await writeFile(resolve(outDir, "index.html"), staticIndex(await readFile(resolve(dashboardRoot, "index.html"), "utf8")));
|
|
8220
|
+
const data = resolve(outDir, "data");
|
|
8221
|
+
const write = async (path, value) => {
|
|
8222
|
+
const file = resolve(data, path);
|
|
8223
|
+
await mkdir(dirname(file), { recursive: true });
|
|
8224
|
+
await writeFile(file, `${JSON.stringify(value)}\n`);
|
|
8225
|
+
};
|
|
8226
|
+
const reports = createDashboardReportReader(reportRoot, registry);
|
|
8227
|
+
const runs = await reports.listRuns();
|
|
8228
|
+
await write("catalog.json", { evals: registry.catalog() });
|
|
8229
|
+
await write("evals.json", { evals: registry.metadata() });
|
|
8230
|
+
await write("suites.json", { suites: registry.suiteMetadata() });
|
|
8231
|
+
await write("runs.json", { runs });
|
|
8232
|
+
let trials = 0;
|
|
8233
|
+
let files = 0;
|
|
8234
|
+
for (const run of runs) {
|
|
8235
|
+
const runTrials = await reports.listTrials(run.id);
|
|
8236
|
+
await write(`runs/${run.id}/trials.json`, { trials: runTrials });
|
|
8237
|
+
for (const trial of runTrials) {
|
|
8238
|
+
trials += 1;
|
|
8239
|
+
const base = `runs/${run.id}/trials/${trial.id}`;
|
|
8240
|
+
await write(`${base}/detail.json`, await reports.trialDetail(run.id, trial.id));
|
|
8241
|
+
await write(`${base}/events.json`, { events: await reports.trajectory(run.id, trial.id) });
|
|
8242
|
+
await write(`${base}/artifacts.json`, { artifacts: await reports.listArtifacts(run.id, trial.id) });
|
|
8243
|
+
const workspace = await reports.listCandidateWorkspace(run.id, trial.id);
|
|
8244
|
+
await write(`${base}/workspace.json`, workspace);
|
|
8245
|
+
for (const entry of workspace) {
|
|
8246
|
+
if (entry.kind !== "file") continue;
|
|
8247
|
+
const response = await reports.candidateFile(run.id, trial.id, entry.path);
|
|
8248
|
+
const target = resolve(data, base, "workspace", entry.path);
|
|
8249
|
+
const root = resolve(data, base, "workspace");
|
|
8250
|
+
if (!target.startsWith(`${root}${sep}`)) throw new Error(`Invalid workspace path: ${entry.path}`);
|
|
8251
|
+
await mkdir(dirname(target), { recursive: true });
|
|
8252
|
+
await writeFile(target, new Uint8Array(await response.arrayBuffer()));
|
|
8253
|
+
files += 1;
|
|
8254
|
+
}
|
|
8255
|
+
}
|
|
8256
|
+
}
|
|
8257
|
+
return {
|
|
8258
|
+
outDir,
|
|
8259
|
+
runs: runs.length,
|
|
8260
|
+
trials,
|
|
8261
|
+
files
|
|
8262
|
+
};
|
|
8263
|
+
}
|
|
8264
|
+
/** Use relative asset URLs and select the static API in the browser bundle. */
|
|
8265
|
+
function staticIndex(html) {
|
|
8266
|
+
const result = html.replace("<html lang=\"en\">", "<html lang=\"en\" data-evalkit-snapshot=\"static\">").replaceAll("\"/assets/", "\"./assets/");
|
|
8267
|
+
if (!result.includes("data-evalkit-snapshot=\"static\"")) throw new Error("Bundled dashboard index.html has an unexpected shape");
|
|
8268
|
+
return result;
|
|
8269
|
+
}
|
|
8270
|
+
async function prepareOutDir(outDir) {
|
|
8271
|
+
let entries;
|
|
8272
|
+
try {
|
|
8273
|
+
if (!(await stat(outDir)).isDirectory()) throw new Error(`Output path is not a directory: ${outDir}`);
|
|
8274
|
+
entries = await readdir(outDir);
|
|
8275
|
+
} catch (error) {
|
|
8276
|
+
if (error.code !== "ENOENT") throw error;
|
|
8277
|
+
await mkdir(outDir, { recursive: true });
|
|
8278
|
+
return;
|
|
8279
|
+
}
|
|
8280
|
+
if (!entries.length) return;
|
|
8281
|
+
if (!entries.includes(MARKER)) throw new Error(`Refusing to overwrite ${outDir}: it is not empty and was not created by evalkit build`);
|
|
8282
|
+
await rm(outDir, {
|
|
8283
|
+
recursive: true,
|
|
8284
|
+
force: true
|
|
8285
|
+
});
|
|
8286
|
+
await mkdir(outDir, { recursive: true });
|
|
8287
|
+
}
|
|
8288
|
+
//#endregion
|
|
8289
|
+
//#region ../cli/src/dashboard-cells.ts
|
|
8290
|
+
const MAX_SCAN = 1e5;
|
|
8291
|
+
function display(value) {
|
|
8292
|
+
return typeof value === "string" ? value : JSON.stringify(value) ?? String(value);
|
|
8293
|
+
}
|
|
8294
|
+
/** Bounded report projection; both count and rows come from the core eligible-cell iterator. */
|
|
8295
|
+
function listMatrixCells(matrix, input) {
|
|
8296
|
+
if (!Number.isSafeInteger(input.offset) || input.offset < 0 || !Number.isSafeInteger(input.limit) || input.limit < 1 || input.limit > 100) throw new Error("Matrix page requires offset >= 0 and limit between 1 and 100");
|
|
8297
|
+
if (![
|
|
8298
|
+
"eval",
|
|
8299
|
+
"agent",
|
|
8300
|
+
"trials",
|
|
8301
|
+
...[...Object.keys(matrix.cases?.[0] ?? {}), ...Object.keys(matrix.parameters)].map((key) => `axis:${key}`)
|
|
8302
|
+
].includes(input.sort)) throw new Error(`Unknown matrix sort: ${input.sort}`);
|
|
8303
|
+
const selection = input.evalIds.length ? { evals: input.evalIds } : {};
|
|
8304
|
+
const planned = matrix.count(selection);
|
|
8305
|
+
if (planned > MAX_SCAN) throw new Error(`Matrix has ${planned} eligible cells; narrow the selection in the CLI (dashboard limit ${MAX_SCAN})`);
|
|
8306
|
+
const terms = input.query.trim().toLowerCase().split(/\s+/).filter(Boolean);
|
|
8307
|
+
const cells = [...matrix.cells(selection)].filter((cell) => {
|
|
8308
|
+
const haystack = [
|
|
8309
|
+
cell.eval.id,
|
|
8310
|
+
cell.eval.name,
|
|
8311
|
+
cell.eval.agent?.identity?.name,
|
|
8312
|
+
cell.eval.agent?.identity?.kind,
|
|
8313
|
+
...Object.entries(cell.parameters).flatMap(([key, value]) => [key, display(value)])
|
|
8314
|
+
].map(display).join(" ").toLowerCase();
|
|
8315
|
+
return terms.every((term) => haystack.includes(term));
|
|
8316
|
+
});
|
|
8317
|
+
function value(cell) {
|
|
8318
|
+
if (input.sort.startsWith("axis:")) {
|
|
8319
|
+
const parameter = cell.parameters[input.sort.slice(5)];
|
|
8320
|
+
return typeof parameter === "number" ? parameter : canonicalParameters(parameter);
|
|
8321
|
+
}
|
|
8322
|
+
if (input.sort === "eval") return cell.eval.name ?? cell.eval.id;
|
|
8323
|
+
if (input.sort === "agent") return cell.eval.agent?.identity?.name ?? cell.eval.agent?.identity?.kind ?? "";
|
|
8324
|
+
return cell.eval.policy?.trials ?? 1;
|
|
8325
|
+
}
|
|
8326
|
+
cells.sort((left, right) => {
|
|
8327
|
+
const a = value(left);
|
|
8328
|
+
const b = value(right);
|
|
8329
|
+
const comparison = typeof a === "number" && typeof b === "number" ? a - b : String(a).localeCompare(String(b), void 0, {
|
|
8330
|
+
numeric: true,
|
|
8331
|
+
sensitivity: "base"
|
|
8332
|
+
});
|
|
8333
|
+
return (input.direction === "asc" ? comparison : -comparison) || left.key.localeCompare(right.key);
|
|
8334
|
+
});
|
|
8335
|
+
return {
|
|
8336
|
+
total: cells.length,
|
|
8337
|
+
planned,
|
|
8338
|
+
cells: cells.slice(input.offset, input.offset + input.limit).map((cell) => ({
|
|
8339
|
+
evalId: cell.eval.id,
|
|
8340
|
+
key: cell.key,
|
|
8341
|
+
parameters: cell.parameters
|
|
8342
|
+
}))
|
|
8343
|
+
};
|
|
8344
|
+
}
|
|
8345
|
+
//#endregion
|
|
8262
8346
|
//#region ../cli/src/dashboard-matrix.ts
|
|
8263
8347
|
/** Dashboard submissions are intentionally one explicit, configured cell at a time. */
|
|
8264
8348
|
function selectDashboardCell(matrix, evalId, parameters) {
|
|
@@ -8388,6 +8472,8 @@ function parseRunArgs(args) {
|
|
|
8388
8472
|
type: "string",
|
|
8389
8473
|
multiple: true
|
|
8390
8474
|
},
|
|
8475
|
+
suite: { type: "string" },
|
|
8476
|
+
matrix: { type: "string" },
|
|
8391
8477
|
model: {
|
|
8392
8478
|
type: "string",
|
|
8393
8479
|
multiple: true
|
|
@@ -8454,6 +8540,8 @@ function normalizeRunOptions(values, positionals) {
|
|
|
8454
8540
|
if (index < 1) throw new Error("--param requires name=JSON");
|
|
8455
8541
|
parameters[value.slice(0, index)] = JSON.parse(value.slice(index + 1));
|
|
8456
8542
|
}
|
|
8543
|
+
if (values.suite && values.matrix) throw new Error("--suite and --matrix cannot be used together");
|
|
8544
|
+
if ((values.suite || values.matrix) && positionals.length) throw new Error("Use --eval to select evals with --suite or --matrix");
|
|
8457
8545
|
return {
|
|
8458
8546
|
values,
|
|
8459
8547
|
positionals,
|
|
@@ -8463,7 +8551,7 @@ function normalizeRunOptions(values, positionals) {
|
|
|
8463
8551
|
trials: integer("trials")
|
|
8464
8552
|
};
|
|
8465
8553
|
}
|
|
8466
|
-
async function runProjectCommand(
|
|
8554
|
+
async function runProjectCommand(args, cwd = process.cwd(), parsed = parseRunArgs(args)) {
|
|
8467
8555
|
const { values, positionals, parameters } = parsed;
|
|
8468
8556
|
const project = await loadProject(cwd, values.config);
|
|
8469
8557
|
process.chdir(project.root);
|
|
@@ -8471,14 +8559,9 @@ async function runProjectCommand(command, args, cwd = process.cwd(), parsed = pa
|
|
|
8471
8559
|
const requestedEvals = values.eval?.flatMap((value) => value.split(",")) ?? [];
|
|
8472
8560
|
let matrix;
|
|
8473
8561
|
let suiteId;
|
|
8474
|
-
if (
|
|
8475
|
-
|
|
8476
|
-
|
|
8477
|
-
if (!matrix) throw new Error(`Unknown matrix: ${positionals[0]}`);
|
|
8478
|
-
} else if (command === "run-suite") {
|
|
8479
|
-
if (positionals.length !== 1) throw new Error("run-suite requires one suite ID");
|
|
8480
|
-
const suite = registry.suites.find((s) => s.id === positionals[0]);
|
|
8481
|
-
if (!suite) throw new Error(`Unknown suite: ${positionals[0]}`);
|
|
8562
|
+
if (values.suite) {
|
|
8563
|
+
const suite = registry.suites.find((item) => item.id === values.suite);
|
|
8564
|
+
if (!suite) throw new Error(`Unknown suite: ${values.suite}`);
|
|
8482
8565
|
suiteId = authoringId(suite);
|
|
8483
8566
|
matrix = defineEvalMatrix({
|
|
8484
8567
|
id: "default",
|
|
@@ -8488,6 +8571,9 @@ async function runProjectCommand(command, args, cwd = process.cwd(), parsed = pa
|
|
|
8488
8571
|
cases: project.config.matrix?.cases,
|
|
8489
8572
|
exclude: project.config.matrix?.exclude
|
|
8490
8573
|
});
|
|
8574
|
+
} else if (values.matrix) {
|
|
8575
|
+
matrix = registry.matrices.find((item) => item.id === values.matrix);
|
|
8576
|
+
if (!matrix) throw new Error(`Unknown matrix: ${values.matrix}`);
|
|
8491
8577
|
} else {
|
|
8492
8578
|
requestedEvals.push(...positionals.flatMap((value) => value.split(",")));
|
|
8493
8579
|
matrix = project.config.matrix ? registry.matrices.at(-1) : defineEvalMatrix({
|
|
@@ -8502,7 +8588,7 @@ async function runProjectCommand(command, args, cwd = process.cwd(), parsed = pa
|
|
|
8502
8588
|
...requestedEvals.length ? { evals: requestedEvals } : {}
|
|
8503
8589
|
};
|
|
8504
8590
|
for (const [axis, choices] of Object.entries(parsed.selection)) if (axis in matrix.parameters || matrix.cases?.some((assignment) => Object.hasOwn(assignment, axis))) selection.parameters[axis] = choices;
|
|
8505
|
-
else if ((axis === "model" || axis === "mode") && choices.length === 1 && !project.config.matrix &&
|
|
8591
|
+
else if ((axis === "model" || axis === "mode") && choices.length === 1 && !project.config.matrix && !values.matrix) selection.overrides[axis] = choices[0];
|
|
8506
8592
|
else throw new Error(`Unknown matrix axis: ${axis}`);
|
|
8507
8593
|
const cells = matrix.count(selection);
|
|
8508
8594
|
if (!cells) throw new Error("Selection contains no eligible cells; check cases and exclusions");
|
|
@@ -8653,21 +8739,23 @@ bun run check # type-check your eval, agent, scorer, and config
|
|
|
8653
8739
|
bun run evals # run the example across both styles
|
|
8654
8740
|
bun run matrix:plan # inspect the matrix without running it
|
|
8655
8741
|
bun run matrix # run the configured matrix
|
|
8656
|
-
bun run dashboard # inspect local reports
|
|
8742
|
+
bun run dashboard # inspect local reports and launch runs
|
|
8743
|
+
bun run dashboard:build # write a read-only, shareable snapshot
|
|
8657
8744
|
\`\`\`
|
|
8658
8745
|
|
|
8659
|
-
EvalKit installs from
|
|
8660
|
-
|
|
8661
|
-
|
|
8746
|
+
EvalKit installs from npm. Reports, trial workspaces, and dashboard snapshots stay local
|
|
8747
|
+
and are ignored by Git. A snapshot in \`_evalkit-dashboard/\` contains report messages,
|
|
8748
|
+
scores, evidence, and candidate files (never evaluator workspaces): review it before sharing.
|
|
8662
8749
|
`;
|
|
8663
8750
|
}
|
|
8664
8751
|
//#endregion
|
|
8665
8752
|
//#region ../cli/src/new-project.ts
|
|
8666
|
-
const npmDependency = "^0.
|
|
8753
|
+
const npmDependency = "^0.2.0";
|
|
8667
8754
|
const ignored = [
|
|
8668
8755
|
"node_modules/",
|
|
8669
8756
|
"_evalkit-results/",
|
|
8670
|
-
"_evalkit-sandbox/"
|
|
8757
|
+
"_evalkit-sandbox/",
|
|
8758
|
+
"_evalkit-dashboard/"
|
|
8671
8759
|
];
|
|
8672
8760
|
const readIfExists = async (path) => {
|
|
8673
8761
|
try {
|
|
@@ -8746,10 +8834,11 @@ async function preparePackage(packagePath, name) {
|
|
|
8746
8834
|
const devDependencies = pkg.devDependencies ?? {};
|
|
8747
8835
|
if (!scripts || typeof scripts !== "object" || Array.isArray(scripts) || !dependencies || typeof dependencies !== "object" || Array.isArray(dependencies) || !devDependencies || typeof devDependencies !== "object" || Array.isArray(devDependencies)) throw new Error(`Invalid package.json scripts/dependencies: ${packagePath}`);
|
|
8748
8836
|
scripts.check ??= "tsc --noEmit";
|
|
8749
|
-
scripts.evals ??= "evalkit run
|
|
8750
|
-
scripts.matrix ??= "evalkit run
|
|
8751
|
-
scripts["matrix:plan"] ??= "evalkit run
|
|
8752
|
-
scripts.dashboard ??= "evalkit
|
|
8837
|
+
scripts.evals ??= "evalkit run greeting --local";
|
|
8838
|
+
scripts.matrix ??= "evalkit run --matrix styles --local";
|
|
8839
|
+
scripts["matrix:plan"] ??= "evalkit run --matrix styles --dry-run";
|
|
8840
|
+
scripts.dashboard ??= "evalkit dev";
|
|
8841
|
+
scripts["dashboard:build"] ??= "evalkit build";
|
|
8753
8842
|
if (!dependencies["@leostera/evalkit"] && !devDependencies["@leostera/evalkit"]) dependencies["@leostera/evalkit"] = npmDependency;
|
|
8754
8843
|
if (!dependencies.typescript && !devDependencies.typescript) devDependencies.typescript = "^6.0.0";
|
|
8755
8844
|
if (!dependencies["@types/bun"] && !devDependencies["@types/bun"]) devDependencies["@types/bun"] = "^1.4.0";
|
|
@@ -8942,10 +9031,21 @@ function registerDashboardAssets(app, dashboardRoot) {
|
|
|
8942
9031
|
return new Response(content, { headers: { "content-type": fileContentType(servedFile) } });
|
|
8943
9032
|
});
|
|
8944
9033
|
}
|
|
9034
|
+
/** The npm package bundles the UI beside cli.mjs; the monorepo uses its Vite build. */
|
|
9035
|
+
function dashboardAssets() {
|
|
9036
|
+
const bundledDashboard = new URL("./dashboard/", import.meta.url);
|
|
9037
|
+
const root = fileURLToPath(existsSync(bundledDashboard) ? bundledDashboard : new URL("../../dashboard/dist/", import.meta.url));
|
|
9038
|
+
if (!existsSync(resolve(root, "index.html"))) throw new Error(`Dashboard assets not found at ${root}; run \`bun run build\` in the EvalKit repository`);
|
|
9039
|
+
return root;
|
|
9040
|
+
}
|
|
9041
|
+
async function useProject(configPath) {
|
|
9042
|
+
project = await loadProject(projectRoot, configPath);
|
|
9043
|
+
projectRoot = project.root;
|
|
9044
|
+
reportRoot = resolve(projectRoot, project.config.reportDir ?? "_evalkit-results");
|
|
9045
|
+
}
|
|
8945
9046
|
async function serveDashboard() {
|
|
8946
9047
|
const registry = await loadRegistry();
|
|
8947
|
-
const
|
|
8948
|
-
const dashboardRoot = fileURLToPath(existsSync(bundledDashboard) ? bundledDashboard : new URL("../../dashboard/dist/", import.meta.url));
|
|
9048
|
+
const dashboardRoot = dashboardAssets();
|
|
8949
9049
|
const app = new Hono();
|
|
8950
9050
|
registerMatrixRoutes(app, registry, project?.config.matrix ? registry.matrices.at(-1) : void 0, project?.config.execution?.trials);
|
|
8951
9051
|
registerReportRoutes(app, registry, createDashboardReportReader(reportRoot, registry));
|
|
@@ -8969,6 +9069,8 @@ const config = text("config").pipe(optional);
|
|
|
8969
9069
|
const runOptions = {
|
|
8970
9070
|
config,
|
|
8971
9071
|
eval: text("eval").pipe(repeated),
|
|
9072
|
+
suite: text("suite").pipe(optional),
|
|
9073
|
+
matrix: text("matrix").pipe(optional),
|
|
8972
9074
|
model: text("model").pipe(repeated),
|
|
8973
9075
|
mode: text("mode").pipe(repeated),
|
|
8974
9076
|
select: text("select").pipe(repeated),
|
|
@@ -8989,6 +9091,8 @@ function runInput(options, positionals) {
|
|
|
8989
9091
|
return normalizeRunOptions({
|
|
8990
9092
|
config: Option.getOrUndefined(options.config),
|
|
8991
9093
|
eval: options.eval,
|
|
9094
|
+
suite: Option.getOrUndefined(options.suite),
|
|
9095
|
+
matrix: Option.getOrUndefined(options.matrix),
|
|
8992
9096
|
model: options.model,
|
|
8993
9097
|
mode: options.mode,
|
|
8994
9098
|
select: options.select,
|
|
@@ -9004,22 +9108,14 @@ function runInput(options, positionals) {
|
|
|
9004
9108
|
local: options.local
|
|
9005
9109
|
}, positionals);
|
|
9006
9110
|
}
|
|
9007
|
-
const execute = (
|
|
9008
|
-
try: () => runProjectCommand(
|
|
9111
|
+
const execute = (input) => Effect.tryPromise({
|
|
9112
|
+
try: () => runProjectCommand([], process.cwd(), input),
|
|
9009
9113
|
catch: (error) => error
|
|
9010
9114
|
});
|
|
9011
|
-
const
|
|
9115
|
+
const runCommand = make$1("run", {
|
|
9012
9116
|
...runOptions,
|
|
9013
9117
|
evalIds: text$2({ name: "eval-id" }).pipe(repeated$2)
|
|
9014
|
-
}, ({ evalIds, ...options }) => execute(
|
|
9015
|
-
const matrixCommand = make$1("run-matrix", {
|
|
9016
|
-
...runOptions,
|
|
9017
|
-
matrixId: text$2({ name: "matrix-id" })
|
|
9018
|
-
}, ({ matrixId, ...options }) => execute("run-matrix", runInput(options, [matrixId])));
|
|
9019
|
-
const suiteCommand = make$1("run-suite", {
|
|
9020
|
-
...runOptions,
|
|
9021
|
-
suiteId: text$2({ name: "suite-id" })
|
|
9022
|
-
}, ({ suiteId, ...options }) => execute("run-suite", runInput(options, [suiteId])));
|
|
9118
|
+
}, ({ evalIds, ...options }) => execute(runInput(options, evalIds)));
|
|
9023
9119
|
const newCommand = make$1("new", { directory: text$2({ name: "directory" }) }, ({ directory }) => Effect.tryPromise({
|
|
9024
9120
|
try: async () => {
|
|
9025
9121
|
const root = await newProject([directory]);
|
|
@@ -9027,24 +9123,38 @@ const newCommand = make$1("new", { directory: text$2({ name: "directory" }) }, (
|
|
|
9027
9123
|
},
|
|
9028
9124
|
catch: (error) => error
|
|
9029
9125
|
}));
|
|
9030
|
-
const
|
|
9126
|
+
const devCommand = make$1("dev", { config }, ({ config }) => Effect.tryPromise({
|
|
9031
9127
|
try: async () => {
|
|
9032
|
-
|
|
9033
|
-
projectRoot = project.root;
|
|
9034
|
-
reportRoot = resolve(projectRoot, project.config.reportDir ?? "_evalkit-results");
|
|
9128
|
+
await useProject(Option.getOrUndefined(config));
|
|
9035
9129
|
await serveDashboard();
|
|
9036
9130
|
},
|
|
9037
9131
|
catch: (error) => error
|
|
9038
9132
|
}));
|
|
9133
|
+
const buildCommand = make$1("build", {
|
|
9134
|
+
config,
|
|
9135
|
+
out: text("out").pipe(optional)
|
|
9136
|
+
}, ({ config, out }) => Effect.tryPromise({
|
|
9137
|
+
try: async () => {
|
|
9138
|
+
await useProject(Option.getOrUndefined(config));
|
|
9139
|
+
const result = await buildDashboard({
|
|
9140
|
+
registry: project.registry,
|
|
9141
|
+
reportRoot,
|
|
9142
|
+
dashboardRoot: dashboardAssets(),
|
|
9143
|
+
outDir: resolve(projectRoot, Option.getOrElse(out, () => "_evalkit-dashboard"))
|
|
9144
|
+
});
|
|
9145
|
+
console.log(`Built a read-only dashboard snapshot at ${result.outDir}\n ${result.runs} run(s), ${result.trials} trial(s), ${result.files} candidate file(s)\nServe it with any static file server. It contains report messages, scores, evidence,
|
|
9146
|
+
and candidate files (never evaluator workspaces): review it before sharing.`);
|
|
9147
|
+
},
|
|
9148
|
+
catch: (error) => error
|
|
9149
|
+
}));
|
|
9039
9150
|
const cli = run(make$1("evalkit").pipe(withSubcommands([
|
|
9040
9151
|
newCommand,
|
|
9041
|
-
|
|
9042
|
-
|
|
9043
|
-
|
|
9044
|
-
dashboardCommand
|
|
9152
|
+
runCommand,
|
|
9153
|
+
devCommand,
|
|
9154
|
+
buildCommand
|
|
9045
9155
|
])), {
|
|
9046
9156
|
name: "EvalKit",
|
|
9047
|
-
version: "0.
|
|
9157
|
+
version: "0.2.0"
|
|
9048
9158
|
});
|
|
9049
9159
|
const argv = process.argv.slice();
|
|
9050
9160
|
if (argv[2] === "help") argv.splice(2, 1, "--help");
|