@leostera/evalkit 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -21,6 +21,7 @@ bun run check # type-check the project
21
21
  bun run evals # both configured styles
22
22
  bun run matrix:plan # inspect the matrix without executing it
23
23
  bun run dashboard # in a second terminal
24
+ bun run dashboard:build # optional read-only snapshot to share
24
25
  ```
25
26
 
26
27
  Already have a Bun project? From its root you can instead run:
package/dist/cli.mjs CHANGED
@@ -3,7 +3,7 @@ import { F as defineEvalMatrix, I as canonicalParameters, m as registerEvals, n
3
3
  import { a as readTrialEvents, c as readTrialSummary, i as readRunSummary, l as localReportStore, n as runMatrix, o as readTrialManifest, r as readRunManifest, t as runEval } from "./src-DluldGKn.mjs";
4
4
  import * as Schema from "effect/Schema";
5
5
  import { Effect, Option } from "effect";
6
- import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises";
6
+ import { cp, mkdir, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
7
7
  import * as Path from "node:path";
8
8
  import { basename, dirname, extname, relative, resolve, sep } from "node:path";
9
9
  import * as OS from "node:os";
@@ -7775,7 +7775,7 @@ const ProcessProto = {
7775
7775
  };
7776
7776
  }
7777
7777
  };
7778
- const runCommand = (fileSystem) => (command) => {
7778
+ const runCommand$1 = (fileSystem) => (command) => {
7779
7779
  switch (command._tag) {
7780
7780
  case "StandardCommand": {
7781
7781
  const spawn = Effect$1.flatMap(Deferred.make(), (exitCode) => Effect$1.async((resume) => {
@@ -7851,13 +7851,13 @@ const runCommand = (fileSystem) => (command) => {
7851
7851
  }
7852
7852
  case "PipedCommand": {
7853
7853
  const flattened = flatten(command);
7854
- if (flattened.length === 1) return pipe(flattened[0], runCommand(fileSystem));
7854
+ if (flattened.length === 1) return pipe(flattened[0], runCommand$1(fileSystem));
7855
7855
  const head = flattened[0];
7856
7856
  const tail = flattened.slice(1);
7857
7857
  const initial = tail.slice(0, tail.length - 1);
7858
7858
  const last = tail[tail.length - 1];
7859
- const stream = initial.reduce((stdin$2, command) => pipe(stdin(command, stdin$2), runCommand(fileSystem), Effect$1.map((process) => process.stdout), Stream.unwrapScoped), pipe(runCommand(fileSystem)(head), Effect$1.map((process) => process.stdout), Stream.unwrapScoped));
7860
- return pipe(stdin(last, stream), runCommand(fileSystem));
7859
+ const stream = initial.reduce((stdin$2, command) => pipe(stdin(command, stdin$2), runCommand$1(fileSystem), Effect$1.map((process) => process.stdout), Stream.unwrapScoped), pipe(runCommand$1(fileSystem)(head), Effect$1.map((process) => process.stdout), Stream.unwrapScoped));
7860
+ return pipe(stdin(last, stream), runCommand$1(fileSystem));
7861
7861
  }
7862
7862
  }
7863
7863
  };
@@ -7867,7 +7867,7 @@ const runCommand = (fileSystem) => (command) => {
7867
7867
  * @since 1.0.0
7868
7868
  * @category layer
7869
7869
  */
7870
- const layer$5 = /* @__PURE__ */ Layer.effect(CommandExecutor, /*#__PURE__*/ pipe(FileSystem, /*#__PURE__*/ Effect$1.map((fileSystem) => makeExecutor(runCommand(fileSystem)))));
7870
+ const layer$5 = /* @__PURE__ */ Layer.effect(CommandExecutor, /*#__PURE__*/ pipe(FileSystem, /*#__PURE__*/ Effect$1.map((fileSystem) => makeExecutor(runCommand$1(fileSystem)))));
7871
7871
  //#endregion
7872
7872
  //#region ../../node_modules/.bun/@effect+platform-node-shared@0.61.1+1edbab697a2570fc/node_modules/@effect/platform-node-shared/dist/esm/internal/path.js
7873
7873
  const fromFileUrl = (url) => Effect$1.try({
@@ -8048,63 +8048,6 @@ const layerManager = /* @__PURE__ */ Layer.provide(layerManager$2, layerWorker);
8048
8048
  */
8049
8049
  const layer = /*#__PURE__*/ pipe(/*#__PURE__*/ Layer.mergeAll(layer$3, layer$5, layer$1, layerManager), /*#__PURE__*/ Layer.provideMerge(layer$7));
8050
8050
  //#endregion
8051
- //#region ../cli/src/dashboard-cells.ts
8052
- const MAX_SCAN = 1e5;
8053
- function display(value) {
8054
- return typeof value === "string" ? value : JSON.stringify(value) ?? String(value);
8055
- }
8056
- /** Bounded report projection; both count and rows come from the core eligible-cell iterator. */
8057
- function listMatrixCells(matrix, input) {
8058
- if (!Number.isSafeInteger(input.offset) || input.offset < 0 || !Number.isSafeInteger(input.limit) || input.limit < 1 || input.limit > 100) throw new Error("Matrix page requires offset >= 0 and limit between 1 and 100");
8059
- if (![
8060
- "eval",
8061
- "agent",
8062
- "trials",
8063
- ...[...Object.keys(matrix.cases?.[0] ?? {}), ...Object.keys(matrix.parameters)].map((key) => `axis:${key}`)
8064
- ].includes(input.sort)) throw new Error(`Unknown matrix sort: ${input.sort}`);
8065
- const selection = input.evalIds.length ? { evals: input.evalIds } : {};
8066
- const planned = matrix.count(selection);
8067
- if (planned > MAX_SCAN) throw new Error(`Matrix has ${planned} eligible cells; narrow the selection in the CLI (dashboard limit ${MAX_SCAN})`);
8068
- const terms = input.query.trim().toLowerCase().split(/\s+/).filter(Boolean);
8069
- const cells = [...matrix.cells(selection)].filter((cell) => {
8070
- const haystack = [
8071
- cell.eval.id,
8072
- cell.eval.name,
8073
- cell.eval.agent?.identity?.name,
8074
- cell.eval.agent?.identity?.kind,
8075
- ...Object.entries(cell.parameters).flatMap(([key, value]) => [key, display(value)])
8076
- ].map(display).join(" ").toLowerCase();
8077
- return terms.every((term) => haystack.includes(term));
8078
- });
8079
- function value(cell) {
8080
- if (input.sort.startsWith("axis:")) {
8081
- const parameter = cell.parameters[input.sort.slice(5)];
8082
- return typeof parameter === "number" ? parameter : canonicalParameters(parameter);
8083
- }
8084
- if (input.sort === "eval") return cell.eval.name ?? cell.eval.id;
8085
- if (input.sort === "agent") return cell.eval.agent?.identity?.name ?? cell.eval.agent?.identity?.kind ?? "";
8086
- return cell.eval.policy?.trials ?? 1;
8087
- }
8088
- cells.sort((left, right) => {
8089
- const a = value(left);
8090
- const b = value(right);
8091
- const comparison = typeof a === "number" && typeof b === "number" ? a - b : String(a).localeCompare(String(b), void 0, {
8092
- numeric: true,
8093
- sensitivity: "base"
8094
- });
8095
- return (input.direction === "asc" ? comparison : -comparison) || left.key.localeCompare(right.key);
8096
- });
8097
- return {
8098
- total: cells.length,
8099
- planned,
8100
- cells: cells.slice(input.offset, input.offset + input.limit).map((cell) => ({
8101
- evalId: cell.eval.id,
8102
- key: cell.key,
8103
- parameters: cell.parameters
8104
- }))
8105
- };
8106
- }
8107
- //#endregion
8108
8051
  //#region ../cli/src/dashboard-reports.ts
8109
8052
  const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
8110
8053
  function isMissing(error) {
@@ -8259,6 +8202,147 @@ async function walkArtifacts(root) {
8259
8202
  return result;
8260
8203
  }
8261
8204
  //#endregion
8205
+ //#region ../cli/src/dashboard-build.ts
8206
+ /** Marks a directory as EvalKit-owned so a rebuild may safely replace it. */
8207
+ const MARKER = ".evalkit-dashboard";
8208
+ /**
8209
+ * Write a read-only static dashboard: the bundled UI plus JSON snapshots of
8210
+ * the catalog and local reports, and each trial's candidate artifact files.
8211
+ * Evaluator workspaces are never copied; reports may still contain sensitive
8212
+ * messages, scores, or evidence, so review the output before sharing it.
8213
+ */
8214
+ async function buildDashboard(options) {
8215
+ const { registry, reportRoot, dashboardRoot, outDir } = options;
8216
+ await prepareOutDir(outDir);
8217
+ await cp(dashboardRoot, outDir, { recursive: true });
8218
+ await writeFile(resolve(outDir, MARKER), "Generated by `evalkit build`.\n");
8219
+ await writeFile(resolve(outDir, "index.html"), staticIndex(await readFile(resolve(dashboardRoot, "index.html"), "utf8")));
8220
+ const data = resolve(outDir, "data");
8221
+ const write = async (path, value) => {
8222
+ const file = resolve(data, path);
8223
+ await mkdir(dirname(file), { recursive: true });
8224
+ await writeFile(file, `${JSON.stringify(value)}\n`);
8225
+ };
8226
+ const reports = createDashboardReportReader(reportRoot, registry);
8227
+ const runs = await reports.listRuns();
8228
+ await write("catalog.json", { evals: registry.catalog() });
8229
+ await write("evals.json", { evals: registry.metadata() });
8230
+ await write("suites.json", { suites: registry.suiteMetadata() });
8231
+ await write("runs.json", { runs });
8232
+ let trials = 0;
8233
+ let files = 0;
8234
+ for (const run of runs) {
8235
+ const runTrials = await reports.listTrials(run.id);
8236
+ await write(`runs/${run.id}/trials.json`, { trials: runTrials });
8237
+ for (const trial of runTrials) {
8238
+ trials += 1;
8239
+ const base = `runs/${run.id}/trials/${trial.id}`;
8240
+ await write(`${base}/detail.json`, await reports.trialDetail(run.id, trial.id));
8241
+ await write(`${base}/events.json`, { events: await reports.trajectory(run.id, trial.id) });
8242
+ await write(`${base}/artifacts.json`, { artifacts: await reports.listArtifacts(run.id, trial.id) });
8243
+ const workspace = await reports.listCandidateWorkspace(run.id, trial.id);
8244
+ await write(`${base}/workspace.json`, workspace);
8245
+ for (const entry of workspace) {
8246
+ if (entry.kind !== "file") continue;
8247
+ const response = await reports.candidateFile(run.id, trial.id, entry.path);
8248
+ const target = resolve(data, base, "workspace", entry.path);
8249
+ const root = resolve(data, base, "workspace");
8250
+ if (!target.startsWith(`${root}${sep}`)) throw new Error(`Invalid workspace path: ${entry.path}`);
8251
+ await mkdir(dirname(target), { recursive: true });
8252
+ await writeFile(target, new Uint8Array(await response.arrayBuffer()));
8253
+ files += 1;
8254
+ }
8255
+ }
8256
+ }
8257
+ return {
8258
+ outDir,
8259
+ runs: runs.length,
8260
+ trials,
8261
+ files
8262
+ };
8263
+ }
8264
+ /** Use relative asset URLs and select the static API in the browser bundle. */
8265
+ function staticIndex(html) {
8266
+ const result = html.replace("<html lang=\"en\">", "<html lang=\"en\" data-evalkit-snapshot=\"static\">").replaceAll("\"/assets/", "\"./assets/");
8267
+ if (!result.includes("data-evalkit-snapshot=\"static\"")) throw new Error("Bundled dashboard index.html has an unexpected shape");
8268
+ return result;
8269
+ }
8270
+ async function prepareOutDir(outDir) {
8271
+ let entries;
8272
+ try {
8273
+ if (!(await stat(outDir)).isDirectory()) throw new Error(`Output path is not a directory: ${outDir}`);
8274
+ entries = await readdir(outDir);
8275
+ } catch (error) {
8276
+ if (error.code !== "ENOENT") throw error;
8277
+ await mkdir(outDir, { recursive: true });
8278
+ return;
8279
+ }
8280
+ if (!entries.length) return;
8281
+ if (!entries.includes(MARKER)) throw new Error(`Refusing to overwrite ${outDir}: it is not empty and was not created by evalkit build`);
8282
+ await rm(outDir, {
8283
+ recursive: true,
8284
+ force: true
8285
+ });
8286
+ await mkdir(outDir, { recursive: true });
8287
+ }
8288
+ //#endregion
8289
+ //#region ../cli/src/dashboard-cells.ts
8290
+ const MAX_SCAN = 1e5;
8291
+ function display(value) {
8292
+ return typeof value === "string" ? value : JSON.stringify(value) ?? String(value);
8293
+ }
8294
+ /** Bounded report projection; both count and rows come from the core eligible-cell iterator. */
8295
+ function listMatrixCells(matrix, input) {
8296
+ if (!Number.isSafeInteger(input.offset) || input.offset < 0 || !Number.isSafeInteger(input.limit) || input.limit < 1 || input.limit > 100) throw new Error("Matrix page requires offset >= 0 and limit between 1 and 100");
8297
+ if (![
8298
+ "eval",
8299
+ "agent",
8300
+ "trials",
8301
+ ...[...Object.keys(matrix.cases?.[0] ?? {}), ...Object.keys(matrix.parameters)].map((key) => `axis:${key}`)
8302
+ ].includes(input.sort)) throw new Error(`Unknown matrix sort: ${input.sort}`);
8303
+ const selection = input.evalIds.length ? { evals: input.evalIds } : {};
8304
+ const planned = matrix.count(selection);
8305
+ if (planned > MAX_SCAN) throw new Error(`Matrix has ${planned} eligible cells; narrow the selection in the CLI (dashboard limit ${MAX_SCAN})`);
8306
+ const terms = input.query.trim().toLowerCase().split(/\s+/).filter(Boolean);
8307
+ const cells = [...matrix.cells(selection)].filter((cell) => {
8308
+ const haystack = [
8309
+ cell.eval.id,
8310
+ cell.eval.name,
8311
+ cell.eval.agent?.identity?.name,
8312
+ cell.eval.agent?.identity?.kind,
8313
+ ...Object.entries(cell.parameters).flatMap(([key, value]) => [key, display(value)])
8314
+ ].map(display).join(" ").toLowerCase();
8315
+ return terms.every((term) => haystack.includes(term));
8316
+ });
8317
+ function value(cell) {
8318
+ if (input.sort.startsWith("axis:")) {
8319
+ const parameter = cell.parameters[input.sort.slice(5)];
8320
+ return typeof parameter === "number" ? parameter : canonicalParameters(parameter);
8321
+ }
8322
+ if (input.sort === "eval") return cell.eval.name ?? cell.eval.id;
8323
+ if (input.sort === "agent") return cell.eval.agent?.identity?.name ?? cell.eval.agent?.identity?.kind ?? "";
8324
+ return cell.eval.policy?.trials ?? 1;
8325
+ }
8326
+ cells.sort((left, right) => {
8327
+ const a = value(left);
8328
+ const b = value(right);
8329
+ const comparison = typeof a === "number" && typeof b === "number" ? a - b : String(a).localeCompare(String(b), void 0, {
8330
+ numeric: true,
8331
+ sensitivity: "base"
8332
+ });
8333
+ return (input.direction === "asc" ? comparison : -comparison) || left.key.localeCompare(right.key);
8334
+ });
8335
+ return {
8336
+ total: cells.length,
8337
+ planned,
8338
+ cells: cells.slice(input.offset, input.offset + input.limit).map((cell) => ({
8339
+ evalId: cell.eval.id,
8340
+ key: cell.key,
8341
+ parameters: cell.parameters
8342
+ }))
8343
+ };
8344
+ }
8345
+ //#endregion
8262
8346
  //#region ../cli/src/dashboard-matrix.ts
8263
8347
  /** Dashboard submissions are intentionally one explicit, configured cell at a time. */
8264
8348
  function selectDashboardCell(matrix, evalId, parameters) {
@@ -8388,6 +8472,8 @@ function parseRunArgs(args) {
8388
8472
  type: "string",
8389
8473
  multiple: true
8390
8474
  },
8475
+ suite: { type: "string" },
8476
+ matrix: { type: "string" },
8391
8477
  model: {
8392
8478
  type: "string",
8393
8479
  multiple: true
@@ -8454,6 +8540,8 @@ function normalizeRunOptions(values, positionals) {
8454
8540
  if (index < 1) throw new Error("--param requires name=JSON");
8455
8541
  parameters[value.slice(0, index)] = JSON.parse(value.slice(index + 1));
8456
8542
  }
8543
+ if (values.suite && values.matrix) throw new Error("--suite and --matrix cannot be used together");
8544
+ if ((values.suite || values.matrix) && positionals.length) throw new Error("Use --eval to select evals with --suite or --matrix");
8457
8545
  return {
8458
8546
  values,
8459
8547
  positionals,
@@ -8463,7 +8551,7 @@ function normalizeRunOptions(values, positionals) {
8463
8551
  trials: integer("trials")
8464
8552
  };
8465
8553
  }
8466
- async function runProjectCommand(command, args, cwd = process.cwd(), parsed = parseRunArgs(args)) {
8554
+ async function runProjectCommand(args, cwd = process.cwd(), parsed = parseRunArgs(args)) {
8467
8555
  const { values, positionals, parameters } = parsed;
8468
8556
  const project = await loadProject(cwd, values.config);
8469
8557
  process.chdir(project.root);
@@ -8471,14 +8559,9 @@ async function runProjectCommand(command, args, cwd = process.cwd(), parsed = pa
8471
8559
  const requestedEvals = values.eval?.flatMap((value) => value.split(",")) ?? [];
8472
8560
  let matrix;
8473
8561
  let suiteId;
8474
- if (command === "run-matrix") {
8475
- if (positionals.length !== 1) throw new Error("run-matrix requires one matrix ID");
8476
- matrix = registry.matrices.find((m) => m.id === positionals[0]);
8477
- if (!matrix) throw new Error(`Unknown matrix: ${positionals[0]}`);
8478
- } else if (command === "run-suite") {
8479
- if (positionals.length !== 1) throw new Error("run-suite requires one suite ID");
8480
- const suite = registry.suites.find((s) => s.id === positionals[0]);
8481
- if (!suite) throw new Error(`Unknown suite: ${positionals[0]}`);
8562
+ if (values.suite) {
8563
+ const suite = registry.suites.find((item) => item.id === values.suite);
8564
+ if (!suite) throw new Error(`Unknown suite: ${values.suite}`);
8482
8565
  suiteId = authoringId(suite);
8483
8566
  matrix = defineEvalMatrix({
8484
8567
  id: "default",
@@ -8488,6 +8571,9 @@ async function runProjectCommand(command, args, cwd = process.cwd(), parsed = pa
8488
8571
  cases: project.config.matrix?.cases,
8489
8572
  exclude: project.config.matrix?.exclude
8490
8573
  });
8574
+ } else if (values.matrix) {
8575
+ matrix = registry.matrices.find((item) => item.id === values.matrix);
8576
+ if (!matrix) throw new Error(`Unknown matrix: ${values.matrix}`);
8491
8577
  } else {
8492
8578
  requestedEvals.push(...positionals.flatMap((value) => value.split(",")));
8493
8579
  matrix = project.config.matrix ? registry.matrices.at(-1) : defineEvalMatrix({
@@ -8502,7 +8588,7 @@ async function runProjectCommand(command, args, cwd = process.cwd(), parsed = pa
8502
8588
  ...requestedEvals.length ? { evals: requestedEvals } : {}
8503
8589
  };
8504
8590
  for (const [axis, choices] of Object.entries(parsed.selection)) if (axis in matrix.parameters || matrix.cases?.some((assignment) => Object.hasOwn(assignment, axis))) selection.parameters[axis] = choices;
8505
- else if ((axis === "model" || axis === "mode") && choices.length === 1 && !project.config.matrix && command !== "run-matrix") selection.overrides[axis] = choices[0];
8591
+ else if ((axis === "model" || axis === "mode") && choices.length === 1 && !project.config.matrix && !values.matrix) selection.overrides[axis] = choices[0];
8506
8592
  else throw new Error(`Unknown matrix axis: ${axis}`);
8507
8593
  const cells = matrix.count(selection);
8508
8594
  if (!cells) throw new Error("Selection contains no eligible cells; check cases and exclusions");
@@ -8653,21 +8739,23 @@ bun run check # type-check your eval, agent, scorer, and config
8653
8739
  bun run evals # run the example across both styles
8654
8740
  bun run matrix:plan # inspect the matrix without running it
8655
8741
  bun run matrix # run the configured matrix
8656
- bun run dashboard # inspect local reports
8742
+ bun run dashboard # inspect local reports and launch runs
8743
+ bun run dashboard:build # write a read-only, shareable snapshot
8657
8744
  \`\`\`
8658
8745
 
8659
- EvalKit installs from the public GitHub repository; no registry token is needed.
8660
- Reports and trial workspaces stay local and are ignored by Git.
8661
- Commit \`bun.lock\` to pin the resolved EvalKit Git revision.
8746
+ EvalKit installs from npm. Reports, trial workspaces, and dashboard snapshots stay local
8747
+ and are ignored by Git. A snapshot in \`_evalkit-dashboard/\` contains report messages,
8748
+ scores, evidence, and candidate files (never evaluator workspaces): review it before sharing.
8662
8749
  `;
8663
8750
  }
8664
8751
  //#endregion
8665
8752
  //#region ../cli/src/new-project.ts
8666
- const npmDependency = "^0.1.0";
8753
+ const npmDependency = "^0.2.0";
8667
8754
  const ignored = [
8668
8755
  "node_modules/",
8669
8756
  "_evalkit-results/",
8670
- "_evalkit-sandbox/"
8757
+ "_evalkit-sandbox/",
8758
+ "_evalkit-dashboard/"
8671
8759
  ];
8672
8760
  const readIfExists = async (path) => {
8673
8761
  try {
@@ -8746,10 +8834,11 @@ async function preparePackage(packagePath, name) {
8746
8834
  const devDependencies = pkg.devDependencies ?? {};
8747
8835
  if (!scripts || typeof scripts !== "object" || Array.isArray(scripts) || !dependencies || typeof dependencies !== "object" || Array.isArray(dependencies) || !devDependencies || typeof devDependencies !== "object" || Array.isArray(devDependencies)) throw new Error(`Invalid package.json scripts/dependencies: ${packagePath}`);
8748
8836
  scripts.check ??= "tsc --noEmit";
8749
- scripts.evals ??= "evalkit run-evals greeting --local";
8750
- scripts.matrix ??= "evalkit run-matrix styles --local";
8751
- scripts["matrix:plan"] ??= "evalkit run-matrix styles --dry-run";
8752
- scripts.dashboard ??= "evalkit serve-dashboard";
8837
+ scripts.evals ??= "evalkit run greeting --local";
8838
+ scripts.matrix ??= "evalkit run --matrix styles --local";
8839
+ scripts["matrix:plan"] ??= "evalkit run --matrix styles --dry-run";
8840
+ scripts.dashboard ??= "evalkit dev";
8841
+ scripts["dashboard:build"] ??= "evalkit build";
8753
8842
  if (!dependencies["@leostera/evalkit"] && !devDependencies["@leostera/evalkit"]) dependencies["@leostera/evalkit"] = npmDependency;
8754
8843
  if (!dependencies.typescript && !devDependencies.typescript) devDependencies.typescript = "^6.0.0";
8755
8844
  if (!dependencies["@types/bun"] && !devDependencies["@types/bun"]) devDependencies["@types/bun"] = "^1.4.0";
@@ -8942,10 +9031,21 @@ function registerDashboardAssets(app, dashboardRoot) {
8942
9031
  return new Response(content, { headers: { "content-type": fileContentType(servedFile) } });
8943
9032
  });
8944
9033
  }
9034
+ /** The npm package bundles the UI beside cli.mjs; the monorepo uses its Vite build. */
9035
+ function dashboardAssets() {
9036
+ const bundledDashboard = new URL("./dashboard/", import.meta.url);
9037
+ const root = fileURLToPath(existsSync(bundledDashboard) ? bundledDashboard : new URL("../../dashboard/dist/", import.meta.url));
9038
+ if (!existsSync(resolve(root, "index.html"))) throw new Error(`Dashboard assets not found at ${root}; run \`bun run build\` in the EvalKit repository`);
9039
+ return root;
9040
+ }
9041
+ async function useProject(configPath) {
9042
+ project = await loadProject(projectRoot, configPath);
9043
+ projectRoot = project.root;
9044
+ reportRoot = resolve(projectRoot, project.config.reportDir ?? "_evalkit-results");
9045
+ }
8945
9046
  async function serveDashboard() {
8946
9047
  const registry = await loadRegistry();
8947
- const bundledDashboard = new URL("./dashboard/", import.meta.url);
8948
- const dashboardRoot = fileURLToPath(existsSync(bundledDashboard) ? bundledDashboard : new URL("../../dashboard/dist/", import.meta.url));
9048
+ const dashboardRoot = dashboardAssets();
8949
9049
  const app = new Hono();
8950
9050
  registerMatrixRoutes(app, registry, project?.config.matrix ? registry.matrices.at(-1) : void 0, project?.config.execution?.trials);
8951
9051
  registerReportRoutes(app, registry, createDashboardReportReader(reportRoot, registry));
@@ -8969,6 +9069,8 @@ const config = text("config").pipe(optional);
8969
9069
  const runOptions = {
8970
9070
  config,
8971
9071
  eval: text("eval").pipe(repeated),
9072
+ suite: text("suite").pipe(optional),
9073
+ matrix: text("matrix").pipe(optional),
8972
9074
  model: text("model").pipe(repeated),
8973
9075
  mode: text("mode").pipe(repeated),
8974
9076
  select: text("select").pipe(repeated),
@@ -8989,6 +9091,8 @@ function runInput(options, positionals) {
8989
9091
  return normalizeRunOptions({
8990
9092
  config: Option.getOrUndefined(options.config),
8991
9093
  eval: options.eval,
9094
+ suite: Option.getOrUndefined(options.suite),
9095
+ matrix: Option.getOrUndefined(options.matrix),
8992
9096
  model: options.model,
8993
9097
  mode: options.mode,
8994
9098
  select: options.select,
@@ -9004,22 +9108,14 @@ function runInput(options, positionals) {
9004
9108
  local: options.local
9005
9109
  }, positionals);
9006
9110
  }
9007
- const execute = (command, input) => Effect.tryPromise({
9008
- try: () => runProjectCommand(command, [], process.cwd(), input),
9111
+ const execute = (input) => Effect.tryPromise({
9112
+ try: () => runProjectCommand([], process.cwd(), input),
9009
9113
  catch: (error) => error
9010
9114
  });
9011
- const evalsCommand = make$1("run-evals", {
9115
+ const runCommand = make$1("run", {
9012
9116
  ...runOptions,
9013
9117
  evalIds: text$2({ name: "eval-id" }).pipe(repeated$2)
9014
- }, ({ evalIds, ...options }) => execute("run-evals", runInput(options, evalIds)));
9015
- const matrixCommand = make$1("run-matrix", {
9016
- ...runOptions,
9017
- matrixId: text$2({ name: "matrix-id" })
9018
- }, ({ matrixId, ...options }) => execute("run-matrix", runInput(options, [matrixId])));
9019
- const suiteCommand = make$1("run-suite", {
9020
- ...runOptions,
9021
- suiteId: text$2({ name: "suite-id" })
9022
- }, ({ suiteId, ...options }) => execute("run-suite", runInput(options, [suiteId])));
9118
+ }, ({ evalIds, ...options }) => execute(runInput(options, evalIds)));
9023
9119
  const newCommand = make$1("new", { directory: text$2({ name: "directory" }) }, ({ directory }) => Effect.tryPromise({
9024
9120
  try: async () => {
9025
9121
  const root = await newProject([directory]);
@@ -9027,24 +9123,38 @@ const newCommand = make$1("new", { directory: text$2({ name: "directory" }) }, (
9027
9123
  },
9028
9124
  catch: (error) => error
9029
9125
  }));
9030
- const dashboardCommand = make$1("serve-dashboard", { config }, ({ config }) => Effect.tryPromise({
9126
+ const devCommand = make$1("dev", { config }, ({ config }) => Effect.tryPromise({
9031
9127
  try: async () => {
9032
- project = await loadProject(projectRoot, Option.getOrUndefined(config));
9033
- projectRoot = project.root;
9034
- reportRoot = resolve(projectRoot, project.config.reportDir ?? "_evalkit-results");
9128
+ await useProject(Option.getOrUndefined(config));
9035
9129
  await serveDashboard();
9036
9130
  },
9037
9131
  catch: (error) => error
9038
9132
  }));
9133
+ const buildCommand = make$1("build", {
9134
+ config,
9135
+ out: text("out").pipe(optional)
9136
+ }, ({ config, out }) => Effect.tryPromise({
9137
+ try: async () => {
9138
+ await useProject(Option.getOrUndefined(config));
9139
+ const result = await buildDashboard({
9140
+ registry: project.registry,
9141
+ reportRoot,
9142
+ dashboardRoot: dashboardAssets(),
9143
+ outDir: resolve(projectRoot, Option.getOrElse(out, () => "_evalkit-dashboard"))
9144
+ });
9145
+ console.log(`Built a read-only dashboard snapshot at ${result.outDir}\n ${result.runs} run(s), ${result.trials} trial(s), ${result.files} candidate file(s)\nServe it with any static file server. It contains report messages, scores, evidence,
9146
+ and candidate files (never evaluator workspaces): review it before sharing.`);
9147
+ },
9148
+ catch: (error) => error
9149
+ }));
9039
9150
  const cli = run(make$1("evalkit").pipe(withSubcommands([
9040
9151
  newCommand,
9041
- evalsCommand,
9042
- matrixCommand,
9043
- suiteCommand,
9044
- dashboardCommand
9152
+ runCommand,
9153
+ devCommand,
9154
+ buildCommand
9045
9155
  ])), {
9046
9156
  name: "EvalKit",
9047
- version: "0.1.0"
9157
+ version: "0.2.0"
9048
9158
  });
9049
9159
  const argv = process.argv.slice();
9050
9160
  if (argv[2] === "help") argv.splice(2, 1, "--help");