@leostera/evalkit 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli.mjs CHANGED
@@ -2,8 +2,8 @@
2
2
  import { F as defineEvalMatrix, I as canonicalParameters, m as registerEvals, n as authoringId } from "./src-DrwwgaPn.mjs";
3
3
  import { a as readTrialEvents, c as readTrialSummary, i as readRunSummary, l as localReportStore, n as runMatrix, o as readTrialManifest, r as readRunManifest, t as runEval } from "./src-DluldGKn.mjs";
4
4
  import * as Schema from "effect/Schema";
5
- import { Effect, Option } from "effect";
6
- import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises";
5
+ import { Effect, LogLevel, Logger, Option } from "effect";
6
+ import { cp, mkdir, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
7
7
  import * as Path from "node:path";
8
8
  import { basename, dirname, extname, relative, resolve, sep } from "node:path";
9
9
  import * as OS from "node:os";
@@ -43,11 +43,11 @@ import * as EffectSecret from "effect/Secret";
43
43
  import * as Effectable from "effect/Effectable";
44
44
  import * as Match from "effect/Match";
45
45
  import * as EffectNumber from "effect/Number";
46
- import * as LogLevel from "effect/LogLevel";
46
+ import * as LogLevel$1 from "effect/LogLevel";
47
47
  import * as Config from "effect/Config";
48
48
  import * as HashMap from "effect/HashMap";
49
49
  import * as Order from "effect/Order";
50
- import * as Logger from "effect/Logger";
50
+ import * as Logger$1 from "effect/Logger";
51
51
  import * as Unify from "effect/Unify";
52
52
  import * as HashSet from "effect/HashSet";
53
53
  import * as SynchronizedRef from "effect/SynchronizedRef";
@@ -392,13 +392,13 @@ const struct = (fields) => {
392
392
  /** @internal */
393
393
  const black = { _tag: "Black" };
394
394
  /** @internal */
395
- const red$3 = { _tag: "Red" };
395
+ const red$4 = { _tag: "Red" };
396
396
  /** @internal */
397
- const green$2 = { _tag: "Green" };
397
+ const green$3 = { _tag: "Green" };
398
398
  /** @internal */
399
399
  const magenta$1 = { _tag: "Magenta" };
400
400
  /** @internal */
401
- const cyan$1 = { _tag: "Cyan" };
401
+ const cyan$2 = { _tag: "Cyan" };
402
402
  /** @internal */
403
403
  const white$3 = { _tag: "White" };
404
404
  /** @internal */
@@ -500,7 +500,7 @@ const ESC = "\x1B[";
500
500
  const BEL = "\x07";
501
501
  const SEP = ";";
502
502
  /** @internal */
503
- const bold$1 = /*#__PURE__*/ make$5({ bold: /*#__PURE__*/ Option$1.some(/*#__PURE__*/ setBold(true)) });
503
+ const bold$2 = /*#__PURE__*/ make$5({ bold: /*#__PURE__*/ Option$1.some(/*#__PURE__*/ setBold(true)) });
504
504
  /** @internal */
505
505
  const italicized$1 = /*#__PURE__*/ make$5({ italicized: /*#__PURE__*/ Option$1.some(/*#__PURE__*/ setItalicized(true)) });
506
506
  /** @internal */
@@ -512,15 +512,15 @@ const brightColor = (color) => make$5({ foreground: Option$1.some(setColor(color
512
512
  /** @internal */
513
513
  const color$1 = (color) => make$5({ foreground: Option$1.some(setColor(color, false, "foreground")) });
514
514
  /** @internal */
515
- const red$2 = /*#__PURE__*/ color$1(red$3);
515
+ const red$3 = /*#__PURE__*/ color$1(red$4);
516
516
  /** @internal */
517
- const green$1 = /*#__PURE__*/ color$1(green$2);
517
+ const green$2 = /*#__PURE__*/ color$1(green$3);
518
518
  /** @internal */
519
519
  const white$2 = /*#__PURE__*/ color$1(white$3);
520
520
  /** @internal */
521
521
  const blackBright$1 = /*#__PURE__*/ brightColor(black);
522
522
  /** @internal */
523
- const cyanBright$1 = /*#__PURE__*/ brightColor(cyan$1);
523
+ const cyanBright$1 = /*#__PURE__*/ brightColor(cyan$2);
524
524
  /** @internal */
525
525
  const beep$2 = /*#__PURE__*/ make$5({ commands: /*#__PURE__*/ Arr.of(BEL) });
526
526
  /** @internal */
@@ -587,7 +587,7 @@ const stringifyInternal = (self) => {
587
587
  * @since 1.0.0
588
588
  * @category constructors
589
589
  */
590
- const bold = bold$1;
590
+ const bold$1 = bold$2;
591
591
  /**
592
592
  * @since 1.0.0
593
593
  * @category constructors
@@ -612,12 +612,12 @@ const color = color$1;
612
612
  * @since 1.0.0
613
613
  * @category colors
614
614
  */
615
- const red$1 = red$2;
615
+ const red$2 = red$3;
616
616
  /**
617
617
  * @since 1.0.0
618
618
  * @category colors
619
619
  */
620
- const green = green$1;
620
+ const green$1 = green$2;
621
621
  /**
622
622
  * @since 1.0.0
623
623
  * @category colors
@@ -2218,7 +2218,7 @@ const optimize = optimize$1;
2218
2218
  * @since 1.0.0
2219
2219
  * @category constructors
2220
2220
  */
2221
- const red = red$3;
2221
+ const red$1 = red$4;
2222
2222
  /**
2223
2223
  * @since 1.0.0
2224
2224
  * @category constructors
@@ -2228,7 +2228,7 @@ const magenta = magenta$1;
2228
2228
  * @since 1.0.0
2229
2229
  * @category constructors
2230
2230
  */
2231
- const cyan = cyan$1;
2231
+ const cyan$1 = cyan$2;
2232
2232
  /**
2233
2233
  * @since 1.0.0
2234
2234
  * @category constructors
@@ -2248,7 +2248,7 @@ const space = /*#__PURE__*/ text$6(" ");
2248
2248
  /** @internal */
2249
2249
  const code = (value) => highlight(value, white);
2250
2250
  /** @internal */
2251
- const error = (value) => highlight(value, red);
2251
+ const error = (value) => highlight(value, red$1);
2252
2252
  /** @internal */
2253
2253
  const highlight = (value, color) => ({
2254
2254
  _tag: "Highlight",
@@ -2307,7 +2307,7 @@ const toAnsiDoc$1 = (self) => {
2307
2307
  switch (self._tag) {
2308
2308
  case "Highlight": return annotate(toAnsiDoc$1(self.value), color(self.color));
2309
2309
  case "Sequence": return cat(toAnsiDoc$1(self.left), toAnsiDoc$1(self.right));
2310
- case "Strong": return annotate(toAnsiDoc$1(self.value), bold);
2310
+ case "Strong": return annotate(toAnsiDoc$1(self.value), bold$1);
2311
2311
  case "Text": return text$7(self.value);
2312
2312
  case "URI": return annotate(text$7(self.value), underlined);
2313
2313
  case "Weak": return annotate(toAnsiDoc$1(self.value), blackBright);
@@ -2373,11 +2373,11 @@ const toAnsiText = (self) => render$1(toAnsiDoc(self), { style: "pretty" });
2373
2373
  const toAnsiDocInternal = (self) => {
2374
2374
  switch (self._tag) {
2375
2375
  case "Empty": return empty$3;
2376
- case "Header": return pipe(annotate(toAnsiDoc$1(self.value), bold), cat(hardLine));
2376
+ case "Header": return pipe(annotate(toAnsiDoc$1(self.value), bold$1), cat(hardLine));
2377
2377
  case "Paragraph": return pipe(toAnsiDoc$1(self.value), cat(hardLine));
2378
2378
  case "DescriptionList": {
2379
2379
  const definitions = self.definitions.map(([span, doc]) => cats([
2380
- annotate(toAnsiDoc$1(span), bold),
2380
+ annotate(toAnsiDoc$1(span), bold$1),
2381
2381
  empty$3,
2382
2382
  indent(toAnsiDocInternal(doc), 2)
2383
2383
  ]));
@@ -2592,8 +2592,8 @@ function renderError$2(state, pointer) {
2592
2592
  onSome: (error) => {
2593
2593
  const errorLines = error.split(NEWLINE_REGEX$3);
2594
2594
  if (Arr.isNonEmptyReadonlyArray(errorLines)) {
2595
- const annotateLine = (line) => annotate(text$7(line), combine(italicized, red$1));
2596
- const prefix = cat(annotate(pointer, red$1), space$1);
2595
+ const annotateLine = (line) => annotate(text$7(line), combine(italicized, red$2));
2596
+ const prefix = cat(annotate(pointer, red$2), space$1);
2597
2597
  const lines = Arr.map(errorLines, (str) => annotateLine(str));
2598
2598
  return cursorSavePosition.pipe(cat(hardLine), cat(prefix), cat(align(vsep(lines))), cat(cursorRestorePosition));
2599
2599
  }
@@ -2612,7 +2612,7 @@ function renderParts(state, submitted = false) {
2612
2612
  });
2613
2613
  }
2614
2614
  function renderOutput$4(leadingSymbol, trailingSymbol, parts, options) {
2615
- const annotateLine = (line) => annotate(text$7(line), bold);
2615
+ const annotateLine = (line) => annotate(text$7(line), bold$1);
2616
2616
  const prefix = cat(leadingSymbol, space$1);
2617
2617
  return Arr.match(options.message.split(/\r?\n/), {
2618
2618
  onEmpty: () => hsep([
@@ -2642,7 +2642,7 @@ function renderSubmission$5(state, options) {
2642
2642
  return Effect$1.gen(function* () {
2643
2643
  const columns = yield* (yield* Terminal).columns;
2644
2644
  const figures$11 = yield* figures;
2645
- return renderOutput$4(annotate(figures$11.tick, green), annotate(figures$11.ellipsis, blackBright), renderParts(state, true), options).pipe(cat(hardLine), optimize(Deep), render$1({
2645
+ return renderOutput$4(annotate(figures$11.tick, green$1), annotate(figures$11.ellipsis, blackBright), renderParts(state, true), options).pipe(cat(hardLine), optimize(Deep), render$1({
2646
2646
  style: "pretty",
2647
2647
  options: { lineWidth: columns }
2648
2648
  }));
@@ -3106,7 +3106,7 @@ function handleClear$4(options) {
3106
3106
  }
3107
3107
  const NEWLINE_REGEX$2 = /\r?\n/;
3108
3108
  function renderPrompt(confirm, message, leadingSymbol, trailingSymbol) {
3109
- const annotateLine = (line) => annotate(text$7(line), bold);
3109
+ const annotateLine = (line) => annotate(text$7(line), bold$1);
3110
3110
  const prefix = cat(leadingSymbol, space$1);
3111
3111
  return Arr.match(message.split(NEWLINE_REGEX$2), {
3112
3112
  onEmpty: () => hsep([
@@ -3159,7 +3159,7 @@ function renderNextFrame$4(state, options) {
3159
3159
  options: { lineWidth: columns }
3160
3160
  }));
3161
3161
  }
3162
- const leadingSymbol = annotate(figures$10.tick, green);
3162
+ const leadingSymbol = annotate(figures$10.tick, green$1);
3163
3163
  const trailingSymbol = annotate(figures$10.ellipsis, blackBright);
3164
3164
  const promptMsg = renderPrompt(empty$3, options.message, leadingSymbol, trailingSymbol);
3165
3165
  const files = renderFiles(state, state.files, figures$10, options);
@@ -3173,7 +3173,7 @@ function renderSubmission$4(value, options) {
3173
3173
  return Effect$1.gen(function* () {
3174
3174
  const columns = yield* (yield* Terminal).columns;
3175
3175
  const figures$9 = yield* figures;
3176
- const leadingSymbol = annotate(figures$9.tick, green);
3176
+ const leadingSymbol = annotate(figures$9.tick, green$1);
3177
3177
  const trailingSymbol = annotate(figures$9.ellipsis, blackBright);
3178
3178
  return renderPrompt(empty$3, options.message, leadingSymbol, trailingSymbol).pipe(cat(space$1), cat(annotate(text$7(value), white$1)), cat(hardLine), render$1({
3179
3179
  style: "pretty",
@@ -3316,7 +3316,7 @@ function handleClear$3(options) {
3316
3316
  function renderInput$1(state, submitted) {
3317
3317
  const annotation = Option$1.match(state.error, {
3318
3318
  onNone: () => combine(underlined, cyanBright),
3319
- onSome: () => red$1
3319
+ onSome: () => red$2
3320
3320
  });
3321
3321
  const value = state.value === "" ? empty$3 : text$7(`${state.value}`);
3322
3322
  return submitted ? value : annotate(value, annotation);
@@ -3328,8 +3328,8 @@ function renderError$1(state, pointer) {
3328
3328
  onSome: (error) => Arr.match(error.split(NEWLINE_REGEX$1), {
3329
3329
  onEmpty: () => empty$3,
3330
3330
  onNonEmpty: (errorLines) => {
3331
- const annotateLine = (line) => annotate(text$7(line), combine(italicized, red$1));
3332
- const prefix = cat(annotate(pointer, red$1), space$1);
3331
+ const annotateLine = (line) => annotate(text$7(line), combine(italicized, red$2));
3332
+ const prefix = cat(annotate(pointer, red$2), space$1);
3333
3333
  const lines = Arr.map(errorLines, (str) => annotateLine(str));
3334
3334
  return cursorSavePosition.pipe(cat(hardLine), cat(prefix), cat(align(vsep(lines))), cat(cursorRestorePosition));
3335
3335
  }
@@ -3337,7 +3337,7 @@ function renderError$1(state, pointer) {
3337
3337
  });
3338
3338
  }
3339
3339
  function renderOutput$3(state, leadingSymbol, trailingSymbol, options, submitted = false) {
3340
- const annotateLine = (line) => annotate(text$7(line), bold);
3340
+ const annotateLine = (line) => annotate(text$7(line), bold$1);
3341
3341
  const prefix = cat(leadingSymbol, space$1);
3342
3342
  return Arr.match(options.message.split(/\r?\n/), {
3343
3343
  onEmpty: () => hsep([
@@ -3368,7 +3368,7 @@ function renderSubmission$3(nextState, options) {
3368
3368
  return Effect$1.gen(function* () {
3369
3369
  const columns = yield* (yield* Terminal).columns;
3370
3370
  const figures$7 = yield* figures;
3371
- return renderOutput$3(nextState, annotate(figures$7.tick, green), annotate(figures$7.ellipsis, blackBright), options, true).pipe(cat(hardLine), optimize(Deep), render$1({
3371
+ return renderOutput$3(nextState, annotate(figures$7.tick, green$1), annotate(figures$7.ellipsis, blackBright), options, true).pipe(cat(hardLine), optimize(Deep), render$1({
3372
3372
  style: "pretty",
3373
3373
  options: { lineWidth: columns }
3374
3374
  }));
@@ -3551,7 +3551,7 @@ const float = (options) => {
3551
3551
  const renderBeep$2 = /*#__PURE__*/ render$1(beep, { style: "pretty" });
3552
3552
  const NEWLINE_REGEX = /\r?\n/;
3553
3553
  function renderOutput$2(leadingSymbol, trailingSymbol, options) {
3554
- const annotateLine = (line) => annotate(text$7(line), bold);
3554
+ const annotateLine = (line) => annotate(text$7(line), bold$1);
3555
3555
  const prefix = cat(leadingSymbol, space$1);
3556
3556
  return Arr.match(options.message.split(NEWLINE_REGEX), {
3557
3557
  onEmpty: () => hsep([prefix, trailingSymbol]),
@@ -3566,7 +3566,7 @@ function renderChoicePrefix(state, choices, toDisplay, currentIndex, figures) {
3566
3566
  if (currentIndex === toDisplay.startIndex && toDisplay.startIndex > 0) prefix = figures.arrowUp;
3567
3567
  else if (currentIndex === toDisplay.endIndex - 1 && toDisplay.endIndex < choices.length) prefix = figures.arrowDown;
3568
3568
  if (choices[currentIndex].disabled) {
3569
- const annotation = combine(bold, blackBright);
3569
+ const annotation = combine(bold$1, blackBright);
3570
3570
  return state === currentIndex ? figures.pointer.pipe(annotate(annotation), cat(prefix)) : prefix.pipe(cat(space$1));
3571
3571
  }
3572
3572
  return state === currentIndex ? figures.pointer.pipe(annotate(cyanBright), cat(prefix)) : prefix.pipe(cat(space$1));
@@ -3611,7 +3611,7 @@ function renderSubmission$2(state, options) {
3611
3611
  const columns = yield* (yield* Terminal).columns;
3612
3612
  const figures$6 = yield* figures;
3613
3613
  const selected = text$7(options.choices[state].title);
3614
- return renderOutput$2(annotate(figures$6.tick, green), annotate(figures$6.ellipsis, blackBright), options).pipe(cat(space$1), cat(annotate(selected, white$1)), cat(hardLine), render$1({
3614
+ return renderOutput$2(annotate(figures$6.tick, green$1), annotate(figures$6.ellipsis, blackBright), options).pipe(cat(space$1), cat(annotate(selected, white$1)), cat(hardLine), render$1({
3615
3615
  style: "pretty",
3616
3616
  options: { lineWidth: columns }
3617
3617
  }));
@@ -3714,7 +3714,7 @@ function renderInput(nextState, options, submitted) {
3714
3714
  if (nextState.value.length === 0) return blackBright;
3715
3715
  return combine(underlined, cyanBright);
3716
3716
  },
3717
- onSome: () => red$1
3717
+ onSome: () => red$2
3718
3718
  });
3719
3719
  switch (options.type) {
3720
3720
  case "hidden": return empty$3;
@@ -3728,8 +3728,8 @@ function renderError(nextState, pointer) {
3728
3728
  onSome: (error) => Arr.match(error.split(/\r?\n/), {
3729
3729
  onEmpty: () => empty$3,
3730
3730
  onNonEmpty: (errorLines) => {
3731
- const annotateLine = (line) => text$7(line).pipe(annotate(combine(italicized, red$1)));
3732
- const prefix = cat(annotate(pointer, red$1), space$1);
3731
+ const annotateLine = (line) => text$7(line).pipe(annotate(combine(italicized, red$2)));
3732
+ const prefix = cat(annotate(pointer, red$2), space$1);
3733
3733
  const lines = Arr.map(errorLines, (str) => annotateLine(str));
3734
3734
  return cursorSavePosition.pipe(cat(hardLine), cat(prefix), cat(align(vsep(lines))), cat(cursorRestorePosition));
3735
3735
  }
@@ -3737,7 +3737,7 @@ function renderError(nextState, pointer) {
3737
3737
  });
3738
3738
  }
3739
3739
  function renderOutput$1(nextState, leadingSymbol, trailingSymbol, options, submitted = false) {
3740
- const annotateLine = (line) => annotate(text$7(line), bold);
3740
+ const annotateLine = (line) => annotate(text$7(line), bold$1);
3741
3741
  const promptLines = options.message.split(/\r?\n/);
3742
3742
  const prefix = cat(leadingSymbol, space$1);
3743
3743
  if (Arr.isNonEmptyReadonlyArray(promptLines)) {
@@ -3767,7 +3767,7 @@ function renderSubmission$1(state, options) {
3767
3767
  return Effect$1.gen(function* () {
3768
3768
  const columns = yield* (yield* Terminal).columns;
3769
3769
  const figures$4 = yield* figures;
3770
- return renderOutput$1(state, annotate(figures$4.tick, green), annotate(figures$4.ellipsis, blackBright), options, true).pipe(cat(hardLine), optimize(Deep), render$1({
3770
+ return renderOutput$1(state, annotate(figures$4.tick, green$1), annotate(figures$4.ellipsis, blackBright), options, true).pipe(cat(hardLine), optimize(Deep), render$1({
3771
3771
  style: "pretty",
3772
3772
  options: { lineWidth: columns }
3773
3773
  }));
@@ -3909,7 +3909,7 @@ function renderToggle(value, options, submitted = false) {
3909
3909
  ]);
3910
3910
  }
3911
3911
  function renderOutput(toggle, leadingSymbol, trailingSymbol, options) {
3912
- const annotateLine = (line) => annotate(text$7(line), bold);
3912
+ const annotateLine = (line) => annotate(text$7(line), bold$1);
3913
3913
  const promptLines = options.message.split(/\r?\n/);
3914
3914
  const prefix = cat(leadingSymbol, space$1);
3915
3915
  if (Arr.isNonEmptyReadonlyArray(promptLines)) {
@@ -3941,7 +3941,7 @@ function renderSubmission(value, options) {
3941
3941
  const terminal = yield* Terminal;
3942
3942
  const figures$2 = yield* figures;
3943
3943
  const columns = yield* terminal.columns;
3944
- const leadingSymbol = annotate(figures$2.tick, green);
3944
+ const leadingSymbol = annotate(figures$2.tick, green$1);
3945
3945
  const trailingSymbol = annotate(figures$2.ellipsis, blackBright);
3946
3946
  return renderOutput(renderToggle(value, options, true), leadingSymbol, trailingSymbol, options).pipe(cat(hardLine), optimize(Deep), render$1({
3947
3947
  style: "pretty",
@@ -5919,7 +5919,7 @@ const completionsOptions = /*#__PURE__*/ choiceWithValue("completions", [
5919
5919
  ["zsh", "zsh"]
5920
5920
  ]).pipe(optional$1, /*#__PURE__*/ withDescription("Generate a completion script for a specific shell."));
5921
5921
  /** @internal */
5922
- const logLevelOptions = /*#__PURE__*/ choiceWithValue("log-level", LogLevel.allLevels.map((level) => [level._tag.toLowerCase(), level])).pipe(optional$1, /*#__PURE__*/ withDescription("Sets the minimum log level for a command."));
5922
+ const logLevelOptions = /*#__PURE__*/ choiceWithValue("log-level", LogLevel$1.allLevels.map((level) => [level._tag.toLowerCase(), level])).pipe(optional$1, /*#__PURE__*/ withDescription("Sets the minimum log level for a command."));
5923
5923
  /** @internal */
5924
5924
  const helpOptions = /*#__PURE__*/ boolean$1("help").pipe(/*#__PURE__*/ withAlias("h"), /*#__PURE__*/ withDescription("Show the help documentation for a command."));
5925
5925
  /** @internal */
@@ -6294,7 +6294,7 @@ const wizardInternal = (self, prefix, config) => {
6294
6294
  case "GetUserInput":
6295
6295
  case "Standard": return Effect$1.gen(function* () {
6296
6296
  const logCurrentCommand = Ref.get(commandLineRef).pipe(Effect$1.flatMap((commandLine) => {
6297
- const currentCommand = p(pipe(strong(highlight("COMMAND:", cyan)), concat$1(space), concat$1(highlight(Arr.join(commandLine, " "), magenta))));
6297
+ const currentCommand = p(pipe(strong(highlight("COMMAND:", cyan$1)), concat$1(space), concat$1(highlight(Arr.join(commandLine, " "), magenta))));
6298
6298
  return Console.log(toAnsiText(currentCommand));
6299
6299
  }));
6300
6300
  if (isStandard(self)) {
@@ -6634,7 +6634,7 @@ const handleBuiltInOption = (self, executable, args, builtIn, execute, config, o
6634
6634
  filteredArgs.push(args[i]);
6635
6635
  }
6636
6636
  const nextArgs = Arr.appendAll(baseArgs, filteredArgs);
6637
- return run$2(self, nextArgs, execute).pipe(Logger.withMinimumLogLevel(builtIn.level));
6637
+ return run$2(self, nextArgs, execute).pipe(Logger$1.withMinimumLogLevel(builtIn.level));
6638
6638
  }
6639
6639
  case "ShowHelp": {
6640
6640
  const banner = h1(code(self.name));
@@ -6740,7 +6740,7 @@ const renderWizardArgs = (args) => {
6740
6740
  return blocks([
6741
6741
  p(strong(code("Wizard Mode Complete!"))),
6742
6742
  p(executeMsg),
6743
- p(concat$1(text$6(" "), highlight(params, cyan)))
6743
+ p(concat$1(text$6(" "), highlight(params, cyan$1)))
6744
6744
  ]);
6745
6745
  };
6746
6746
  const isLogLevelArg = (arg) => {
@@ -7775,7 +7775,7 @@ const ProcessProto = {
7775
7775
  };
7776
7776
  }
7777
7777
  };
7778
- const runCommand = (fileSystem) => (command) => {
7778
+ const runCommand$1 = (fileSystem) => (command) => {
7779
7779
  switch (command._tag) {
7780
7780
  case "StandardCommand": {
7781
7781
  const spawn = Effect$1.flatMap(Deferred.make(), (exitCode) => Effect$1.async((resume) => {
@@ -7851,13 +7851,13 @@ const runCommand = (fileSystem) => (command) => {
7851
7851
  }
7852
7852
  case "PipedCommand": {
7853
7853
  const flattened = flatten(command);
7854
- if (flattened.length === 1) return pipe(flattened[0], runCommand(fileSystem));
7854
+ if (flattened.length === 1) return pipe(flattened[0], runCommand$1(fileSystem));
7855
7855
  const head = flattened[0];
7856
7856
  const tail = flattened.slice(1);
7857
7857
  const initial = tail.slice(0, tail.length - 1);
7858
7858
  const last = tail[tail.length - 1];
7859
- const stream = initial.reduce((stdin$2, command) => pipe(stdin(command, stdin$2), runCommand(fileSystem), Effect$1.map((process) => process.stdout), Stream.unwrapScoped), pipe(runCommand(fileSystem)(head), Effect$1.map((process) => process.stdout), Stream.unwrapScoped));
7860
- return pipe(stdin(last, stream), runCommand(fileSystem));
7859
+ const stream = initial.reduce((stdin$2, command) => pipe(stdin(command, stdin$2), runCommand$1(fileSystem), Effect$1.map((process) => process.stdout), Stream.unwrapScoped), pipe(runCommand$1(fileSystem)(head), Effect$1.map((process) => process.stdout), Stream.unwrapScoped));
7860
+ return pipe(stdin(last, stream), runCommand$1(fileSystem));
7861
7861
  }
7862
7862
  }
7863
7863
  };
@@ -7867,7 +7867,7 @@ const runCommand = (fileSystem) => (command) => {
7867
7867
  * @since 1.0.0
7868
7868
  * @category layer
7869
7869
  */
7870
- const layer$5 = /* @__PURE__ */ Layer.effect(CommandExecutor, /*#__PURE__*/ pipe(FileSystem, /*#__PURE__*/ Effect$1.map((fileSystem) => makeExecutor(runCommand(fileSystem)))));
7870
+ const layer$5 = /* @__PURE__ */ Layer.effect(CommandExecutor, /*#__PURE__*/ pipe(FileSystem, /*#__PURE__*/ Effect$1.map((fileSystem) => makeExecutor(runCommand$1(fileSystem)))));
7871
7871
  //#endregion
7872
7872
  //#region ../../node_modules/.bun/@effect+platform-node-shared@0.61.1+1edbab697a2570fc/node_modules/@effect/platform-node-shared/dist/esm/internal/path.js
7873
7873
  const fromFileUrl = (url) => Effect$1.try({
@@ -8048,65 +8048,10 @@ const layerManager = /* @__PURE__ */ Layer.provide(layerManager$2, layerWorker);
8048
8048
  */
8049
8049
  const layer = /*#__PURE__*/ pipe(/*#__PURE__*/ Layer.mergeAll(layer$3, layer$5, layer$1, layerManager), /*#__PURE__*/ Layer.provideMerge(layer$7));
8050
8050
  //#endregion
8051
- //#region ../cli/src/dashboard-cells.ts
8052
- const MAX_SCAN = 1e5;
8053
- function display(value) {
8054
- return typeof value === "string" ? value : JSON.stringify(value) ?? String(value);
8055
- }
8056
- /** Bounded report projection; both count and rows come from the core eligible-cell iterator. */
8057
- function listMatrixCells(matrix, input) {
8058
- if (!Number.isSafeInteger(input.offset) || input.offset < 0 || !Number.isSafeInteger(input.limit) || input.limit < 1 || input.limit > 100) throw new Error("Matrix page requires offset >= 0 and limit between 1 and 100");
8059
- if (![
8060
- "eval",
8061
- "agent",
8062
- "trials",
8063
- ...[...Object.keys(matrix.cases?.[0] ?? {}), ...Object.keys(matrix.parameters)].map((key) => `axis:${key}`)
8064
- ].includes(input.sort)) throw new Error(`Unknown matrix sort: ${input.sort}`);
8065
- const selection = input.evalIds.length ? { evals: input.evalIds } : {};
8066
- const planned = matrix.count(selection);
8067
- if (planned > MAX_SCAN) throw new Error(`Matrix has ${planned} eligible cells; narrow the selection in the CLI (dashboard limit ${MAX_SCAN})`);
8068
- const terms = input.query.trim().toLowerCase().split(/\s+/).filter(Boolean);
8069
- const cells = [...matrix.cells(selection)].filter((cell) => {
8070
- const haystack = [
8071
- cell.eval.id,
8072
- cell.eval.name,
8073
- cell.eval.agent?.identity?.name,
8074
- cell.eval.agent?.identity?.kind,
8075
- ...Object.entries(cell.parameters).flatMap(([key, value]) => [key, display(value)])
8076
- ].map(display).join(" ").toLowerCase();
8077
- return terms.every((term) => haystack.includes(term));
8078
- });
8079
- function value(cell) {
8080
- if (input.sort.startsWith("axis:")) {
8081
- const parameter = cell.parameters[input.sort.slice(5)];
8082
- return typeof parameter === "number" ? parameter : canonicalParameters(parameter);
8083
- }
8084
- if (input.sort === "eval") return cell.eval.name ?? cell.eval.id;
8085
- if (input.sort === "agent") return cell.eval.agent?.identity?.name ?? cell.eval.agent?.identity?.kind ?? "";
8086
- return cell.eval.policy?.trials ?? 1;
8087
- }
8088
- cells.sort((left, right) => {
8089
- const a = value(left);
8090
- const b = value(right);
8091
- const comparison = typeof a === "number" && typeof b === "number" ? a - b : String(a).localeCompare(String(b), void 0, {
8092
- numeric: true,
8093
- sensitivity: "base"
8094
- });
8095
- return (input.direction === "asc" ? comparison : -comparison) || left.key.localeCompare(right.key);
8096
- });
8097
- return {
8098
- total: cells.length,
8099
- planned,
8100
- cells: cells.slice(input.offset, input.offset + input.limit).map((cell) => ({
8101
- evalId: cell.eval.id,
8102
- key: cell.key,
8103
- parameters: cell.parameters
8104
- }))
8105
- };
8106
- }
8107
- //#endregion
8108
8051
  //#region ../cli/src/dashboard-reports.ts
8109
8052
  const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
8053
+ /** Written into a run directory by `evalkit tidy`; deleting it restores the run. */
8054
+ const ARCHIVE_MARKER = "archived.json";
8110
8055
  function isMissing(error) {
8111
8056
  return error.code === "ENOENT";
8112
8057
  }
@@ -8143,15 +8088,35 @@ function fileContentType(file) {
8143
8088
  ".js": "text/javascript"
8144
8089
  }[extname(file)] ?? "application/octet-stream";
8145
8090
  }
8146
- /** Read saved local reports. A missing summary means "running"; malformed reports are errors. */
8091
+ /**
8092
+ * Read saved local reports. A missing summary means "running"; malformed reports are errors.
8093
+ * Runs written by an older EvalKit (manifest `schemaVersion` < 2) are skipped and counted.
8094
+ */
8147
8095
  function createDashboardReportReader(root, registry) {
8096
+ const legacyRuns = /* @__PURE__ */ new Set();
8148
8097
  async function listRuns() {
8149
8098
  const entries = await ifPresent(() => readdir(root, { withFileTypes: true }));
8150
8099
  if (!entries) return [];
8151
8100
  const ids = entries.filter((entry) => entry.isDirectory() && UUID_PATTERN.test(entry.name)).map((entry) => entry.name);
8152
8101
  return (await Promise.all(ids.map(readRun))).filter((run) => run !== void 0);
8153
8102
  }
8103
+ /** True for a run from a report format this version cannot read; never for corrupt JSON. */
8104
+ async function isLegacyRun(id) {
8105
+ const text = await ifPresent(() => readFile(resolve(root, id, "manifest.json"), "utf8"));
8106
+ if (text === void 0) return false;
8107
+ let version;
8108
+ try {
8109
+ version = JSON.parse(text).schemaVersion;
8110
+ } catch {
8111
+ return false;
8112
+ }
8113
+ return typeof version === "number" && version < 2;
8114
+ }
8154
8115
  async function readRun(id) {
8116
+ if (await isLegacyRun(id)) {
8117
+ legacyRuns.add(id);
8118
+ return;
8119
+ }
8155
8120
  const manifest = await ifPresent(() => readRunManifest(root, id));
8156
8121
  if (!manifest) return void 0;
8157
8122
  const summary = await ifPresent(() => readRunSummary(root, id));
@@ -8160,6 +8125,7 @@ function createDashboardReportReader(root, registry) {
8160
8125
  const scores = trials.flatMap((trial) => trial?.scoring?.overall === void 0 ? [] : [trial.scoring.overall]);
8161
8126
  return {
8162
8127
  id,
8128
+ ...await ifPresent(() => stat(resolve(root, id, "archived.json"))) ? { archived: true } : {},
8163
8129
  evalId: manifest.evalId,
8164
8130
  ...manifest.suiteId ? { suiteId: manifest.suiteId } : {},
8165
8131
  ...manifest.matrix ? {
@@ -8228,6 +8194,8 @@ function createDashboardReportReader(root, registry) {
8228
8194
  }
8229
8195
  return {
8230
8196
  listRuns,
8197
+ /** Run IDs skipped by the last `listRuns()` because an older EvalKit wrote them. */
8198
+ legacyRunIds: () => [...legacyRuns],
8231
8199
  listTrials,
8232
8200
  trialDetail,
8233
8201
  listArtifacts,
@@ -8259,6 +8227,235 @@ async function walkArtifacts(root) {
8259
8227
  return result;
8260
8228
  }
8261
8229
  //#endregion
8230
+ //#region ../cli/src/dashboard-build.ts
8231
+ /** Marks a directory as EvalKit-owned so a rebuild may safely replace it. */
8232
+ const MARKER = ".evalkit-dashboard";
8233
+ const quietStep = (_label, work) => work(() => {});
8234
+ async function buildDashboard(options, step = quietStep) {
8235
+ const { registry, reportRoot, dashboardRoot, outDir } = options;
8236
+ const template = await readFile(resolve(dashboardRoot, "index.html"), "utf8");
8237
+ const data = resolve(outDir, "data");
8238
+ const write = async (path, value) => {
8239
+ const file = resolve(data, path);
8240
+ await mkdir(dirname(file), { recursive: true });
8241
+ await writeFile(file, `${JSON.stringify(value)}\n`);
8242
+ };
8243
+ await step("Dashboard assets", async () => {
8244
+ await prepareOutDir(outDir);
8245
+ await cp(dashboardRoot, outDir, { recursive: true });
8246
+ await writeFile(resolve(outDir, MARKER), "Generated by `evalkit build`.\n");
8247
+ await writeFile(resolve(outDir, "index.html"), staticIndex(template));
8248
+ }, () => "read-only UI with relative assets");
8249
+ await step("Eval catalog", async () => {
8250
+ await write("catalog.json", { evals: registry.catalog() });
8251
+ await write("evals.json", { evals: registry.metadata() });
8252
+ await write("suites.json", { suites: registry.suiteMetadata() });
8253
+ return {
8254
+ evals: registry.evals.length,
8255
+ suites: registry.suites.length
8256
+ };
8257
+ }, ({ evals, suites }) => `${count(evals, "eval")} · ${count(suites, "suite")}`);
8258
+ const reports = createDashboardReportReader(reportRoot, registry);
8259
+ const snapshot = await step("Run reports", async (progress) => {
8260
+ const all = await reports.listRuns();
8261
+ const runs = all.filter((run) => !run.archived);
8262
+ await write("runs.json", { runs });
8263
+ let trials = 0;
8264
+ let files = 0;
8265
+ const trialIds = [];
8266
+ for (const [index, run] of runs.entries()) {
8267
+ progress(`run ${index + 1}/${runs.length} · ${run.id.slice(0, 8)}`);
8268
+ const runTrials = await reports.listTrials(run.id);
8269
+ await write(`runs/${run.id}/trials.json`, { trials: runTrials });
8270
+ for (const trial of runTrials) {
8271
+ trials += 1;
8272
+ trialIds.push(trial.id);
8273
+ files += await snapshotTrial(reports, write, data, run.id, trial.id);
8274
+ }
8275
+ }
8276
+ return {
8277
+ runs: runs.length,
8278
+ trials,
8279
+ files,
8280
+ skipped: reports.legacyRunIds().length,
8281
+ archived: all.length - runs.length,
8282
+ runIds: runs.map((run) => run.id),
8283
+ trialIds
8284
+ };
8285
+ }, ({ runs, trials, files, skipped, archived }) => `${count(runs, "run")} · ${count(trials, "trial")} · ${count(files, "candidate file")}` + (archived ? ` · ${archived} archived excluded` : "") + (skipped ? ` · ${skipped} older-format skipped` : ""));
8286
+ await step("Route pages", async () => {
8287
+ const routes = staticRoutes({
8288
+ suites: registry.suites.map(authoringId),
8289
+ evals: registry.catalog().map((entry) => entry.path),
8290
+ runs: snapshot.runIds,
8291
+ trials: snapshot.trialIds
8292
+ });
8293
+ for (const route of routes) {
8294
+ const file = resolve(outDir, `${route}.html`);
8295
+ await mkdir(dirname(file), { recursive: true });
8296
+ await writeFile(file, staticIndex(template, baseFor(route)));
8297
+ }
8298
+ await writeFile(resolve(outDir, "404.html"), staticIndex(template, "/"));
8299
+ return routes.length;
8300
+ }, (pages) => `${count(pages, "page")} with clean /path URLs`);
8301
+ const { runIds: _runs, trialIds: _trials, ...summary } = snapshot;
8302
+ return {
8303
+ outDir,
8304
+ ...summary
8305
+ };
8306
+ }
8307
+ /**
8308
+ * Every dashboard URL known at build time, as file paths without `.html`.
8309
+ * Static hosts serve `/runs/abc` from `runs/abc.html`, so links and reloads
8310
+ * work without SPA rewrite rules. IDs that cannot be a safe file name are
8311
+ * skipped; those pages still open by navigating inside the app.
8312
+ */
8313
+ function staticRoutes(ids) {
8314
+ const safe = (id) => id.length > 0 && !/[/\\\0]/.test(id) && id !== "." && id !== "..";
8315
+ const under = (prefix, values) => values.filter(safe).map((id) => `${prefix}/${id}`);
8316
+ return [
8317
+ "suites",
8318
+ "agents",
8319
+ "fixtures",
8320
+ "runs",
8321
+ ...under("suites", ids.suites),
8322
+ ...under("evals", ids.evals),
8323
+ ...under("runs", ids.runs),
8324
+ ...under("trials", ids.trials),
8325
+ ...under("workspaces", ids.trials)
8326
+ ];
8327
+ }
8328
+ /** `runs/abc.html` is served at `<mount>/runs/abc`; its base is one level up. */
8329
+ function baseFor(route) {
8330
+ const depth = route.split("/").length - 1;
8331
+ return depth ? "../".repeat(depth) : "./";
8332
+ }
8333
+ function count(value, noun) {
8334
+ return `${value} ${noun}${value === 1 ? "" : "s"}`;
8335
+ }
8336
+ /** Writes one trial's JSON views and candidate files; returns the file count. */
8337
+ async function snapshotTrial(reports, write, data, runId, trialId) {
8338
+ const base = `runs/${runId}/trials/${trialId}`;
8339
+ await write(`${base}/detail.json`, await reports.trialDetail(runId, trialId));
8340
+ await write(`${base}/events.json`, { events: await reports.trajectory(runId, trialId) });
8341
+ await write(`${base}/artifacts.json`, { artifacts: await reports.listArtifacts(runId, trialId) });
8342
+ const workspace = await reports.listCandidateWorkspace(runId, trialId);
8343
+ await write(`${base}/workspace.json`, workspace);
8344
+ const root = resolve(data, base, "workspace");
8345
+ let files = 0;
8346
+ for (const entry of workspace) {
8347
+ if (entry.kind !== "file") continue;
8348
+ const target = resolve(root, entry.path);
8349
+ if (!target.startsWith(`${root}${sep}`)) throw new Error(`Invalid workspace path: ${entry.path}`);
8350
+ const response = await reports.candidateFile(runId, trialId, entry.path);
8351
+ await mkdir(dirname(target), { recursive: true });
8352
+ await writeFile(target, new Uint8Array(await response.arrayBuffer()));
8353
+ files += 1;
8354
+ }
8355
+ return files;
8356
+ }
8357
+ /** Total size and file count of a directory, for the build summary. */
8358
+ async function directorySize(root) {
8359
+ let bytes = 0;
8360
+ let files = 0;
8361
+ for (const entry of await readdir(root, {
8362
+ withFileTypes: true,
8363
+ recursive: true
8364
+ })) {
8365
+ if (!entry.isFile()) continue;
8366
+ bytes += (await stat(resolve(entry.parentPath, entry.name))).size;
8367
+ files += 1;
8368
+ }
8369
+ return {
8370
+ bytes,
8371
+ files
8372
+ };
8373
+ }
8374
+ /**
8375
+ * Select the static API and make every URL relative to `<base href>`, which
8376
+ * points at the snapshot root. The app derives its router basename from it.
8377
+ */
8378
+ function staticIndex(html, base = "./") {
8379
+ const result = html.replace("<html lang=\"en\">", "<html lang=\"en\" data-evalkit-snapshot=\"static\">").replace("<head>", `<head>\n <base href="${base}" />`).replaceAll("\"/assets/", "\"./assets/");
8380
+ if (!result.includes("data-evalkit-snapshot=\"static\"") || !result.includes("<base href=")) throw new Error("Bundled dashboard index.html has an unexpected shape");
8381
+ return result;
8382
+ }
8383
+ async function prepareOutDir(outDir) {
8384
+ let entries;
8385
+ try {
8386
+ if (!(await stat(outDir)).isDirectory()) throw new Error(`Output path is not a directory: ${outDir}`);
8387
+ entries = await readdir(outDir);
8388
+ } catch (error) {
8389
+ if (error.code !== "ENOENT") throw error;
8390
+ await mkdir(outDir, { recursive: true });
8391
+ return;
8392
+ }
8393
+ if (!entries.length) return;
8394
+ if (!entries.includes(MARKER)) throw new Error(`Refusing to overwrite ${outDir}: it is not empty and was not created by evalkit build`);
8395
+ await rm(outDir, {
8396
+ recursive: true,
8397
+ force: true
8398
+ });
8399
+ await mkdir(outDir, { recursive: true });
8400
+ }
8401
+ //#endregion
8402
+ //#region ../cli/src/dashboard-cells.ts
8403
+ const MAX_SCAN = 1e5;
8404
+ function display(value) {
8405
+ return typeof value === "string" ? value : JSON.stringify(value) ?? String(value);
8406
+ }
8407
+ /** Bounded report projection; both count and rows come from the core eligible-cell iterator. */
8408
+ function listMatrixCells(matrix, input) {
8409
+ if (!Number.isSafeInteger(input.offset) || input.offset < 0 || !Number.isSafeInteger(input.limit) || input.limit < 1 || input.limit > 100) throw new Error("Matrix page requires offset >= 0 and limit between 1 and 100");
8410
+ if (![
8411
+ "eval",
8412
+ "agent",
8413
+ "trials",
8414
+ ...[...Object.keys(matrix.cases?.[0] ?? {}), ...Object.keys(matrix.parameters)].map((key) => `axis:${key}`)
8415
+ ].includes(input.sort)) throw new Error(`Unknown matrix sort: ${input.sort}`);
8416
+ const selection = input.evalIds.length ? { evals: input.evalIds } : {};
8417
+ const planned = matrix.count(selection);
8418
+ if (planned > MAX_SCAN) throw new Error(`Matrix has ${planned} eligible cells; narrow the selection in the CLI (dashboard limit ${MAX_SCAN})`);
8419
+ const terms = input.query.trim().toLowerCase().split(/\s+/).filter(Boolean);
8420
+ const cells = [...matrix.cells(selection)].filter((cell) => {
8421
+ const haystack = [
8422
+ cell.eval.id,
8423
+ cell.eval.name,
8424
+ cell.eval.agent?.identity?.name,
8425
+ cell.eval.agent?.identity?.kind,
8426
+ ...Object.entries(cell.parameters).flatMap(([key, value]) => [key, display(value)])
8427
+ ].map(display).join(" ").toLowerCase();
8428
+ return terms.every((term) => haystack.includes(term));
8429
+ });
8430
+ function value(cell) {
8431
+ if (input.sort.startsWith("axis:")) {
8432
+ const parameter = cell.parameters[input.sort.slice(5)];
8433
+ return typeof parameter === "number" ? parameter : canonicalParameters(parameter);
8434
+ }
8435
+ if (input.sort === "eval") return cell.eval.name ?? cell.eval.id;
8436
+ if (input.sort === "agent") return cell.eval.agent?.identity?.name ?? cell.eval.agent?.identity?.kind ?? "";
8437
+ return cell.eval.policy?.trials ?? 1;
8438
+ }
8439
+ cells.sort((left, right) => {
8440
+ const a = value(left);
8441
+ const b = value(right);
8442
+ const comparison = typeof a === "number" && typeof b === "number" ? a - b : String(a).localeCompare(String(b), void 0, {
8443
+ numeric: true,
8444
+ sensitivity: "base"
8445
+ });
8446
+ return (input.direction === "asc" ? comparison : -comparison) || left.key.localeCompare(right.key);
8447
+ });
8448
+ return {
8449
+ total: cells.length,
8450
+ planned,
8451
+ cells: cells.slice(input.offset, input.offset + input.limit).map((cell) => ({
8452
+ evalId: cell.eval.id,
8453
+ key: cell.key,
8454
+ parameters: cell.parameters
8455
+ }))
8456
+ };
8457
+ }
8458
+ //#endregion
8262
8459
  //#region ../cli/src/dashboard-matrix.ts
8263
8460
  /** Dashboard submissions are intentionally one explicit, configured cell at a time. */
8264
8461
  function selectDashboardCell(matrix, evalId, parameters) {
@@ -8376,6 +8573,111 @@ async function loadProject(cwd, configPath) {
8376
8573
  };
8377
8574
  }
8378
8575
  //#endregion
8576
+ //#region ../cli/src/terminal.ts
8577
+ /** Human CLI output. Colors and live progress only appear on an interactive terminal. */
8578
+ const interactive = Boolean(process.stdout.isTTY);
8579
+ const colored = interactive && !process.env.NO_COLOR && process.env.TERM !== "dumb";
8580
+ const paint = (code, reset) => (text) => colored ? `\x1b[${code}m${text}\x1b[${reset}m` : text;
8581
+ const bold = paint(1, 22);
8582
+ const dim = paint(2, 22);
8583
+ const red = paint(31, 39);
8584
+ const green = paint(32, 39);
8585
+ const yellow = paint(33, 39);
8586
+ const cyan = paint(36, 39);
8587
+ function banner(command, version) {
8588
+ console.log(`\n ${bold(cyan("EvalKit"))} ${bold(command)} ${dim(`v${version}`)}\n`);
8589
+ }
8590
+ function duration(ms) {
8591
+ return ms < 1e3 ? `${Math.round(ms)}ms` : `${(ms / 1e3).toFixed(2)}s`;
8592
+ }
8593
+ function bytes(size) {
8594
+ if (size < 1024) return `${size} B`;
8595
+ if (size < 1024 ** 2) return `${(size / 1024).toFixed(1)} kB`;
8596
+ return `${(size / 1024 ** 2).toFixed(1)} MB`;
8597
+ }
8598
+ function plural(count, noun) {
8599
+ return `${count} ${noun}${count === 1 ? "" : "s"}`;
8600
+ }
8601
+ /** Times one build step, showing a live line while it runs. */
8602
+ async function step(label, work, describe) {
8603
+ const started = performance.now();
8604
+ const progress = (detail) => {
8605
+ if (interactive) process.stdout.write(`\r\x1b[2K ${cyan("◇")} ${label.padEnd(20)} ${dim(detail)}`);
8606
+ };
8607
+ progress("…");
8608
+ try {
8609
+ const value = await work(progress);
8610
+ const elapsed = duration(performance.now() - started);
8611
+ if (interactive) process.stdout.write("\r\x1B[2K");
8612
+ console.log(` ${green("✓")} ${label.padEnd(20)} ${describe(value)} ${dim(elapsed)}`);
8613
+ return value;
8614
+ } catch (error) {
8615
+ if (interactive) process.stdout.write("\r\x1B[2K");
8616
+ console.log(` ${red("✗")} ${label}`);
8617
+ throw error;
8618
+ }
8619
+ }
8620
+ function clock() {
8621
+ return dim((/* @__PURE__ */ new Date()).toTimeString().slice(0, 8));
8622
+ }
8623
+ function cellLabel(name, parameters) {
8624
+ const shown = Object.entries(parameters).map(([key, value]) => `${key}=${typeof value === "string" ? value : JSON.stringify(value)}`).join(" ");
8625
+ return shown ? `${bold(name)} ${dim(shown)}` : bold(name);
8626
+ }
8627
+ function passed(result) {
8628
+ return result.status === "completed" && (result.aggregateScoring?.passRate === 1 || Boolean(result.scoring?.passed));
8629
+ }
8630
+ /** Live log for runs launched from the `evalkit dev` dashboard. */
8631
+ const runLog = {
8632
+ started(name, parameters = {}, suiteId) {
8633
+ const suite = suiteId ? dim(` (suite ${suiteId})`) : "";
8634
+ console.log(`${clock()} ${cyan("▶ run ")} ${cellLabel(name, parameters)}${suite}`);
8635
+ },
8636
+ finished(name, parameters, result, ms) {
8637
+ const ok = passed(result);
8638
+ const trials = result.trials?.length ?? 1;
8639
+ const passedTrials = result.aggregateScoring?.passed ?? (ok ? trials : 0);
8640
+ const status = result.status === "completed" ? ok ? green("✓ pass") : red("✗ fail") : red(`✗ ${result.status}`);
8641
+ console.log(`${clock()} ${status} ${cellLabel(name, parameters)} ${dim(`${passedTrials}/${trials} trials · ${duration(ms)}`)}`);
8642
+ console.log(` ${dim(`report ${result.reportLocation}`)}`);
8643
+ },
8644
+ cell(cell, result, ms) {
8645
+ runLog.finished(cell.eval.name ?? authoringId(cell.eval), cell.parameters, result, ms);
8646
+ },
8647
+ failed(name, error) {
8648
+ console.log(`${clock()} ${red("✗ error")} ${bold(name)} ${error instanceof Error ? error.message : String(error)}`);
8649
+ }
8650
+ };
8651
+ /**
8652
+ * The runner logs structured Effect lines; the CLI prints its own summaries, so
8653
+ * only warnings show by default. `EVALKIT_LOG_LEVEL=info` (or `debug`) restores them.
8654
+ */
8655
+ function runnerLogs(effect) {
8656
+ const requested = process.env.EVALKIT_LOG_LEVEL?.toLowerCase();
8657
+ const level = LogLevel.allLevels.find((item) => item.label.toLowerCase() === requested) ?? LogLevel.Warning;
8658
+ return effect.pipe(Logger.withMinimumLogLevel(level));
8659
+ }
8660
+ /** True when a person can answer a prompt (not CI, pipes, or redirected input). */
8661
+ function canPrompt() {
8662
+ return Boolean(process.stdin.isTTY && process.stdout.isTTY);
8663
+ }
8664
+ async function confirm(question) {
8665
+ const { createInterface } = await import("node:readline/promises");
8666
+ const prompt = createInterface({
8667
+ input: process.stdin,
8668
+ output: process.stdout
8669
+ });
8670
+ try {
8671
+ const answer = await prompt.question(` ${cyan("?")} ${question} ${dim("(y/N)")} `);
8672
+ return /^y(es)?$/i.test(answer.trim());
8673
+ } catch {
8674
+ console.log("");
8675
+ return false;
8676
+ } finally {
8677
+ prompt.close();
8678
+ }
8679
+ }
8680
+ //#endregion
8379
8681
  //#region ../cli/src/run-command.ts
8380
8682
  function parseRunArgs(args) {
8381
8683
  const { values, positionals } = parseArgs({
@@ -8388,6 +8690,8 @@ function parseRunArgs(args) {
8388
8690
  type: "string",
8389
8691
  multiple: true
8390
8692
  },
8693
+ suite: { type: "string" },
8694
+ matrix: { type: "string" },
8391
8695
  model: {
8392
8696
  type: "string",
8393
8697
  multiple: true
@@ -8454,6 +8758,8 @@ function normalizeRunOptions(values, positionals) {
8454
8758
  if (index < 1) throw new Error("--param requires name=JSON");
8455
8759
  parameters[value.slice(0, index)] = JSON.parse(value.slice(index + 1));
8456
8760
  }
8761
+ if (values.suite && values.matrix) throw new Error("--suite and --matrix cannot be used together");
8762
+ if ((values.suite || values.matrix) && positionals.length) throw new Error("Use --eval to select evals with --suite or --matrix");
8457
8763
  return {
8458
8764
  values,
8459
8765
  positionals,
@@ -8463,7 +8769,7 @@ function normalizeRunOptions(values, positionals) {
8463
8769
  trials: integer("trials")
8464
8770
  };
8465
8771
  }
8466
- async function runProjectCommand(command, args, cwd = process.cwd(), parsed = parseRunArgs(args)) {
8772
+ async function runProjectCommand(args, cwd = process.cwd(), parsed = parseRunArgs(args)) {
8467
8773
  const { values, positionals, parameters } = parsed;
8468
8774
  const project = await loadProject(cwd, values.config);
8469
8775
  process.chdir(project.root);
@@ -8471,14 +8777,9 @@ async function runProjectCommand(command, args, cwd = process.cwd(), parsed = pa
8471
8777
  const requestedEvals = values.eval?.flatMap((value) => value.split(",")) ?? [];
8472
8778
  let matrix;
8473
8779
  let suiteId;
8474
- if (command === "run-matrix") {
8475
- if (positionals.length !== 1) throw new Error("run-matrix requires one matrix ID");
8476
- matrix = registry.matrices.find((m) => m.id === positionals[0]);
8477
- if (!matrix) throw new Error(`Unknown matrix: ${positionals[0]}`);
8478
- } else if (command === "run-suite") {
8479
- if (positionals.length !== 1) throw new Error("run-suite requires one suite ID");
8480
- const suite = registry.suites.find((s) => s.id === positionals[0]);
8481
- if (!suite) throw new Error(`Unknown suite: ${positionals[0]}`);
8780
+ if (values.suite) {
8781
+ const suite = registry.suites.find((item) => item.id === values.suite);
8782
+ if (!suite) throw new Error(`Unknown suite: ${values.suite}`);
8482
8783
  suiteId = authoringId(suite);
8483
8784
  matrix = defineEvalMatrix({
8484
8785
  id: "default",
@@ -8488,6 +8789,9 @@ async function runProjectCommand(command, args, cwd = process.cwd(), parsed = pa
8488
8789
  cases: project.config.matrix?.cases,
8489
8790
  exclude: project.config.matrix?.exclude
8490
8791
  });
8792
+ } else if (values.matrix) {
8793
+ matrix = registry.matrices.find((item) => item.id === values.matrix);
8794
+ if (!matrix) throw new Error(`Unknown matrix: ${values.matrix}`);
8491
8795
  } else {
8492
8796
  requestedEvals.push(...positionals.flatMap((value) => value.split(",")));
8493
8797
  matrix = project.config.matrix ? registry.matrices.at(-1) : defineEvalMatrix({
@@ -8502,7 +8806,7 @@ async function runProjectCommand(command, args, cwd = process.cwd(), parsed = pa
8502
8806
  ...requestedEvals.length ? { evals: requestedEvals } : {}
8503
8807
  };
8504
8808
  for (const [axis, choices] of Object.entries(parsed.selection)) if (axis in matrix.parameters || matrix.cases?.some((assignment) => Object.hasOwn(assignment, axis))) selection.parameters[axis] = choices;
8505
- else if ((axis === "model" || axis === "mode") && choices.length === 1 && !project.config.matrix && command !== "run-matrix") selection.overrides[axis] = choices[0];
8809
+ else if ((axis === "model" || axis === "mode") && choices.length === 1 && !project.config.matrix && !values.matrix) selection.overrides[axis] = choices[0];
8506
8810
  else throw new Error(`Unknown matrix axis: ${axis}`);
8507
8811
  const cells = matrix.count(selection);
8508
8812
  if (!cells) throw new Error("Selection contains no eligible cells; check cases and exclusions");
@@ -8521,7 +8825,7 @@ async function runProjectCommand(command, args, cwd = process.cwd(), parsed = pa
8521
8825
  }
8522
8826
  if (cells > (project.config.execution?.maxCells ?? 100) && !values.all) throw new Error(`Selected ${cells} cells. Narrow the selection, inspect --dry-run, or pass --all.`);
8523
8827
  if (!values.json) console.log(`Running ${cells} cell(s) from ${plan.matrix}`);
8524
- if ((await Effect.runPromise(runMatrix(matrix, {
8828
+ if ((await Effect.runPromise(runnerLogs(runMatrix(matrix, {
8525
8829
  selection,
8526
8830
  concurrency: parsed.concurrency ?? project.config.execution?.concurrency ?? 4,
8527
8831
  trials,
@@ -8547,7 +8851,56 @@ async function runProjectCommand(command, args, cwd = process.cwd(), parsed = pa
8547
8851
  }
8548
8852
  }
8549
8853
  }
8550
- }))).failed) process.exitCode = 1;
8854
+ })))).failed) process.exitCode = 1;
8855
+ }
8856
+ //#endregion
8857
+ //#region ../cli/src/tidy.ts
8858
+ /**
8859
+ * Every eval the project can run: standalone, in suites, or only in matrices.
8860
+ * A run is archived only when its eval is in none of these.
8861
+ */
8862
+ function projectEvalIds(registry) {
8863
+ return new Set([
8864
+ ...registry.evals,
8865
+ ...registry.suites.flatMap((suite) => suite.evals),
8866
+ ...registry.matrices.flatMap((matrix) => matrix.evals)
8867
+ ].map(authoringId));
8868
+ }
8869
+ /**
8870
+ * Mark runs of removed evals as archived, and restore runs whose eval returned.
8871
+ * Reports are never deleted: archiving writes a marker file beside the manifest.
8872
+ */
8873
+ async function tidyReports(options) {
8874
+ const known = projectEvalIds(options.registry);
8875
+ const reports = createDashboardReportReader(options.reportRoot, options.registry);
8876
+ const result = {
8877
+ archived: [],
8878
+ restored: [],
8879
+ stillArchived: 0,
8880
+ current: 0
8881
+ };
8882
+ for (const run of await reports.listRuns()) {
8883
+ const marker = resolve(options.reportRoot, run.id, ARCHIVE_MARKER);
8884
+ const entry = {
8885
+ runId: run.id,
8886
+ evalId: run.evalId
8887
+ };
8888
+ if (known.has(run.evalId)) {
8889
+ result.current += 1;
8890
+ if (run.archived) {
8891
+ await rm(marker, { force: true });
8892
+ result.restored.push(entry);
8893
+ }
8894
+ } else if (run.archived) result.stillArchived += 1;
8895
+ else {
8896
+ await writeFile(marker, `${JSON.stringify({
8897
+ archivedAt: (/* @__PURE__ */ new Date()).toISOString(),
8898
+ reason: `Eval "${run.evalId}" is no longer in the project`
8899
+ })}\n`);
8900
+ result.archived.push(entry);
8901
+ }
8902
+ }
8903
+ return result;
8551
8904
  }
8552
8905
  //#endregion
8553
8906
  //#region ../cli/src/new-project-template.ts
@@ -8653,21 +9006,23 @@ bun run check # type-check your eval, agent, scorer, and config
8653
9006
  bun run evals # run the example across both styles
8654
9007
  bun run matrix:plan # inspect the matrix without running it
8655
9008
  bun run matrix # run the configured matrix
8656
- bun run dashboard # inspect local reports
9009
+ bun run dashboard # inspect local reports and launch runs
9010
+ bun run dashboard:build # write a read-only, shareable snapshot
8657
9011
  \`\`\`
8658
9012
 
8659
- EvalKit installs from the public GitHub repository; no registry token is needed.
8660
- Reports and trial workspaces stay local and are ignored by Git.
8661
- Commit \`bun.lock\` to pin the resolved EvalKit Git revision.
9013
+ EvalKit installs from npm. Reports, trial workspaces, and dashboard snapshots stay local
9014
+ and are ignored by Git. A snapshot in \`_evalkit-dashboard/\` contains report messages,
9015
+ scores, evidence, and candidate files (never evaluator workspaces): review it before sharing.
8662
9016
  `;
8663
9017
  }
8664
9018
  //#endregion
8665
9019
  //#region ../cli/src/new-project.ts
8666
- const npmDependency = "^0.1.0";
9020
+ const npmDependency = "^0.3.0";
8667
9021
  const ignored = [
8668
9022
  "node_modules/",
8669
9023
  "_evalkit-results/",
8670
- "_evalkit-sandbox/"
9024
+ "_evalkit-sandbox/",
9025
+ "_evalkit-dashboard/"
8671
9026
  ];
8672
9027
  const readIfExists = async (path) => {
8673
9028
  try {
@@ -8746,10 +9101,11 @@ async function preparePackage(packagePath, name) {
8746
9101
  const devDependencies = pkg.devDependencies ?? {};
8747
9102
  if (!scripts || typeof scripts !== "object" || Array.isArray(scripts) || !dependencies || typeof dependencies !== "object" || Array.isArray(dependencies) || !devDependencies || typeof devDependencies !== "object" || Array.isArray(devDependencies)) throw new Error(`Invalid package.json scripts/dependencies: ${packagePath}`);
8748
9103
  scripts.check ??= "tsc --noEmit";
8749
- scripts.evals ??= "evalkit run-evals greeting --local";
8750
- scripts.matrix ??= "evalkit run-matrix styles --local";
8751
- scripts["matrix:plan"] ??= "evalkit run-matrix styles --dry-run";
8752
- scripts.dashboard ??= "evalkit serve-dashboard";
9104
+ scripts.evals ??= "evalkit run greeting --local";
9105
+ scripts.matrix ??= "evalkit run --matrix styles --local";
9106
+ scripts["matrix:plan"] ??= "evalkit run --matrix styles --dry-run";
9107
+ scripts.dashboard ??= "evalkit dev";
9108
+ scripts["dashboard:build"] ??= "evalkit build";
8753
9109
  if (!dependencies["@leostera/evalkit"] && !devDependencies["@leostera/evalkit"]) dependencies["@leostera/evalkit"] = npmDependency;
8754
9110
  if (!dependencies.typescript && !devDependencies.typescript) devDependencies.typescript = "^6.0.0";
8755
9111
  if (!dependencies["@types/bun"] && !devDependencies["@types/bun"]) devDependencies["@types/bun"] = "^1.4.0";
@@ -8778,6 +9134,7 @@ async function writeStarterReadme(root, name, inPlace) {
8778
9134
  }
8779
9135
  //#endregion
8780
9136
  //#region ../cli/src/index.ts
9137
+ const VERSION = "0.3.0";
8781
9138
  let projectRoot = process.cwd();
8782
9139
  let reportRoot = resolve(projectRoot, "_evalkit-results");
8783
9140
  const sandboxRoot = resolve(projectRoot, "_evalkit-sandbox");
@@ -8797,13 +9154,17 @@ async function runEvals(options) {
8797
9154
  const registry = await loadRegistry();
8798
9155
  const evaluations = options.evalIds.map((id) => registry.get(id));
8799
9156
  if (evaluations.some((e) => !e)) throw new Error("Unknown eval");
8800
- await Effect.runPromise(Effect.all(evaluations.map((evaluation) => runEval(evaluation, {
8801
- report: localReportStore(reportRoot),
8802
- workspaceRoot: sandboxRoot,
8803
- suiteId: options.suiteId,
8804
- runtime: runtimeFor(evaluation),
8805
- concurrency: options.concurrency
8806
- })), { concurrency: options.concurrency }));
9157
+ await Effect.runPromise(runnerLogs(Effect.all(evaluations.map((evaluation) => {
9158
+ const name = evaluation.name ?? authoringId(evaluation);
9159
+ const started = performance.now();
9160
+ return Effect.sync(() => runLog.started(name, {}, options.suiteId)).pipe(Effect.andThen(runEval(evaluation, {
9161
+ report: localReportStore(reportRoot),
9162
+ workspaceRoot: sandboxRoot,
9163
+ suiteId: options.suiteId,
9164
+ runtime: runtimeFor(evaluation),
9165
+ concurrency: options.concurrency
9166
+ })), Effect.tap((result) => runLog.finished(name, {}, result, performance.now() - started)), Effect.tapError((error) => Effect.sync(() => runLog.failed(name, error))));
9167
+ }), { concurrency: options.concurrency })));
8807
9168
  }
8808
9169
  function registerMatrixRoutes(app, registry, dashboardMatrix, trials) {
8809
9170
  let matrixRunning = false;
@@ -8846,14 +9207,18 @@ function registerMatrixRoutes(app, registry, dashboardMatrix, trials) {
8846
9207
  }
8847
9208
  if (matrixRunning) return context.text("A dashboard matrix cell is already running", 409);
8848
9209
  matrixRunning = true;
8849
- Effect.runPromise(runMatrix(dashboardMatrix, {
9210
+ const name = evaluation.name ?? authoringId(evaluation);
9211
+ const started = performance.now();
9212
+ runLog.started(name, body.parameters ?? {}, suiteId);
9213
+ Effect.runPromise(runnerLogs(runMatrix(dashboardMatrix, {
8850
9214
  selection,
8851
9215
  concurrency: 1,
8852
9216
  trials,
8853
9217
  ...suiteId ? { suiteId } : {},
8854
9218
  report: localReportStore(reportRoot),
8855
- workspaceRoot: sandboxRoot
8856
- })).catch((error) => console.error("Dashboard matrix run failed:", error)).finally(() => {
9219
+ workspaceRoot: sandboxRoot,
9220
+ onResult: (cell, result) => runLog.cell(cell, result, performance.now() - started)
9221
+ }))).catch((error) => runLog.failed(name, error)).finally(() => {
8857
9222
  matrixRunning = false;
8858
9223
  });
8859
9224
  } else {
@@ -8862,7 +9227,7 @@ function registerMatrixRoutes(app, registry, dashboardMatrix, trials) {
8862
9227
  evalIds: [authoringId(evaluation)],
8863
9228
  suiteId,
8864
9229
  concurrency: 32
8865
- }).catch((error) => console.error("Dashboard run failed:", error));
9230
+ }).catch(() => {});
8866
9231
  }
8867
9232
  return context.json({ accepted: true }, 202);
8868
9233
  });
@@ -8874,7 +9239,7 @@ function registerMatrixRoutes(app, registry, dashboardMatrix, trials) {
8874
9239
  evalIds: suite.evals.map(authoringId),
8875
9240
  suiteId: authoringId(suite),
8876
9241
  concurrency: 32
8877
- }).catch((error) => console.error("Dashboard suite run failed:", error));
9242
+ }).catch(() => {});
8878
9243
  return context.json({ accepted: true }, 202);
8879
9244
  });
8880
9245
  }
@@ -8942,19 +9307,48 @@ function registerDashboardAssets(app, dashboardRoot) {
8942
9307
  return new Response(content, { headers: { "content-type": fileContentType(servedFile) } });
8943
9308
  });
8944
9309
  }
9310
+ /** The npm package bundles the UI beside cli.mjs; the monorepo uses its Vite build. */
9311
+ function dashboardAssets() {
9312
+ const bundledDashboard = new URL("./dashboard/", import.meta.url);
9313
+ const root = fileURLToPath(existsSync(bundledDashboard) ? bundledDashboard : new URL("../../dashboard/dist/", import.meta.url));
9314
+ if (!existsSync(resolve(root, "index.html"))) throw new Error(`Dashboard assets not found at ${root}; run \`bun run build\` in the EvalKit repository`);
9315
+ return root;
9316
+ }
9317
+ async function useProject(configPath) {
9318
+ project = await loadProject(projectRoot, configPath);
9319
+ projectRoot = project.root;
9320
+ reportRoot = resolve(projectRoot, project.config.reportDir ?? "_evalkit-results");
9321
+ }
8945
9322
  async function serveDashboard() {
8946
9323
  const registry = await loadRegistry();
8947
- const bundledDashboard = new URL("./dashboard/", import.meta.url);
8948
- const dashboardRoot = fileURLToPath(existsSync(bundledDashboard) ? bundledDashboard : new URL("../../dashboard/dist/", import.meta.url));
9324
+ const dashboardRoot = dashboardAssets();
8949
9325
  const app = new Hono();
8950
- registerMatrixRoutes(app, registry, project?.config.matrix ? registry.matrices.at(-1) : void 0, project?.config.execution?.trials);
9326
+ const dashboardMatrix = project?.config.matrix ? registry.matrices.at(-1) : void 0;
9327
+ registerMatrixRoutes(app, registry, dashboardMatrix, project?.config.execution?.trials);
8951
9328
  registerReportRoutes(app, registry, createDashboardReportReader(reportRoot, registry));
8952
9329
  registerDashboardAssets(app, dashboardRoot);
8953
- const url = `http://localhost:${Bun.serve({
8954
- port: Number(process.env.PORT ?? 4317),
8955
- fetch: app.fetch
8956
- }).port}`;
8957
- console.log(`EvalKit dashboard: ${url}`);
9330
+ const port = Number(process.env.PORT ?? 4317);
9331
+ let server;
9332
+ try {
9333
+ server = Bun.serve({
9334
+ port,
9335
+ fetch: app.fetch
9336
+ });
9337
+ } catch (error) {
9338
+ if (error.code === "EADDRINUSE") throw new Error(`Port ${port} is already in use. Stop the other server or choose another port: PORT=4318 evalkit dev`);
9339
+ throw error;
9340
+ }
9341
+ const url = `http://localhost:${server.port}`;
9342
+ const reader = createDashboardReportReader(reportRoot, registry);
9343
+ const runs = (await reader.listRuns()).length;
9344
+ const legacy = reader.legacyRunIds().length;
9345
+ const matrixCells = dashboardMatrix?.count({ parameters: {} });
9346
+ const row = (label, value) => console.log(` ${cyan("➜")} ${bold(label.padEnd(9))}${value}`);
9347
+ banner("dev", VERSION);
9348
+ row("Local:", cyan(url));
9349
+ row("Project:", `${plural(registry.evals.length, "eval")} · ${plural(registry.suites.length, "suite")}` + (dashboardMatrix ? ` · matrix ${authoringId(dashboardMatrix)} (${plural(matrixCells, "cell")})` : ""));
9350
+ row("Reports:", `${relative(process.cwd(), reportRoot) || "."} ${dim(`(${plural(runs, "run")}${legacy ? `, ${legacy} from an older EvalKit hidden` : ""})`)}`);
9351
+ console.log(`\n ${dim("Runs started from the dashboard are logged below. Press Ctrl+C to stop.")}\n`);
8958
9352
  if (process.env.EVALKIT_NO_OPEN !== "1") {
8959
9353
  const opener = process.platform === "darwin" ? "open" : process.platform === "win32" ? "start" : "xdg-open";
8960
9354
  try {
@@ -8969,6 +9363,8 @@ const config = text("config").pipe(optional);
8969
9363
  const runOptions = {
8970
9364
  config,
8971
9365
  eval: text("eval").pipe(repeated),
9366
+ suite: text("suite").pipe(optional),
9367
+ matrix: text("matrix").pipe(optional),
8972
9368
  model: text("model").pipe(repeated),
8973
9369
  mode: text("mode").pipe(repeated),
8974
9370
  select: text("select").pipe(repeated),
@@ -8989,6 +9385,8 @@ function runInput(options, positionals) {
8989
9385
  return normalizeRunOptions({
8990
9386
  config: Option.getOrUndefined(options.config),
8991
9387
  eval: options.eval,
9388
+ suite: Option.getOrUndefined(options.suite),
9389
+ matrix: Option.getOrUndefined(options.matrix),
8992
9390
  model: options.model,
8993
9391
  mode: options.mode,
8994
9392
  select: options.select,
@@ -9004,22 +9402,14 @@ function runInput(options, positionals) {
9004
9402
  local: options.local
9005
9403
  }, positionals);
9006
9404
  }
9007
- const execute = (command, input) => Effect.tryPromise({
9008
- try: () => runProjectCommand(command, [], process.cwd(), input),
9405
+ const execute = (input) => Effect.tryPromise({
9406
+ try: () => runProjectCommand([], process.cwd(), input),
9009
9407
  catch: (error) => error
9010
9408
  });
9011
- const evalsCommand = make$1("run-evals", {
9409
+ const runCommand = make$1("run", {
9012
9410
  ...runOptions,
9013
9411
  evalIds: text$2({ name: "eval-id" }).pipe(repeated$2)
9014
- }, ({ evalIds, ...options }) => execute("run-evals", runInput(options, evalIds)));
9015
- const matrixCommand = make$1("run-matrix", {
9016
- ...runOptions,
9017
- matrixId: text$2({ name: "matrix-id" })
9018
- }, ({ matrixId, ...options }) => execute("run-matrix", runInput(options, [matrixId])));
9019
- const suiteCommand = make$1("run-suite", {
9020
- ...runOptions,
9021
- suiteId: text$2({ name: "suite-id" })
9022
- }, ({ suiteId, ...options }) => execute("run-suite", runInput(options, [suiteId])));
9412
+ }, ({ evalIds, ...options }) => execute(runInput(options, evalIds)));
9023
9413
  const newCommand = make$1("new", { directory: text$2({ name: "directory" }) }, ({ directory }) => Effect.tryPromise({
9024
9414
  try: async () => {
9025
9415
  const root = await newProject([directory]);
@@ -9027,24 +9417,103 @@ const newCommand = make$1("new", { directory: text$2({ name: "directory" }) }, (
9027
9417
  },
9028
9418
  catch: (error) => error
9029
9419
  }));
9030
- const dashboardCommand = make$1("serve-dashboard", { config }, ({ config }) => Effect.tryPromise({
9420
+ const devCommand = make$1("dev", { config }, ({ config }) => Effect.tryPromise({
9031
9421
  try: async () => {
9032
- project = await loadProject(projectRoot, Option.getOrUndefined(config));
9033
- projectRoot = project.root;
9034
- reportRoot = resolve(projectRoot, project.config.reportDir ?? "_evalkit-results");
9422
+ await useProject(Option.getOrUndefined(config));
9035
9423
  await serveDashboard();
9036
9424
  },
9037
9425
  catch: (error) => error
9038
9426
  }));
9427
+ const buildCommand = make$1("build", {
9428
+ config,
9429
+ out: text("out").pipe(optional),
9430
+ run: boolean("run"),
9431
+ noRun: boolean("no-run")
9432
+ }, ({ config, out, run, noRun }) => Effect.tryPromise({
9433
+ try: async () => {
9434
+ if (run && noRun) throw new Error("Choose either --run or --no-run");
9435
+ const started = performance.now();
9436
+ const configPath = Option.getOrUndefined(config);
9437
+ banner("build", VERSION);
9438
+ await loadProjectStep(configPath);
9439
+ if (await shouldRunEvals({
9440
+ run,
9441
+ noRun
9442
+ })) await runAllEvals(configPath);
9443
+ const result = await buildDashboard({
9444
+ registry: project.registry,
9445
+ reportRoot,
9446
+ dashboardRoot: dashboardAssets(),
9447
+ outDir: resolve(projectRoot, Option.getOrElse(out, () => "_evalkit-dashboard"))
9448
+ }, step);
9449
+ const size = await directorySize(result.outDir);
9450
+ const shown = relative(process.cwd(), result.outDir) || ".";
9451
+ console.log(`\n ${bold(`${shown}/`)} ${dim(`${bytes(size.bytes)} · ${plural(size.files, "file")}`)}`);
9452
+ console.log(`\n ${bold("Built read-only dashboard")} in ${duration(performance.now() - started)}\n`);
9453
+ console.log(` ${cyan("➜")} ${bold("Preview:")} bunx serve ${shown}`);
9454
+ console.log(` ${yellow("⚠")} Contains transcripts, scores, evidence, and candidate files\n (never evaluator workspaces). Review before sharing.\n`);
9455
+ },
9456
+ catch: (error) => error
9457
+ }));
9458
+ async function loadProjectStep(configPath) {
9459
+ await step("Eval project", () => useProject(configPath), () => {
9460
+ const where = relative(process.cwd(), projectRoot);
9461
+ return `${plural(project.registry.evals.length, "eval")} loaded${where ? ` from ${where}` : ""}`;
9462
+ });
9463
+ }
9464
+ /**
9465
+ * A snapshot needs data: with no current results, run the evals. Otherwise ask,
9466
+ * or keep the existing results when no one can answer (CI, pipes).
9467
+ */
9468
+ async function shouldRunEvals(flags) {
9469
+ if (flags.run) return true;
9470
+ if (flags.noRun) return false;
9471
+ const runs = (await createDashboardReportReader(reportRoot, project.registry).listRuns()).filter((entry) => !entry.archived).length;
9472
+ if (!runs) {
9473
+ console.log(` ${cyan("◇")} No results yet, so running the evals to have data to build.`);
9474
+ return true;
9475
+ }
9476
+ if (!canPrompt()) {
9477
+ console.log(` ${dim(`Using ${plural(runs, "existing run")}; pass --run to run the evals first.`)}`);
9478
+ return false;
9479
+ }
9480
+ return confirm(`Run the evals before building? ${dim(`(${plural(runs, "existing run")})`)}`);
9481
+ }
9482
+ async function runAllEvals(configPath) {
9483
+ const started = performance.now();
9484
+ console.log(`\n ${cyan("▶")} ${bold("Running evals")}\n`);
9485
+ await runProjectCommand([], process.cwd(), parseRunArgs(configPath ? ["--config", configPath] : []));
9486
+ const failed = process.exitCode === 1;
9487
+ console.log(`\n ${failed ? yellow("⚠") : cyan("✓")} ${bold("Evals finished")} in ${duration(performance.now() - started)}` + (failed ? dim(" (some failed; building with their results)") : "") + "\n");
9488
+ }
9489
+ const tidyCommand = make$1("tidy", { config }, ({ config }) => Effect.tryPromise({
9490
+ try: async () => {
9491
+ banner("tidy", VERSION);
9492
+ await loadProjectStep(Option.getOrUndefined(config));
9493
+ const result = await step("Run reports", () => tidyReports({
9494
+ registry: project.registry,
9495
+ reportRoot
9496
+ }), (value) => `${plural(value.current + value.archived.length + value.stillArchived, "run")} checked`);
9497
+ const byEval = (entries) => [...entries.reduce((groups, entry) => groups.set(entry.evalId, (groups.get(entry.evalId) ?? 0) + 1), /* @__PURE__ */ new Map()).entries()].map(([id, total]) => ` ${bold(id)} ${dim(`(${plural(total, "run")})`)}`).join("\n");
9498
+ console.log("");
9499
+ if (result.archived.length) console.log(` ${yellow("◆")} Archived ${plural(result.archived.length, "run")} of evals no longer in the project:\n${byEval(result.archived)}`);
9500
+ if (result.restored.length) console.log(` ${cyan("◆")} Restored ${plural(result.restored.length, "run")} whose eval is back:\n${byEval(result.restored)}`);
9501
+ if (!result.archived.length && !result.restored.length) console.log(` ${cyan("✓")} Nothing new to tidy.`);
9502
+ const totalArchived = result.archived.length + result.stillArchived;
9503
+ if (totalArchived) console.log(`\n ${dim(`Archived runs (${totalArchived}) are hidden in \`evalkit dev\` (filter "Show archived")`)}\n ${dim("and left out of `evalkit build`. Nothing was deleted.")}`);
9504
+ console.log("");
9505
+ },
9506
+ catch: (error) => error
9507
+ }));
9039
9508
  const cli = run(make$1("evalkit").pipe(withSubcommands([
9040
9509
  newCommand,
9041
- evalsCommand,
9042
- matrixCommand,
9043
- suiteCommand,
9044
- dashboardCommand
9510
+ runCommand,
9511
+ devCommand,
9512
+ buildCommand,
9513
+ tidyCommand
9045
9514
  ])), {
9046
9515
  name: "EvalKit",
9047
- version: "0.1.0"
9516
+ version: VERSION
9048
9517
  });
9049
9518
  const argv = process.argv.slice();
9050
9519
  if (argv[2] === "help") argv.splice(2, 1, "--help");