@leostera/evalkit 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli.mjs CHANGED
@@ -2,7 +2,7 @@
2
2
  import { F as defineEvalMatrix, I as canonicalParameters, m as registerEvals, n as authoringId } from "./src-DrwwgaPn.mjs";
3
3
  import { a as readTrialEvents, c as readTrialSummary, i as readRunSummary, l as localReportStore, n as runMatrix, o as readTrialManifest, r as readRunManifest, t as runEval } from "./src-DluldGKn.mjs";
4
4
  import * as Schema from "effect/Schema";
5
- import { Effect, Option } from "effect";
5
+ import { Effect, LogLevel, Logger, Option } from "effect";
6
6
  import { cp, mkdir, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
7
7
  import * as Path from "node:path";
8
8
  import { basename, dirname, extname, relative, resolve, sep } from "node:path";
@@ -43,11 +43,11 @@ import * as EffectSecret from "effect/Secret";
43
43
  import * as Effectable from "effect/Effectable";
44
44
  import * as Match from "effect/Match";
45
45
  import * as EffectNumber from "effect/Number";
46
- import * as LogLevel from "effect/LogLevel";
46
+ import * as LogLevel$1 from "effect/LogLevel";
47
47
  import * as Config from "effect/Config";
48
48
  import * as HashMap from "effect/HashMap";
49
49
  import * as Order from "effect/Order";
50
- import * as Logger from "effect/Logger";
50
+ import * as Logger$1 from "effect/Logger";
51
51
  import * as Unify from "effect/Unify";
52
52
  import * as HashSet from "effect/HashSet";
53
53
  import * as SynchronizedRef from "effect/SynchronizedRef";
@@ -392,13 +392,13 @@ const struct = (fields) => {
392
392
  /** @internal */
393
393
  const black = { _tag: "Black" };
394
394
  /** @internal */
395
- const red$3 = { _tag: "Red" };
395
+ const red$4 = { _tag: "Red" };
396
396
  /** @internal */
397
- const green$2 = { _tag: "Green" };
397
+ const green$3 = { _tag: "Green" };
398
398
  /** @internal */
399
399
  const magenta$1 = { _tag: "Magenta" };
400
400
  /** @internal */
401
- const cyan$1 = { _tag: "Cyan" };
401
+ const cyan$2 = { _tag: "Cyan" };
402
402
  /** @internal */
403
403
  const white$3 = { _tag: "White" };
404
404
  /** @internal */
@@ -500,7 +500,7 @@ const ESC = "\x1B[";
500
500
  const BEL = "\x07";
501
501
  const SEP = ";";
502
502
  /** @internal */
503
- const bold$1 = /*#__PURE__*/ make$5({ bold: /*#__PURE__*/ Option$1.some(/*#__PURE__*/ setBold(true)) });
503
+ const bold$2 = /*#__PURE__*/ make$5({ bold: /*#__PURE__*/ Option$1.some(/*#__PURE__*/ setBold(true)) });
504
504
  /** @internal */
505
505
  const italicized$1 = /*#__PURE__*/ make$5({ italicized: /*#__PURE__*/ Option$1.some(/*#__PURE__*/ setItalicized(true)) });
506
506
  /** @internal */
@@ -512,15 +512,15 @@ const brightColor = (color) => make$5({ foreground: Option$1.some(setColor(color
512
512
  /** @internal */
513
513
  const color$1 = (color) => make$5({ foreground: Option$1.some(setColor(color, false, "foreground")) });
514
514
  /** @internal */
515
- const red$2 = /*#__PURE__*/ color$1(red$3);
515
+ const red$3 = /*#__PURE__*/ color$1(red$4);
516
516
  /** @internal */
517
- const green$1 = /*#__PURE__*/ color$1(green$2);
517
+ const green$2 = /*#__PURE__*/ color$1(green$3);
518
518
  /** @internal */
519
519
  const white$2 = /*#__PURE__*/ color$1(white$3);
520
520
  /** @internal */
521
521
  const blackBright$1 = /*#__PURE__*/ brightColor(black);
522
522
  /** @internal */
523
- const cyanBright$1 = /*#__PURE__*/ brightColor(cyan$1);
523
+ const cyanBright$1 = /*#__PURE__*/ brightColor(cyan$2);
524
524
  /** @internal */
525
525
  const beep$2 = /*#__PURE__*/ make$5({ commands: /*#__PURE__*/ Arr.of(BEL) });
526
526
  /** @internal */
@@ -587,7 +587,7 @@ const stringifyInternal = (self) => {
587
587
  * @since 1.0.0
588
588
  * @category constructors
589
589
  */
590
- const bold = bold$1;
590
+ const bold$1 = bold$2;
591
591
  /**
592
592
  * @since 1.0.0
593
593
  * @category constructors
@@ -612,12 +612,12 @@ const color = color$1;
612
612
  * @since 1.0.0
613
613
  * @category colors
614
614
  */
615
- const red$1 = red$2;
615
+ const red$2 = red$3;
616
616
  /**
617
617
  * @since 1.0.0
618
618
  * @category colors
619
619
  */
620
- const green = green$1;
620
+ const green$1 = green$2;
621
621
  /**
622
622
  * @since 1.0.0
623
623
  * @category colors
@@ -2218,7 +2218,7 @@ const optimize = optimize$1;
2218
2218
  * @since 1.0.0
2219
2219
  * @category constructors
2220
2220
  */
2221
- const red = red$3;
2221
+ const red$1 = red$4;
2222
2222
  /**
2223
2223
  * @since 1.0.0
2224
2224
  * @category constructors
@@ -2228,7 +2228,7 @@ const magenta = magenta$1;
2228
2228
  * @since 1.0.0
2229
2229
  * @category constructors
2230
2230
  */
2231
- const cyan = cyan$1;
2231
+ const cyan$1 = cyan$2;
2232
2232
  /**
2233
2233
  * @since 1.0.0
2234
2234
  * @category constructors
@@ -2248,7 +2248,7 @@ const space = /*#__PURE__*/ text$6(" ");
2248
2248
  /** @internal */
2249
2249
  const code = (value) => highlight(value, white);
2250
2250
  /** @internal */
2251
- const error = (value) => highlight(value, red);
2251
+ const error = (value) => highlight(value, red$1);
2252
2252
  /** @internal */
2253
2253
  const highlight = (value, color) => ({
2254
2254
  _tag: "Highlight",
@@ -2307,7 +2307,7 @@ const toAnsiDoc$1 = (self) => {
2307
2307
  switch (self._tag) {
2308
2308
  case "Highlight": return annotate(toAnsiDoc$1(self.value), color(self.color));
2309
2309
  case "Sequence": return cat(toAnsiDoc$1(self.left), toAnsiDoc$1(self.right));
2310
- case "Strong": return annotate(toAnsiDoc$1(self.value), bold);
2310
+ case "Strong": return annotate(toAnsiDoc$1(self.value), bold$1);
2311
2311
  case "Text": return text$7(self.value);
2312
2312
  case "URI": return annotate(text$7(self.value), underlined);
2313
2313
  case "Weak": return annotate(toAnsiDoc$1(self.value), blackBright);
@@ -2373,11 +2373,11 @@ const toAnsiText = (self) => render$1(toAnsiDoc(self), { style: "pretty" });
2373
2373
  const toAnsiDocInternal = (self) => {
2374
2374
  switch (self._tag) {
2375
2375
  case "Empty": return empty$3;
2376
- case "Header": return pipe(annotate(toAnsiDoc$1(self.value), bold), cat(hardLine));
2376
+ case "Header": return pipe(annotate(toAnsiDoc$1(self.value), bold$1), cat(hardLine));
2377
2377
  case "Paragraph": return pipe(toAnsiDoc$1(self.value), cat(hardLine));
2378
2378
  case "DescriptionList": {
2379
2379
  const definitions = self.definitions.map(([span, doc]) => cats([
2380
- annotate(toAnsiDoc$1(span), bold),
2380
+ annotate(toAnsiDoc$1(span), bold$1),
2381
2381
  empty$3,
2382
2382
  indent(toAnsiDocInternal(doc), 2)
2383
2383
  ]));
@@ -2592,8 +2592,8 @@ function renderError$2(state, pointer) {
2592
2592
  onSome: (error) => {
2593
2593
  const errorLines = error.split(NEWLINE_REGEX$3);
2594
2594
  if (Arr.isNonEmptyReadonlyArray(errorLines)) {
2595
- const annotateLine = (line) => annotate(text$7(line), combine(italicized, red$1));
2596
- const prefix = cat(annotate(pointer, red$1), space$1);
2595
+ const annotateLine = (line) => annotate(text$7(line), combine(italicized, red$2));
2596
+ const prefix = cat(annotate(pointer, red$2), space$1);
2597
2597
  const lines = Arr.map(errorLines, (str) => annotateLine(str));
2598
2598
  return cursorSavePosition.pipe(cat(hardLine), cat(prefix), cat(align(vsep(lines))), cat(cursorRestorePosition));
2599
2599
  }
@@ -2612,7 +2612,7 @@ function renderParts(state, submitted = false) {
2612
2612
  });
2613
2613
  }
2614
2614
  function renderOutput$4(leadingSymbol, trailingSymbol, parts, options) {
2615
- const annotateLine = (line) => annotate(text$7(line), bold);
2615
+ const annotateLine = (line) => annotate(text$7(line), bold$1);
2616
2616
  const prefix = cat(leadingSymbol, space$1);
2617
2617
  return Arr.match(options.message.split(/\r?\n/), {
2618
2618
  onEmpty: () => hsep([
@@ -2642,7 +2642,7 @@ function renderSubmission$5(state, options) {
2642
2642
  return Effect$1.gen(function* () {
2643
2643
  const columns = yield* (yield* Terminal).columns;
2644
2644
  const figures$11 = yield* figures;
2645
- return renderOutput$4(annotate(figures$11.tick, green), annotate(figures$11.ellipsis, blackBright), renderParts(state, true), options).pipe(cat(hardLine), optimize(Deep), render$1({
2645
+ return renderOutput$4(annotate(figures$11.tick, green$1), annotate(figures$11.ellipsis, blackBright), renderParts(state, true), options).pipe(cat(hardLine), optimize(Deep), render$1({
2646
2646
  style: "pretty",
2647
2647
  options: { lineWidth: columns }
2648
2648
  }));
@@ -3106,7 +3106,7 @@ function handleClear$4(options) {
3106
3106
  }
3107
3107
  const NEWLINE_REGEX$2 = /\r?\n/;
3108
3108
  function renderPrompt(confirm, message, leadingSymbol, trailingSymbol) {
3109
- const annotateLine = (line) => annotate(text$7(line), bold);
3109
+ const annotateLine = (line) => annotate(text$7(line), bold$1);
3110
3110
  const prefix = cat(leadingSymbol, space$1);
3111
3111
  return Arr.match(message.split(NEWLINE_REGEX$2), {
3112
3112
  onEmpty: () => hsep([
@@ -3159,7 +3159,7 @@ function renderNextFrame$4(state, options) {
3159
3159
  options: { lineWidth: columns }
3160
3160
  }));
3161
3161
  }
3162
- const leadingSymbol = annotate(figures$10.tick, green);
3162
+ const leadingSymbol = annotate(figures$10.tick, green$1);
3163
3163
  const trailingSymbol = annotate(figures$10.ellipsis, blackBright);
3164
3164
  const promptMsg = renderPrompt(empty$3, options.message, leadingSymbol, trailingSymbol);
3165
3165
  const files = renderFiles(state, state.files, figures$10, options);
@@ -3173,7 +3173,7 @@ function renderSubmission$4(value, options) {
3173
3173
  return Effect$1.gen(function* () {
3174
3174
  const columns = yield* (yield* Terminal).columns;
3175
3175
  const figures$9 = yield* figures;
3176
- const leadingSymbol = annotate(figures$9.tick, green);
3176
+ const leadingSymbol = annotate(figures$9.tick, green$1);
3177
3177
  const trailingSymbol = annotate(figures$9.ellipsis, blackBright);
3178
3178
  return renderPrompt(empty$3, options.message, leadingSymbol, trailingSymbol).pipe(cat(space$1), cat(annotate(text$7(value), white$1)), cat(hardLine), render$1({
3179
3179
  style: "pretty",
@@ -3316,7 +3316,7 @@ function handleClear$3(options) {
3316
3316
  function renderInput$1(state, submitted) {
3317
3317
  const annotation = Option$1.match(state.error, {
3318
3318
  onNone: () => combine(underlined, cyanBright),
3319
- onSome: () => red$1
3319
+ onSome: () => red$2
3320
3320
  });
3321
3321
  const value = state.value === "" ? empty$3 : text$7(`${state.value}`);
3322
3322
  return submitted ? value : annotate(value, annotation);
@@ -3328,8 +3328,8 @@ function renderError$1(state, pointer) {
3328
3328
  onSome: (error) => Arr.match(error.split(NEWLINE_REGEX$1), {
3329
3329
  onEmpty: () => empty$3,
3330
3330
  onNonEmpty: (errorLines) => {
3331
- const annotateLine = (line) => annotate(text$7(line), combine(italicized, red$1));
3332
- const prefix = cat(annotate(pointer, red$1), space$1);
3331
+ const annotateLine = (line) => annotate(text$7(line), combine(italicized, red$2));
3332
+ const prefix = cat(annotate(pointer, red$2), space$1);
3333
3333
  const lines = Arr.map(errorLines, (str) => annotateLine(str));
3334
3334
  return cursorSavePosition.pipe(cat(hardLine), cat(prefix), cat(align(vsep(lines))), cat(cursorRestorePosition));
3335
3335
  }
@@ -3337,7 +3337,7 @@ function renderError$1(state, pointer) {
3337
3337
  });
3338
3338
  }
3339
3339
  function renderOutput$3(state, leadingSymbol, trailingSymbol, options, submitted = false) {
3340
- const annotateLine = (line) => annotate(text$7(line), bold);
3340
+ const annotateLine = (line) => annotate(text$7(line), bold$1);
3341
3341
  const prefix = cat(leadingSymbol, space$1);
3342
3342
  return Arr.match(options.message.split(/\r?\n/), {
3343
3343
  onEmpty: () => hsep([
@@ -3368,7 +3368,7 @@ function renderSubmission$3(nextState, options) {
3368
3368
  return Effect$1.gen(function* () {
3369
3369
  const columns = yield* (yield* Terminal).columns;
3370
3370
  const figures$7 = yield* figures;
3371
- return renderOutput$3(nextState, annotate(figures$7.tick, green), annotate(figures$7.ellipsis, blackBright), options, true).pipe(cat(hardLine), optimize(Deep), render$1({
3371
+ return renderOutput$3(nextState, annotate(figures$7.tick, green$1), annotate(figures$7.ellipsis, blackBright), options, true).pipe(cat(hardLine), optimize(Deep), render$1({
3372
3372
  style: "pretty",
3373
3373
  options: { lineWidth: columns }
3374
3374
  }));
@@ -3551,7 +3551,7 @@ const float = (options) => {
3551
3551
  const renderBeep$2 = /*#__PURE__*/ render$1(beep, { style: "pretty" });
3552
3552
  const NEWLINE_REGEX = /\r?\n/;
3553
3553
  function renderOutput$2(leadingSymbol, trailingSymbol, options) {
3554
- const annotateLine = (line) => annotate(text$7(line), bold);
3554
+ const annotateLine = (line) => annotate(text$7(line), bold$1);
3555
3555
  const prefix = cat(leadingSymbol, space$1);
3556
3556
  return Arr.match(options.message.split(NEWLINE_REGEX), {
3557
3557
  onEmpty: () => hsep([prefix, trailingSymbol]),
@@ -3566,7 +3566,7 @@ function renderChoicePrefix(state, choices, toDisplay, currentIndex, figures) {
3566
3566
  if (currentIndex === toDisplay.startIndex && toDisplay.startIndex > 0) prefix = figures.arrowUp;
3567
3567
  else if (currentIndex === toDisplay.endIndex - 1 && toDisplay.endIndex < choices.length) prefix = figures.arrowDown;
3568
3568
  if (choices[currentIndex].disabled) {
3569
- const annotation = combine(bold, blackBright);
3569
+ const annotation = combine(bold$1, blackBright);
3570
3570
  return state === currentIndex ? figures.pointer.pipe(annotate(annotation), cat(prefix)) : prefix.pipe(cat(space$1));
3571
3571
  }
3572
3572
  return state === currentIndex ? figures.pointer.pipe(annotate(cyanBright), cat(prefix)) : prefix.pipe(cat(space$1));
@@ -3611,7 +3611,7 @@ function renderSubmission$2(state, options) {
3611
3611
  const columns = yield* (yield* Terminal).columns;
3612
3612
  const figures$6 = yield* figures;
3613
3613
  const selected = text$7(options.choices[state].title);
3614
- return renderOutput$2(annotate(figures$6.tick, green), annotate(figures$6.ellipsis, blackBright), options).pipe(cat(space$1), cat(annotate(selected, white$1)), cat(hardLine), render$1({
3614
+ return renderOutput$2(annotate(figures$6.tick, green$1), annotate(figures$6.ellipsis, blackBright), options).pipe(cat(space$1), cat(annotate(selected, white$1)), cat(hardLine), render$1({
3615
3615
  style: "pretty",
3616
3616
  options: { lineWidth: columns }
3617
3617
  }));
@@ -3714,7 +3714,7 @@ function renderInput(nextState, options, submitted) {
3714
3714
  if (nextState.value.length === 0) return blackBright;
3715
3715
  return combine(underlined, cyanBright);
3716
3716
  },
3717
- onSome: () => red$1
3717
+ onSome: () => red$2
3718
3718
  });
3719
3719
  switch (options.type) {
3720
3720
  case "hidden": return empty$3;
@@ -3728,8 +3728,8 @@ function renderError(nextState, pointer) {
3728
3728
  onSome: (error) => Arr.match(error.split(/\r?\n/), {
3729
3729
  onEmpty: () => empty$3,
3730
3730
  onNonEmpty: (errorLines) => {
3731
- const annotateLine = (line) => text$7(line).pipe(annotate(combine(italicized, red$1)));
3732
- const prefix = cat(annotate(pointer, red$1), space$1);
3731
+ const annotateLine = (line) => text$7(line).pipe(annotate(combine(italicized, red$2)));
3732
+ const prefix = cat(annotate(pointer, red$2), space$1);
3733
3733
  const lines = Arr.map(errorLines, (str) => annotateLine(str));
3734
3734
  return cursorSavePosition.pipe(cat(hardLine), cat(prefix), cat(align(vsep(lines))), cat(cursorRestorePosition));
3735
3735
  }
@@ -3737,7 +3737,7 @@ function renderError(nextState, pointer) {
3737
3737
  });
3738
3738
  }
3739
3739
  function renderOutput$1(nextState, leadingSymbol, trailingSymbol, options, submitted = false) {
3740
- const annotateLine = (line) => annotate(text$7(line), bold);
3740
+ const annotateLine = (line) => annotate(text$7(line), bold$1);
3741
3741
  const promptLines = options.message.split(/\r?\n/);
3742
3742
  const prefix = cat(leadingSymbol, space$1);
3743
3743
  if (Arr.isNonEmptyReadonlyArray(promptLines)) {
@@ -3767,7 +3767,7 @@ function renderSubmission$1(state, options) {
3767
3767
  return Effect$1.gen(function* () {
3768
3768
  const columns = yield* (yield* Terminal).columns;
3769
3769
  const figures$4 = yield* figures;
3770
- return renderOutput$1(state, annotate(figures$4.tick, green), annotate(figures$4.ellipsis, blackBright), options, true).pipe(cat(hardLine), optimize(Deep), render$1({
3770
+ return renderOutput$1(state, annotate(figures$4.tick, green$1), annotate(figures$4.ellipsis, blackBright), options, true).pipe(cat(hardLine), optimize(Deep), render$1({
3771
3771
  style: "pretty",
3772
3772
  options: { lineWidth: columns }
3773
3773
  }));
@@ -3909,7 +3909,7 @@ function renderToggle(value, options, submitted = false) {
3909
3909
  ]);
3910
3910
  }
3911
3911
  function renderOutput(toggle, leadingSymbol, trailingSymbol, options) {
3912
- const annotateLine = (line) => annotate(text$7(line), bold);
3912
+ const annotateLine = (line) => annotate(text$7(line), bold$1);
3913
3913
  const promptLines = options.message.split(/\r?\n/);
3914
3914
  const prefix = cat(leadingSymbol, space$1);
3915
3915
  if (Arr.isNonEmptyReadonlyArray(promptLines)) {
@@ -3941,7 +3941,7 @@ function renderSubmission(value, options) {
3941
3941
  const terminal = yield* Terminal;
3942
3942
  const figures$2 = yield* figures;
3943
3943
  const columns = yield* terminal.columns;
3944
- const leadingSymbol = annotate(figures$2.tick, green);
3944
+ const leadingSymbol = annotate(figures$2.tick, green$1);
3945
3945
  const trailingSymbol = annotate(figures$2.ellipsis, blackBright);
3946
3946
  return renderOutput(renderToggle(value, options, true), leadingSymbol, trailingSymbol, options).pipe(cat(hardLine), optimize(Deep), render$1({
3947
3947
  style: "pretty",
@@ -5919,7 +5919,7 @@ const completionsOptions = /*#__PURE__*/ choiceWithValue("completions", [
5919
5919
  ["zsh", "zsh"]
5920
5920
  ]).pipe(optional$1, /*#__PURE__*/ withDescription("Generate a completion script for a specific shell."));
5921
5921
  /** @internal */
5922
- const logLevelOptions = /*#__PURE__*/ choiceWithValue("log-level", LogLevel.allLevels.map((level) => [level._tag.toLowerCase(), level])).pipe(optional$1, /*#__PURE__*/ withDescription("Sets the minimum log level for a command."));
5922
+ const logLevelOptions = /*#__PURE__*/ choiceWithValue("log-level", LogLevel$1.allLevels.map((level) => [level._tag.toLowerCase(), level])).pipe(optional$1, /*#__PURE__*/ withDescription("Sets the minimum log level for a command."));
5923
5923
  /** @internal */
5924
5924
  const helpOptions = /*#__PURE__*/ boolean$1("help").pipe(/*#__PURE__*/ withAlias("h"), /*#__PURE__*/ withDescription("Show the help documentation for a command."));
5925
5925
  /** @internal */
@@ -6294,7 +6294,7 @@ const wizardInternal = (self, prefix, config) => {
6294
6294
  case "GetUserInput":
6295
6295
  case "Standard": return Effect$1.gen(function* () {
6296
6296
  const logCurrentCommand = Ref.get(commandLineRef).pipe(Effect$1.flatMap((commandLine) => {
6297
- const currentCommand = p(pipe(strong(highlight("COMMAND:", cyan)), concat$1(space), concat$1(highlight(Arr.join(commandLine, " "), magenta))));
6297
+ const currentCommand = p(pipe(strong(highlight("COMMAND:", cyan$1)), concat$1(space), concat$1(highlight(Arr.join(commandLine, " "), magenta))));
6298
6298
  return Console.log(toAnsiText(currentCommand));
6299
6299
  }));
6300
6300
  if (isStandard(self)) {
@@ -6634,7 +6634,7 @@ const handleBuiltInOption = (self, executable, args, builtIn, execute, config, o
6634
6634
  filteredArgs.push(args[i]);
6635
6635
  }
6636
6636
  const nextArgs = Arr.appendAll(baseArgs, filteredArgs);
6637
- return run$2(self, nextArgs, execute).pipe(Logger.withMinimumLogLevel(builtIn.level));
6637
+ return run$2(self, nextArgs, execute).pipe(Logger$1.withMinimumLogLevel(builtIn.level));
6638
6638
  }
6639
6639
  case "ShowHelp": {
6640
6640
  const banner = h1(code(self.name));
@@ -6740,7 +6740,7 @@ const renderWizardArgs = (args) => {
6740
6740
  return blocks([
6741
6741
  p(strong(code("Wizard Mode Complete!"))),
6742
6742
  p(executeMsg),
6743
- p(concat$1(text$6(" "), highlight(params, cyan)))
6743
+ p(concat$1(text$6(" "), highlight(params, cyan$1)))
6744
6744
  ]);
6745
6745
  };
6746
6746
  const isLogLevelArg = (arg) => {
@@ -8050,6 +8050,8 @@ const layer = /*#__PURE__*/ pipe(/*#__PURE__*/ Layer.mergeAll(layer$3, layer$5,
8050
8050
  //#endregion
8051
8051
  //#region ../cli/src/dashboard-reports.ts
8052
8052
  const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
8053
+ /** Written into a run directory by `evalkit tidy`; deleting it restores the run. */
8054
+ const ARCHIVE_MARKER = "archived.json";
8053
8055
  function isMissing(error) {
8054
8056
  return error.code === "ENOENT";
8055
8057
  }
@@ -8086,15 +8088,35 @@ function fileContentType(file) {
8086
8088
  ".js": "text/javascript"
8087
8089
  }[extname(file)] ?? "application/octet-stream";
8088
8090
  }
8089
- /** Read saved local reports. A missing summary means "running"; malformed reports are errors. */
8091
+ /**
8092
+ * Read saved local reports. A missing summary means "running"; malformed reports are errors.
8093
+ * Runs written by an older EvalKit (manifest `schemaVersion` < 2) are skipped and counted.
8094
+ */
8090
8095
  function createDashboardReportReader(root, registry) {
8096
+ const legacyRuns = /* @__PURE__ */ new Set();
8091
8097
  async function listRuns() {
8092
8098
  const entries = await ifPresent(() => readdir(root, { withFileTypes: true }));
8093
8099
  if (!entries) return [];
8094
8100
  const ids = entries.filter((entry) => entry.isDirectory() && UUID_PATTERN.test(entry.name)).map((entry) => entry.name);
8095
8101
  return (await Promise.all(ids.map(readRun))).filter((run) => run !== void 0);
8096
8102
  }
8103
+ /** True for a run from a report format this version cannot read; never for corrupt JSON. */
8104
+ async function isLegacyRun(id) {
8105
+ const text = await ifPresent(() => readFile(resolve(root, id, "manifest.json"), "utf8"));
8106
+ if (text === void 0) return false;
8107
+ let version;
8108
+ try {
8109
+ version = JSON.parse(text).schemaVersion;
8110
+ } catch {
8111
+ return false;
8112
+ }
8113
+ return typeof version === "number" && version < 2;
8114
+ }
8097
8115
  async function readRun(id) {
8116
+ if (await isLegacyRun(id)) {
8117
+ legacyRuns.add(id);
8118
+ return;
8119
+ }
8098
8120
  const manifest = await ifPresent(() => readRunManifest(root, id));
8099
8121
  if (!manifest) return void 0;
8100
8122
  const summary = await ifPresent(() => readRunSummary(root, id));
@@ -8103,6 +8125,7 @@ function createDashboardReportReader(root, registry) {
8103
8125
  const scores = trials.flatMap((trial) => trial?.scoring?.overall === void 0 ? [] : [trial.scoring.overall]);
8104
8126
  return {
8105
8127
  id,
8128
+ ...await ifPresent(() => stat(resolve(root, id, "archived.json"))) ? { archived: true } : {},
8106
8129
  evalId: manifest.evalId,
8107
8130
  ...manifest.suiteId ? { suiteId: manifest.suiteId } : {},
8108
8131
  ...manifest.matrix ? {
@@ -8171,6 +8194,8 @@ function createDashboardReportReader(root, registry) {
8171
8194
  }
8172
8195
  return {
8173
8196
  listRuns,
8197
+ /** Run IDs skipped by the last `listRuns()` because an older EvalKit wrote them. */
8198
+ legacyRunIds: () => [...legacyRuns],
8174
8199
  listTrials,
8175
8200
  trialDetail,
8176
8201
  listArtifacts,
@@ -8205,66 +8230,154 @@ async function walkArtifacts(root) {
8205
8230
  //#region ../cli/src/dashboard-build.ts
8206
8231
  /** Marks a directory as EvalKit-owned so a rebuild may safely replace it. */
8207
8232
  const MARKER = ".evalkit-dashboard";
8208
- /**
8209
- * Write a read-only static dashboard: the bundled UI plus JSON snapshots of
8210
- * the catalog and local reports, and each trial's candidate artifact files.
8211
- * Evaluator workspaces are never copied; reports may still contain sensitive
8212
- * messages, scores, or evidence, so review the output before sharing it.
8213
- */
8214
- async function buildDashboard(options) {
8233
+ const quietStep = (_label, work) => work(() => {});
8234
+ async function buildDashboard(options, step = quietStep) {
8215
8235
  const { registry, reportRoot, dashboardRoot, outDir } = options;
8216
- await prepareOutDir(outDir);
8217
- await cp(dashboardRoot, outDir, { recursive: true });
8218
- await writeFile(resolve(outDir, MARKER), "Generated by `evalkit build`.\n");
8219
- await writeFile(resolve(outDir, "index.html"), staticIndex(await readFile(resolve(dashboardRoot, "index.html"), "utf8")));
8236
+ const template = await readFile(resolve(dashboardRoot, "index.html"), "utf8");
8220
8237
  const data = resolve(outDir, "data");
8221
8238
  const write = async (path, value) => {
8222
8239
  const file = resolve(data, path);
8223
8240
  await mkdir(dirname(file), { recursive: true });
8224
8241
  await writeFile(file, `${JSON.stringify(value)}\n`);
8225
8242
  };
8243
+ await step("Dashboard assets", async () => {
8244
+ await prepareOutDir(outDir);
8245
+ await cp(dashboardRoot, outDir, { recursive: true });
8246
+ await writeFile(resolve(outDir, MARKER), "Generated by `evalkit build`.\n");
8247
+ await writeFile(resolve(outDir, "index.html"), staticIndex(template));
8248
+ }, () => "read-only UI with relative assets");
8249
+ await step("Eval catalog", async () => {
8250
+ await write("catalog.json", { evals: registry.catalog() });
8251
+ await write("evals.json", { evals: registry.metadata() });
8252
+ await write("suites.json", { suites: registry.suiteMetadata() });
8253
+ return {
8254
+ evals: registry.evals.length,
8255
+ suites: registry.suites.length
8256
+ };
8257
+ }, ({ evals, suites }) => `${count(evals, "eval")} · ${count(suites, "suite")}`);
8226
8258
  const reports = createDashboardReportReader(reportRoot, registry);
8227
- const runs = await reports.listRuns();
8228
- await write("catalog.json", { evals: registry.catalog() });
8229
- await write("evals.json", { evals: registry.metadata() });
8230
- await write("suites.json", { suites: registry.suiteMetadata() });
8231
- await write("runs.json", { runs });
8232
- let trials = 0;
8233
- let files = 0;
8234
- for (const run of runs) {
8235
- const runTrials = await reports.listTrials(run.id);
8236
- await write(`runs/${run.id}/trials.json`, { trials: runTrials });
8237
- for (const trial of runTrials) {
8238
- trials += 1;
8239
- const base = `runs/${run.id}/trials/${trial.id}`;
8240
- await write(`${base}/detail.json`, await reports.trialDetail(run.id, trial.id));
8241
- await write(`${base}/events.json`, { events: await reports.trajectory(run.id, trial.id) });
8242
- await write(`${base}/artifacts.json`, { artifacts: await reports.listArtifacts(run.id, trial.id) });
8243
- const workspace = await reports.listCandidateWorkspace(run.id, trial.id);
8244
- await write(`${base}/workspace.json`, workspace);
8245
- for (const entry of workspace) {
8246
- if (entry.kind !== "file") continue;
8247
- const response = await reports.candidateFile(run.id, trial.id, entry.path);
8248
- const target = resolve(data, base, "workspace", entry.path);
8249
- const root = resolve(data, base, "workspace");
8250
- if (!target.startsWith(`${root}${sep}`)) throw new Error(`Invalid workspace path: ${entry.path}`);
8251
- await mkdir(dirname(target), { recursive: true });
8252
- await writeFile(target, new Uint8Array(await response.arrayBuffer()));
8253
- files += 1;
8259
+ const snapshot = await step("Run reports", async (progress) => {
8260
+ const all = await reports.listRuns();
8261
+ const runs = all.filter((run) => !run.archived);
8262
+ await write("runs.json", { runs });
8263
+ let trials = 0;
8264
+ let files = 0;
8265
+ const trialIds = [];
8266
+ for (const [index, run] of runs.entries()) {
8267
+ progress(`run ${index + 1}/${runs.length} · ${run.id.slice(0, 8)}`);
8268
+ const runTrials = await reports.listTrials(run.id);
8269
+ await write(`runs/${run.id}/trials.json`, { trials: runTrials });
8270
+ for (const trial of runTrials) {
8271
+ trials += 1;
8272
+ trialIds.push(trial.id);
8273
+ files += await snapshotTrial(reports, write, data, run.id, trial.id);
8254
8274
  }
8255
8275
  }
8256
- }
8276
+ return {
8277
+ runs: runs.length,
8278
+ trials,
8279
+ files,
8280
+ skipped: reports.legacyRunIds().length,
8281
+ archived: all.length - runs.length,
8282
+ runIds: runs.map((run) => run.id),
8283
+ trialIds
8284
+ };
8285
+ }, ({ runs, trials, files, skipped, archived }) => `${count(runs, "run")} · ${count(trials, "trial")} · ${count(files, "candidate file")}` + (archived ? ` · ${archived} archived excluded` : "") + (skipped ? ` · ${skipped} older-format skipped` : ""));
8286
+ await step("Route pages", async () => {
8287
+ const routes = staticRoutes({
8288
+ suites: registry.suites.map(authoringId),
8289
+ evals: registry.catalog().map((entry) => entry.path),
8290
+ runs: snapshot.runIds,
8291
+ trials: snapshot.trialIds
8292
+ });
8293
+ for (const route of routes) {
8294
+ const file = resolve(outDir, `${route}.html`);
8295
+ await mkdir(dirname(file), { recursive: true });
8296
+ await writeFile(file, staticIndex(template, baseFor(route)));
8297
+ }
8298
+ await writeFile(resolve(outDir, "404.html"), staticIndex(template, "/"));
8299
+ return routes.length;
8300
+ }, (pages) => `${count(pages, "page")} with clean /path URLs`);
8301
+ const { runIds: _runs, trialIds: _trials, ...summary } = snapshot;
8257
8302
  return {
8258
8303
  outDir,
8259
- runs: runs.length,
8260
- trials,
8304
+ ...summary
8305
+ };
8306
+ }
8307
+ /**
8308
+ * Every dashboard URL known at build time, as file paths without `.html`.
8309
+ * Static hosts serve `/runs/abc` from `runs/abc.html`, so links and reloads
8310
+ * work without SPA rewrite rules. IDs that cannot be a safe file name are
8311
+ * skipped; those pages still open by navigating inside the app.
8312
+ */
8313
+ function staticRoutes(ids) {
8314
+ const safe = (id) => id.length > 0 && !/[/\\\0]/.test(id) && id !== "." && id !== "..";
8315
+ const under = (prefix, values) => values.filter(safe).map((id) => `${prefix}/${id}`);
8316
+ return [
8317
+ "suites",
8318
+ "agents",
8319
+ "fixtures",
8320
+ "runs",
8321
+ ...under("suites", ids.suites),
8322
+ ...under("evals", ids.evals),
8323
+ ...under("runs", ids.runs),
8324
+ ...under("trials", ids.trials),
8325
+ ...under("workspaces", ids.trials)
8326
+ ];
8327
+ }
8328
+ /** `runs/abc.html` is served at `<mount>/runs/abc`; its base is one level up. */
8329
+ function baseFor(route) {
8330
+ const depth = route.split("/").length - 1;
8331
+ return depth ? "../".repeat(depth) : "./";
8332
+ }
8333
+ function count(value, noun) {
8334
+ return `${value} ${noun}${value === 1 ? "" : "s"}`;
8335
+ }
8336
+ /** Writes one trial's JSON views and candidate files; returns the file count. */
8337
+ async function snapshotTrial(reports, write, data, runId, trialId) {
8338
+ const base = `runs/${runId}/trials/${trialId}`;
8339
+ await write(`${base}/detail.json`, await reports.trialDetail(runId, trialId));
8340
+ await write(`${base}/events.json`, { events: await reports.trajectory(runId, trialId) });
8341
+ await write(`${base}/artifacts.json`, { artifacts: await reports.listArtifacts(runId, trialId) });
8342
+ const workspace = await reports.listCandidateWorkspace(runId, trialId);
8343
+ await write(`${base}/workspace.json`, workspace);
8344
+ const root = resolve(data, base, "workspace");
8345
+ let files = 0;
8346
+ for (const entry of workspace) {
8347
+ if (entry.kind !== "file") continue;
8348
+ const target = resolve(root, entry.path);
8349
+ if (!target.startsWith(`${root}${sep}`)) throw new Error(`Invalid workspace path: ${entry.path}`);
8350
+ const response = await reports.candidateFile(runId, trialId, entry.path);
8351
+ await mkdir(dirname(target), { recursive: true });
8352
+ await writeFile(target, new Uint8Array(await response.arrayBuffer()));
8353
+ files += 1;
8354
+ }
8355
+ return files;
8356
+ }
8357
+ /** Total size and file count of a directory, for the build summary. */
8358
+ async function directorySize(root) {
8359
+ let bytes = 0;
8360
+ let files = 0;
8361
+ for (const entry of await readdir(root, {
8362
+ withFileTypes: true,
8363
+ recursive: true
8364
+ })) {
8365
+ if (!entry.isFile()) continue;
8366
+ bytes += (await stat(resolve(entry.parentPath, entry.name))).size;
8367
+ files += 1;
8368
+ }
8369
+ return {
8370
+ bytes,
8261
8371
  files
8262
8372
  };
8263
8373
  }
8264
- /** Use relative asset URLs and select the static API in the browser bundle. */
8265
- function staticIndex(html) {
8266
- const result = html.replace("<html lang=\"en\">", "<html lang=\"en\" data-evalkit-snapshot=\"static\">").replaceAll("\"/assets/", "\"./assets/");
8267
- if (!result.includes("data-evalkit-snapshot=\"static\"")) throw new Error("Bundled dashboard index.html has an unexpected shape");
8374
+ /**
8375
+ * Select the static API and make every URL relative to `<base href>`, which
8376
+ * points at the snapshot root. The app derives its router basename from it.
8377
+ */
8378
+ function staticIndex(html, base = "./") {
8379
+ const result = html.replace("<html lang=\"en\">", "<html lang=\"en\" data-evalkit-snapshot=\"static\">").replace("<head>", `<head>\n <base href="${base}" />`).replaceAll("\"/assets/", "\"./assets/");
8380
+ if (!result.includes("data-evalkit-snapshot=\"static\"") || !result.includes("<base href=")) throw new Error("Bundled dashboard index.html has an unexpected shape");
8268
8381
  return result;
8269
8382
  }
8270
8383
  async function prepareOutDir(outDir) {
@@ -8460,6 +8573,111 @@ async function loadProject(cwd, configPath) {
8460
8573
  };
8461
8574
  }
8462
8575
  //#endregion
8576
+ //#region ../cli/src/terminal.ts
8577
+ /** Human CLI output. Colors and live progress only appear on an interactive terminal. */
8578
+ const interactive = Boolean(process.stdout.isTTY);
8579
+ const colored = interactive && !process.env.NO_COLOR && process.env.TERM !== "dumb";
8580
+ const paint = (code, reset) => (text) => colored ? `\x1b[${code}m${text}\x1b[${reset}m` : text;
8581
+ const bold = paint(1, 22);
8582
+ const dim = paint(2, 22);
8583
+ const red = paint(31, 39);
8584
+ const green = paint(32, 39);
8585
+ const yellow = paint(33, 39);
8586
+ const cyan = paint(36, 39);
8587
+ function banner(command, version) {
8588
+ console.log(`\n ${bold(cyan("EvalKit"))} ${bold(command)} ${dim(`v${version}`)}\n`);
8589
+ }
8590
+ function duration(ms) {
8591
+ return ms < 1e3 ? `${Math.round(ms)}ms` : `${(ms / 1e3).toFixed(2)}s`;
8592
+ }
8593
+ function bytes(size) {
8594
+ if (size < 1024) return `${size} B`;
8595
+ if (size < 1024 ** 2) return `${(size / 1024).toFixed(1)} kB`;
8596
+ return `${(size / 1024 ** 2).toFixed(1)} MB`;
8597
+ }
8598
+ function plural(count, noun) {
8599
+ return `${count} ${noun}${count === 1 ? "" : "s"}`;
8600
+ }
8601
+ /** Times one build step, showing a live line while it runs. */
8602
+ async function step(label, work, describe) {
8603
+ const started = performance.now();
8604
+ const progress = (detail) => {
8605
+ if (interactive) process.stdout.write(`\r\x1b[2K ${cyan("◇")} ${label.padEnd(20)} ${dim(detail)}`);
8606
+ };
8607
+ progress("…");
8608
+ try {
8609
+ const value = await work(progress);
8610
+ const elapsed = duration(performance.now() - started);
8611
+ if (interactive) process.stdout.write("\r\x1B[2K");
8612
+ console.log(` ${green("✓")} ${label.padEnd(20)} ${describe(value)} ${dim(elapsed)}`);
8613
+ return value;
8614
+ } catch (error) {
8615
+ if (interactive) process.stdout.write("\r\x1B[2K");
8616
+ console.log(` ${red("✗")} ${label}`);
8617
+ throw error;
8618
+ }
8619
+ }
8620
+ function clock() {
8621
+ return dim((/* @__PURE__ */ new Date()).toTimeString().slice(0, 8));
8622
+ }
8623
+ function cellLabel(name, parameters) {
8624
+ const shown = Object.entries(parameters).map(([key, value]) => `${key}=${typeof value === "string" ? value : JSON.stringify(value)}`).join(" ");
8625
+ return shown ? `${bold(name)} ${dim(shown)}` : bold(name);
8626
+ }
8627
+ function passed(result) {
8628
+ return result.status === "completed" && (result.aggregateScoring?.passRate === 1 || Boolean(result.scoring?.passed));
8629
+ }
8630
+ /** Live log for runs launched from the `evalkit dev` dashboard. */
8631
+ const runLog = {
8632
+ started(name, parameters = {}, suiteId) {
8633
+ const suite = suiteId ? dim(` (suite ${suiteId})`) : "";
8634
+ console.log(`${clock()} ${cyan("▶ run ")} ${cellLabel(name, parameters)}${suite}`);
8635
+ },
8636
+ finished(name, parameters, result, ms) {
8637
+ const ok = passed(result);
8638
+ const trials = result.trials?.length ?? 1;
8639
+ const passedTrials = result.aggregateScoring?.passed ?? (ok ? trials : 0);
8640
+ const status = result.status === "completed" ? ok ? green("✓ pass") : red("✗ fail") : red(`✗ ${result.status}`);
8641
+ console.log(`${clock()} ${status} ${cellLabel(name, parameters)} ${dim(`${passedTrials}/${trials} trials · ${duration(ms)}`)}`);
8642
+ console.log(` ${dim(`report ${result.reportLocation}`)}`);
8643
+ },
8644
+ cell(cell, result, ms) {
8645
+ runLog.finished(cell.eval.name ?? authoringId(cell.eval), cell.parameters, result, ms);
8646
+ },
8647
+ failed(name, error) {
8648
+ console.log(`${clock()} ${red("✗ error")} ${bold(name)} ${error instanceof Error ? error.message : String(error)}`);
8649
+ }
8650
+ };
8651
+ /**
8652
+ * The runner logs structured Effect lines; the CLI prints its own summaries, so
8653
+ * only warnings show by default. `EVALKIT_LOG_LEVEL=info` (or `debug`) restores them.
8654
+ */
8655
+ function runnerLogs(effect) {
8656
+ const requested = process.env.EVALKIT_LOG_LEVEL?.toLowerCase();
8657
+ const level = LogLevel.allLevels.find((item) => item.label.toLowerCase() === requested) ?? LogLevel.Warning;
8658
+ return effect.pipe(Logger.withMinimumLogLevel(level));
8659
+ }
8660
+ /** True when a person can answer a prompt (not CI, pipes, or redirected input). */
8661
+ function canPrompt() {
8662
+ return Boolean(process.stdin.isTTY && process.stdout.isTTY);
8663
+ }
8664
+ async function confirm(question) {
8665
+ const { createInterface } = await import("node:readline/promises");
8666
+ const prompt = createInterface({
8667
+ input: process.stdin,
8668
+ output: process.stdout
8669
+ });
8670
+ try {
8671
+ const answer = await prompt.question(` ${cyan("?")} ${question} ${dim("(y/N)")} `);
8672
+ return /^y(es)?$/i.test(answer.trim());
8673
+ } catch {
8674
+ console.log("");
8675
+ return false;
8676
+ } finally {
8677
+ prompt.close();
8678
+ }
8679
+ }
8680
+ //#endregion
8463
8681
  //#region ../cli/src/run-command.ts
8464
8682
  function parseRunArgs(args) {
8465
8683
  const { values, positionals } = parseArgs({
@@ -8607,7 +8825,7 @@ async function runProjectCommand(args, cwd = process.cwd(), parsed = parseRunArg
8607
8825
  }
8608
8826
  if (cells > (project.config.execution?.maxCells ?? 100) && !values.all) throw new Error(`Selected ${cells} cells. Narrow the selection, inspect --dry-run, or pass --all.`);
8609
8827
  if (!values.json) console.log(`Running ${cells} cell(s) from ${plan.matrix}`);
8610
- if ((await Effect.runPromise(runMatrix(matrix, {
8828
+ if ((await Effect.runPromise(runnerLogs(runMatrix(matrix, {
8611
8829
  selection,
8612
8830
  concurrency: parsed.concurrency ?? project.config.execution?.concurrency ?? 4,
8613
8831
  trials,
@@ -8633,7 +8851,56 @@ async function runProjectCommand(args, cwd = process.cwd(), parsed = parseRunArg
8633
8851
  }
8634
8852
  }
8635
8853
  }
8636
- }))).failed) process.exitCode = 1;
8854
+ })))).failed) process.exitCode = 1;
8855
+ }
8856
+ //#endregion
8857
+ //#region ../cli/src/tidy.ts
8858
+ /**
8859
+ * Every eval the project can run: standalone, in suites, or only in matrices.
8860
+ * A run is archived only when its eval is in none of these.
8861
+ */
8862
+ function projectEvalIds(registry) {
8863
+ return new Set([
8864
+ ...registry.evals,
8865
+ ...registry.suites.flatMap((suite) => suite.evals),
8866
+ ...registry.matrices.flatMap((matrix) => matrix.evals)
8867
+ ].map(authoringId));
8868
+ }
8869
+ /**
8870
+ * Mark runs of removed evals as archived, and restore runs whose eval returned.
8871
+ * Reports are never deleted: archiving writes a marker file beside the manifest.
8872
+ */
8873
+ async function tidyReports(options) {
8874
+ const known = projectEvalIds(options.registry);
8875
+ const reports = createDashboardReportReader(options.reportRoot, options.registry);
8876
+ const result = {
8877
+ archived: [],
8878
+ restored: [],
8879
+ stillArchived: 0,
8880
+ current: 0
8881
+ };
8882
+ for (const run of await reports.listRuns()) {
8883
+ const marker = resolve(options.reportRoot, run.id, ARCHIVE_MARKER);
8884
+ const entry = {
8885
+ runId: run.id,
8886
+ evalId: run.evalId
8887
+ };
8888
+ if (known.has(run.evalId)) {
8889
+ result.current += 1;
8890
+ if (run.archived) {
8891
+ await rm(marker, { force: true });
8892
+ result.restored.push(entry);
8893
+ }
8894
+ } else if (run.archived) result.stillArchived += 1;
8895
+ else {
8896
+ await writeFile(marker, `${JSON.stringify({
8897
+ archivedAt: (/* @__PURE__ */ new Date()).toISOString(),
8898
+ reason: `Eval "${run.evalId}" is no longer in the project`
8899
+ })}\n`);
8900
+ result.archived.push(entry);
8901
+ }
8902
+ }
8903
+ return result;
8637
8904
  }
8638
8905
  //#endregion
8639
8906
  //#region ../cli/src/new-project-template.ts
@@ -8750,7 +9017,7 @@ scores, evidence, and candidate files (never evaluator workspaces): review it be
8750
9017
  }
8751
9018
  //#endregion
8752
9019
  //#region ../cli/src/new-project.ts
8753
- const npmDependency = "^0.2.0";
9020
+ const npmDependency = "^0.3.0";
8754
9021
  const ignored = [
8755
9022
  "node_modules/",
8756
9023
  "_evalkit-results/",
@@ -8867,6 +9134,7 @@ async function writeStarterReadme(root, name, inPlace) {
8867
9134
  }
8868
9135
  //#endregion
8869
9136
  //#region ../cli/src/index.ts
9137
+ const VERSION = "0.3.0";
8870
9138
  let projectRoot = process.cwd();
8871
9139
  let reportRoot = resolve(projectRoot, "_evalkit-results");
8872
9140
  const sandboxRoot = resolve(projectRoot, "_evalkit-sandbox");
@@ -8886,13 +9154,17 @@ async function runEvals(options) {
8886
9154
  const registry = await loadRegistry();
8887
9155
  const evaluations = options.evalIds.map((id) => registry.get(id));
8888
9156
  if (evaluations.some((e) => !e)) throw new Error("Unknown eval");
8889
- await Effect.runPromise(Effect.all(evaluations.map((evaluation) => runEval(evaluation, {
8890
- report: localReportStore(reportRoot),
8891
- workspaceRoot: sandboxRoot,
8892
- suiteId: options.suiteId,
8893
- runtime: runtimeFor(evaluation),
8894
- concurrency: options.concurrency
8895
- })), { concurrency: options.concurrency }));
9157
+ await Effect.runPromise(runnerLogs(Effect.all(evaluations.map((evaluation) => {
9158
+ const name = evaluation.name ?? authoringId(evaluation);
9159
+ const started = performance.now();
9160
+ return Effect.sync(() => runLog.started(name, {}, options.suiteId)).pipe(Effect.andThen(runEval(evaluation, {
9161
+ report: localReportStore(reportRoot),
9162
+ workspaceRoot: sandboxRoot,
9163
+ suiteId: options.suiteId,
9164
+ runtime: runtimeFor(evaluation),
9165
+ concurrency: options.concurrency
9166
+ })), Effect.tap((result) => runLog.finished(name, {}, result, performance.now() - started)), Effect.tapError((error) => Effect.sync(() => runLog.failed(name, error))));
9167
+ }), { concurrency: options.concurrency })));
8896
9168
  }
8897
9169
  function registerMatrixRoutes(app, registry, dashboardMatrix, trials) {
8898
9170
  let matrixRunning = false;
@@ -8935,14 +9207,18 @@ function registerMatrixRoutes(app, registry, dashboardMatrix, trials) {
8935
9207
  }
8936
9208
  if (matrixRunning) return context.text("A dashboard matrix cell is already running", 409);
8937
9209
  matrixRunning = true;
8938
- Effect.runPromise(runMatrix(dashboardMatrix, {
9210
+ const name = evaluation.name ?? authoringId(evaluation);
9211
+ const started = performance.now();
9212
+ runLog.started(name, body.parameters ?? {}, suiteId);
9213
+ Effect.runPromise(runnerLogs(runMatrix(dashboardMatrix, {
8939
9214
  selection,
8940
9215
  concurrency: 1,
8941
9216
  trials,
8942
9217
  ...suiteId ? { suiteId } : {},
8943
9218
  report: localReportStore(reportRoot),
8944
- workspaceRoot: sandboxRoot
8945
- })).catch((error) => console.error("Dashboard matrix run failed:", error)).finally(() => {
9219
+ workspaceRoot: sandboxRoot,
9220
+ onResult: (cell, result) => runLog.cell(cell, result, performance.now() - started)
9221
+ }))).catch((error) => runLog.failed(name, error)).finally(() => {
8946
9222
  matrixRunning = false;
8947
9223
  });
8948
9224
  } else {
@@ -8951,7 +9227,7 @@ function registerMatrixRoutes(app, registry, dashboardMatrix, trials) {
8951
9227
  evalIds: [authoringId(evaluation)],
8952
9228
  suiteId,
8953
9229
  concurrency: 32
8954
- }).catch((error) => console.error("Dashboard run failed:", error));
9230
+ }).catch(() => {});
8955
9231
  }
8956
9232
  return context.json({ accepted: true }, 202);
8957
9233
  });
@@ -8963,7 +9239,7 @@ function registerMatrixRoutes(app, registry, dashboardMatrix, trials) {
8963
9239
  evalIds: suite.evals.map(authoringId),
8964
9240
  suiteId: authoringId(suite),
8965
9241
  concurrency: 32
8966
- }).catch((error) => console.error("Dashboard suite run failed:", error));
9242
+ }).catch(() => {});
8967
9243
  return context.json({ accepted: true }, 202);
8968
9244
  });
8969
9245
  }
@@ -9047,14 +9323,32 @@ async function serveDashboard() {
9047
9323
  const registry = await loadRegistry();
9048
9324
  const dashboardRoot = dashboardAssets();
9049
9325
  const app = new Hono();
9050
- registerMatrixRoutes(app, registry, project?.config.matrix ? registry.matrices.at(-1) : void 0, project?.config.execution?.trials);
9326
+ const dashboardMatrix = project?.config.matrix ? registry.matrices.at(-1) : void 0;
9327
+ registerMatrixRoutes(app, registry, dashboardMatrix, project?.config.execution?.trials);
9051
9328
  registerReportRoutes(app, registry, createDashboardReportReader(reportRoot, registry));
9052
9329
  registerDashboardAssets(app, dashboardRoot);
9053
- const url = `http://localhost:${Bun.serve({
9054
- port: Number(process.env.PORT ?? 4317),
9055
- fetch: app.fetch
9056
- }).port}`;
9057
- console.log(`EvalKit dashboard: ${url}`);
9330
+ const port = Number(process.env.PORT ?? 4317);
9331
+ let server;
9332
+ try {
9333
+ server = Bun.serve({
9334
+ port,
9335
+ fetch: app.fetch
9336
+ });
9337
+ } catch (error) {
9338
+ if (error.code === "EADDRINUSE") throw new Error(`Port ${port} is already in use. Stop the other server or choose another port: PORT=4318 evalkit dev`);
9339
+ throw error;
9340
+ }
9341
+ const url = `http://localhost:${server.port}`;
9342
+ const reader = createDashboardReportReader(reportRoot, registry);
9343
+ const runs = (await reader.listRuns()).length;
9344
+ const legacy = reader.legacyRunIds().length;
9345
+ const matrixCells = dashboardMatrix?.count({ parameters: {} });
9346
+ const row = (label, value) => console.log(` ${cyan("➜")} ${bold(label.padEnd(9))}${value}`);
9347
+ banner("dev", VERSION);
9348
+ row("Local:", cyan(url));
9349
+ row("Project:", `${plural(registry.evals.length, "eval")} · ${plural(registry.suites.length, "suite")}` + (dashboardMatrix ? ` · matrix ${authoringId(dashboardMatrix)} (${plural(matrixCells, "cell")})` : ""));
9350
+ row("Reports:", `${relative(process.cwd(), reportRoot) || "."} ${dim(`(${plural(runs, "run")}${legacy ? `, ${legacy} from an older EvalKit hidden` : ""})`)}`);
9351
+ console.log(`\n ${dim("Runs started from the dashboard are logged below. Press Ctrl+C to stop.")}\n`);
9058
9352
  if (process.env.EVALKIT_NO_OPEN !== "1") {
9059
9353
  const opener = process.platform === "darwin" ? "open" : process.platform === "win32" ? "start" : "xdg-open";
9060
9354
  try {
@@ -9132,18 +9426,82 @@ const devCommand = make$1("dev", { config }, ({ config }) => Effect.tryPromise({
9132
9426
  }));
9133
9427
  const buildCommand = make$1("build", {
9134
9428
  config,
9135
- out: text("out").pipe(optional)
9136
- }, ({ config, out }) => Effect.tryPromise({
9429
+ out: text("out").pipe(optional),
9430
+ run: boolean("run"),
9431
+ noRun: boolean("no-run")
9432
+ }, ({ config, out, run, noRun }) => Effect.tryPromise({
9137
9433
  try: async () => {
9138
- await useProject(Option.getOrUndefined(config));
9434
+ if (run && noRun) throw new Error("Choose either --run or --no-run");
9435
+ const started = performance.now();
9436
+ const configPath = Option.getOrUndefined(config);
9437
+ banner("build", VERSION);
9438
+ await loadProjectStep(configPath);
9439
+ if (await shouldRunEvals({
9440
+ run,
9441
+ noRun
9442
+ })) await runAllEvals(configPath);
9139
9443
  const result = await buildDashboard({
9140
9444
  registry: project.registry,
9141
9445
  reportRoot,
9142
9446
  dashboardRoot: dashboardAssets(),
9143
9447
  outDir: resolve(projectRoot, Option.getOrElse(out, () => "_evalkit-dashboard"))
9144
- });
9145
- console.log(`Built a read-only dashboard snapshot at ${result.outDir}\n ${result.runs} run(s), ${result.trials} trial(s), ${result.files} candidate file(s)\nServe it with any static file server. It contains report messages, scores, evidence,
9146
- and candidate files (never evaluator workspaces): review it before sharing.`);
9448
+ }, step);
9449
+ const size = await directorySize(result.outDir);
9450
+ const shown = relative(process.cwd(), result.outDir) || ".";
9451
+ console.log(`\n ${bold(`${shown}/`)} ${dim(`${bytes(size.bytes)} · ${plural(size.files, "file")}`)}`);
9452
+ console.log(`\n ${bold("Built read-only dashboard")} in ${duration(performance.now() - started)}\n`);
9453
+ console.log(` ${cyan("➜")} ${bold("Preview:")} bunx serve ${shown}`);
9454
+ console.log(` ${yellow("⚠")} Contains transcripts, scores, evidence, and candidate files\n (never evaluator workspaces). Review before sharing.\n`);
9455
+ },
9456
+ catch: (error) => error
9457
+ }));
9458
+ async function loadProjectStep(configPath) {
9459
+ await step("Eval project", () => useProject(configPath), () => {
9460
+ const where = relative(process.cwd(), projectRoot);
9461
+ return `${plural(project.registry.evals.length, "eval")} loaded${where ? ` from ${where}` : ""}`;
9462
+ });
9463
+ }
9464
+ /**
9465
+ * A snapshot needs data: with no current results, run the evals. Otherwise ask,
9466
+ * or keep the existing results when no one can answer (CI, pipes).
9467
+ */
9468
+ async function shouldRunEvals(flags) {
9469
+ if (flags.run) return true;
9470
+ if (flags.noRun) return false;
9471
+ const runs = (await createDashboardReportReader(reportRoot, project.registry).listRuns()).filter((entry) => !entry.archived).length;
9472
+ if (!runs) {
9473
+ console.log(` ${cyan("◇")} No results yet, so running the evals to have data to build.`);
9474
+ return true;
9475
+ }
9476
+ if (!canPrompt()) {
9477
+ console.log(` ${dim(`Using ${plural(runs, "existing run")}; pass --run to run the evals first.`)}`);
9478
+ return false;
9479
+ }
9480
+ return confirm(`Run the evals before building? ${dim(`(${plural(runs, "existing run")})`)}`);
9481
+ }
9482
+ async function runAllEvals(configPath) {
9483
+ const started = performance.now();
9484
+ console.log(`\n ${cyan("▶")} ${bold("Running evals")}\n`);
9485
+ await runProjectCommand([], process.cwd(), parseRunArgs(configPath ? ["--config", configPath] : []));
9486
+ const failed = process.exitCode === 1;
9487
+ console.log(`\n ${failed ? yellow("⚠") : cyan("✓")} ${bold("Evals finished")} in ${duration(performance.now() - started)}` + (failed ? dim(" (some failed; building with their results)") : "") + "\n");
9488
+ }
9489
+ const tidyCommand = make$1("tidy", { config }, ({ config }) => Effect.tryPromise({
9490
+ try: async () => {
9491
+ banner("tidy", VERSION);
9492
+ await loadProjectStep(Option.getOrUndefined(config));
9493
+ const result = await step("Run reports", () => tidyReports({
9494
+ registry: project.registry,
9495
+ reportRoot
9496
+ }), (value) => `${plural(value.current + value.archived.length + value.stillArchived, "run")} checked`);
9497
+ const byEval = (entries) => [...entries.reduce((groups, entry) => groups.set(entry.evalId, (groups.get(entry.evalId) ?? 0) + 1), /* @__PURE__ */ new Map()).entries()].map(([id, total]) => ` ${bold(id)} ${dim(`(${plural(total, "run")})`)}`).join("\n");
9498
+ console.log("");
9499
+ if (result.archived.length) console.log(` ${yellow("◆")} Archived ${plural(result.archived.length, "run")} of evals no longer in the project:\n${byEval(result.archived)}`);
9500
+ if (result.restored.length) console.log(` ${cyan("◆")} Restored ${plural(result.restored.length, "run")} whose eval is back:\n${byEval(result.restored)}`);
9501
+ if (!result.archived.length && !result.restored.length) console.log(` ${cyan("✓")} Nothing new to tidy.`);
9502
+ const totalArchived = result.archived.length + result.stillArchived;
9503
+ if (totalArchived) console.log(`\n ${dim(`Archived runs (${totalArchived}) are hidden in \`evalkit dev\` (filter "Show archived")`)}\n ${dim("and left out of `evalkit build`. Nothing was deleted.")}`);
9504
+ console.log("");
9147
9505
  },
9148
9506
  catch: (error) => error
9149
9507
  }));
@@ -9151,10 +9509,11 @@ const cli = run(make$1("evalkit").pipe(withSubcommands([
9151
9509
  newCommand,
9152
9510
  runCommand,
9153
9511
  devCommand,
9154
- buildCommand
9512
+ buildCommand,
9513
+ tidyCommand
9155
9514
  ])), {
9156
9515
  name: "EvalKit",
9157
- version: "0.2.0"
9516
+ version: VERSION
9158
9517
  });
9159
9518
  const argv = process.argv.slice();
9160
9519
  if (argv[2] === "help") argv.splice(2, 1, "--help");