@leostera/evalkit 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/CHANGELOG.md +81 -0
  2. package/README.md +8 -7
  3. package/dist/cli.mjs +909 -149
  4. package/dist/cli.mjs.map +1 -1
  5. package/dist/dashboard/assets/index-D9NtA46z.css +2 -0
  6. package/dist/dashboard/assets/index-DrBwLKJD.js +30 -0
  7. package/dist/dashboard/index.html +2 -2
  8. package/dist/files-C-U0-PWW.mjs +26 -0
  9. package/dist/files-C-U0-PWW.mjs.map +1 -0
  10. package/dist/{index-M5eRlMfd.d.mts → index-U6TUTBf7.d.mts} +87 -9
  11. package/dist/index-U6TUTBf7.d.mts.map +1 -0
  12. package/dist/index-db-HdZXelOS.d.mts +142 -0
  13. package/dist/index-db-HdZXelOS.d.mts.map +1 -0
  14. package/dist/index-db-vbqEGFaU.mjs +548 -0
  15. package/dist/index-db-vbqEGFaU.mjs.map +1 -0
  16. package/dist/index.d.mts +2 -2
  17. package/dist/index.mjs +2 -2
  18. package/dist/index.mjs.map +1 -1
  19. package/dist/runner.d.mts +21 -3
  20. package/dist/runner.d.mts.map +1 -1
  21. package/dist/runner.mjs +1 -1
  22. package/dist/share-worker.d.mts +51 -0
  23. package/dist/share-worker.d.mts.map +1 -0
  24. package/dist/share-worker.mjs +73 -0
  25. package/dist/share-worker.mjs.map +1 -0
  26. package/dist/share.d.mts +116 -0
  27. package/dist/share.d.mts.map +1 -0
  28. package/dist/share.mjs +4 -0
  29. package/dist/{src-DrwwgaPn.mjs → src-ButXPY8P.mjs} +20 -5
  30. package/dist/src-ButXPY8P.mjs.map +1 -0
  31. package/dist/{src-DluldGKn.mjs → src-DoPs9Yug.mjs} +11 -7
  32. package/dist/src-DoPs9Yug.mjs.map +1 -0
  33. package/dist/src-DzZueFn3.mjs +98 -0
  34. package/dist/src-DzZueFn3.mjs.map +1 -0
  35. package/package.json +10 -1
  36. package/dist/dashboard/assets/index-Z6raM-uW.js +0 -30
  37. package/dist/dashboard/assets/index-fUDsY6z-.css +0 -2
  38. package/dist/index-M5eRlMfd.d.mts.map +0 -1
  39. package/dist/src-DluldGKn.mjs.map +0 -1
  40. package/dist/src-DrwwgaPn.mjs.map +0 -1
package/dist/cli.mjs CHANGED
@@ -1,11 +1,13 @@
1
1
  #!/usr/bin/env bun
2
- import { F as defineEvalMatrix, I as canonicalParameters, m as registerEvals, n as authoringId } from "./src-DrwwgaPn.mjs";
3
- import { a as readTrialEvents, c as readTrialSummary, i as readRunSummary, l as localReportStore, n as runMatrix, o as readTrialManifest, r as readRunManifest, t as runEval } from "./src-DluldGKn.mjs";
2
+ import { I as defineEvalMatrix, L as canonicalParameters, m as registerEvals, n as authoringId } from "./src-ButXPY8P.mjs";
3
+ import { a as readTrialEvents, c as readTrialSummary, i as readRunSummary, l as localReportStore, n as runMatrix, o as readTrialManifest, r as readRunManifest, t as runEval } from "./src-DoPs9Yug.mjs";
4
+ import { c as dashboardIndex, d as encodeBundle, o as repoKey } from "./index-db-vbqEGFaU.mjs";
5
+ import { t as createDirectoryShareApp } from "./src-DzZueFn3.mjs";
4
6
  import * as Schema from "effect/Schema";
5
- import { Effect, Option } from "effect";
6
- import { cp, mkdir, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
7
+ import { Effect, LogLevel, Logger, Option } from "effect";
8
+ import { cp, mkdir, readFile, readdir, realpath, rm, stat, writeFile } from "node:fs/promises";
7
9
  import * as Path from "node:path";
8
- import { basename, dirname, extname, relative, resolve, sep } from "node:path";
10
+ import { basename, dirname, extname, join, relative, resolve, sep } from "node:path";
9
11
  import * as OS from "node:os";
10
12
  import * as Crypto from "node:crypto";
11
13
  import * as NFS from "node:fs";
@@ -43,11 +45,11 @@ import * as EffectSecret from "effect/Secret";
43
45
  import * as Effectable from "effect/Effectable";
44
46
  import * as Match from "effect/Match";
45
47
  import * as EffectNumber from "effect/Number";
46
- import * as LogLevel from "effect/LogLevel";
48
+ import * as LogLevel$1 from "effect/LogLevel";
47
49
  import * as Config from "effect/Config";
48
50
  import * as HashMap from "effect/HashMap";
49
51
  import * as Order from "effect/Order";
50
- import * as Logger from "effect/Logger";
52
+ import * as Logger$1 from "effect/Logger";
51
53
  import * as Unify from "effect/Unify";
52
54
  import * as HashSet from "effect/HashSet";
53
55
  import * as SynchronizedRef from "effect/SynchronizedRef";
@@ -65,8 +67,9 @@ import * as FiberSet from "effect/FiberSet";
65
67
  import "effect/Fiber";
66
68
  import "effect/Pool";
67
69
  import * as MutableRef from "effect/MutableRef";
68
- import { parseArgs } from "node:util";
70
+ import { parseArgs, promisify } from "node:util";
69
71
  import * as ChildProcess from "node:child_process";
72
+ import { execFile } from "node:child_process";
70
73
  import * as RcRef from "effect/RcRef";
71
74
  import * as readline from "node:readline";
72
75
  import { Hono } from "hono";
@@ -392,13 +395,13 @@ const struct = (fields) => {
392
395
  /** @internal */
393
396
  const black = { _tag: "Black" };
394
397
  /** @internal */
395
- const red$3 = { _tag: "Red" };
398
+ const red$4 = { _tag: "Red" };
396
399
  /** @internal */
397
- const green$2 = { _tag: "Green" };
400
+ const green$3 = { _tag: "Green" };
398
401
  /** @internal */
399
402
  const magenta$1 = { _tag: "Magenta" };
400
403
  /** @internal */
401
- const cyan$1 = { _tag: "Cyan" };
404
+ const cyan$2 = { _tag: "Cyan" };
402
405
  /** @internal */
403
406
  const white$3 = { _tag: "White" };
404
407
  /** @internal */
@@ -500,7 +503,7 @@ const ESC = "\x1B[";
500
503
  const BEL = "\x07";
501
504
  const SEP = ";";
502
505
  /** @internal */
503
- const bold$1 = /*#__PURE__*/ make$5({ bold: /*#__PURE__*/ Option$1.some(/*#__PURE__*/ setBold(true)) });
506
+ const bold$2 = /*#__PURE__*/ make$5({ bold: /*#__PURE__*/ Option$1.some(/*#__PURE__*/ setBold(true)) });
504
507
  /** @internal */
505
508
  const italicized$1 = /*#__PURE__*/ make$5({ italicized: /*#__PURE__*/ Option$1.some(/*#__PURE__*/ setItalicized(true)) });
506
509
  /** @internal */
@@ -512,15 +515,15 @@ const brightColor = (color) => make$5({ foreground: Option$1.some(setColor(color
512
515
  /** @internal */
513
516
  const color$1 = (color) => make$5({ foreground: Option$1.some(setColor(color, false, "foreground")) });
514
517
  /** @internal */
515
- const red$2 = /*#__PURE__*/ color$1(red$3);
518
+ const red$3 = /*#__PURE__*/ color$1(red$4);
516
519
  /** @internal */
517
- const green$1 = /*#__PURE__*/ color$1(green$2);
520
+ const green$2 = /*#__PURE__*/ color$1(green$3);
518
521
  /** @internal */
519
522
  const white$2 = /*#__PURE__*/ color$1(white$3);
520
523
  /** @internal */
521
524
  const blackBright$1 = /*#__PURE__*/ brightColor(black);
522
525
  /** @internal */
523
- const cyanBright$1 = /*#__PURE__*/ brightColor(cyan$1);
526
+ const cyanBright$1 = /*#__PURE__*/ brightColor(cyan$2);
524
527
  /** @internal */
525
528
  const beep$2 = /*#__PURE__*/ make$5({ commands: /*#__PURE__*/ Arr.of(BEL) });
526
529
  /** @internal */
@@ -587,7 +590,7 @@ const stringifyInternal = (self) => {
587
590
  * @since 1.0.0
588
591
  * @category constructors
589
592
  */
590
- const bold = bold$1;
593
+ const bold$1 = bold$2;
591
594
  /**
592
595
  * @since 1.0.0
593
596
  * @category constructors
@@ -612,12 +615,12 @@ const color = color$1;
612
615
  * @since 1.0.0
613
616
  * @category colors
614
617
  */
615
- const red$1 = red$2;
618
+ const red$2 = red$3;
616
619
  /**
617
620
  * @since 1.0.0
618
621
  * @category colors
619
622
  */
620
- const green = green$1;
623
+ const green$1 = green$2;
621
624
  /**
622
625
  * @since 1.0.0
623
626
  * @category colors
@@ -2218,7 +2221,7 @@ const optimize = optimize$1;
2218
2221
  * @since 1.0.0
2219
2222
  * @category constructors
2220
2223
  */
2221
- const red = red$3;
2224
+ const red$1 = red$4;
2222
2225
  /**
2223
2226
  * @since 1.0.0
2224
2227
  * @category constructors
@@ -2228,7 +2231,7 @@ const magenta = magenta$1;
2228
2231
  * @since 1.0.0
2229
2232
  * @category constructors
2230
2233
  */
2231
- const cyan = cyan$1;
2234
+ const cyan$1 = cyan$2;
2232
2235
  /**
2233
2236
  * @since 1.0.0
2234
2237
  * @category constructors
@@ -2248,7 +2251,7 @@ const space = /*#__PURE__*/ text$6(" ");
2248
2251
  /** @internal */
2249
2252
  const code = (value) => highlight(value, white);
2250
2253
  /** @internal */
2251
- const error = (value) => highlight(value, red);
2254
+ const error = (value) => highlight(value, red$1);
2252
2255
  /** @internal */
2253
2256
  const highlight = (value, color) => ({
2254
2257
  _tag: "Highlight",
@@ -2307,7 +2310,7 @@ const toAnsiDoc$1 = (self) => {
2307
2310
  switch (self._tag) {
2308
2311
  case "Highlight": return annotate(toAnsiDoc$1(self.value), color(self.color));
2309
2312
  case "Sequence": return cat(toAnsiDoc$1(self.left), toAnsiDoc$1(self.right));
2310
- case "Strong": return annotate(toAnsiDoc$1(self.value), bold);
2313
+ case "Strong": return annotate(toAnsiDoc$1(self.value), bold$1);
2311
2314
  case "Text": return text$7(self.value);
2312
2315
  case "URI": return annotate(text$7(self.value), underlined);
2313
2316
  case "Weak": return annotate(toAnsiDoc$1(self.value), blackBright);
@@ -2373,11 +2376,11 @@ const toAnsiText = (self) => render$1(toAnsiDoc(self), { style: "pretty" });
2373
2376
  const toAnsiDocInternal = (self) => {
2374
2377
  switch (self._tag) {
2375
2378
  case "Empty": return empty$3;
2376
- case "Header": return pipe(annotate(toAnsiDoc$1(self.value), bold), cat(hardLine));
2379
+ case "Header": return pipe(annotate(toAnsiDoc$1(self.value), bold$1), cat(hardLine));
2377
2380
  case "Paragraph": return pipe(toAnsiDoc$1(self.value), cat(hardLine));
2378
2381
  case "DescriptionList": {
2379
2382
  const definitions = self.definitions.map(([span, doc]) => cats([
2380
- annotate(toAnsiDoc$1(span), bold),
2383
+ annotate(toAnsiDoc$1(span), bold$1),
2381
2384
  empty$3,
2382
2385
  indent(toAnsiDocInternal(doc), 2)
2383
2386
  ]));
@@ -2592,8 +2595,8 @@ function renderError$2(state, pointer) {
2592
2595
  onSome: (error) => {
2593
2596
  const errorLines = error.split(NEWLINE_REGEX$3);
2594
2597
  if (Arr.isNonEmptyReadonlyArray(errorLines)) {
2595
- const annotateLine = (line) => annotate(text$7(line), combine(italicized, red$1));
2596
- const prefix = cat(annotate(pointer, red$1), space$1);
2598
+ const annotateLine = (line) => annotate(text$7(line), combine(italicized, red$2));
2599
+ const prefix = cat(annotate(pointer, red$2), space$1);
2597
2600
  const lines = Arr.map(errorLines, (str) => annotateLine(str));
2598
2601
  return cursorSavePosition.pipe(cat(hardLine), cat(prefix), cat(align(vsep(lines))), cat(cursorRestorePosition));
2599
2602
  }
@@ -2612,7 +2615,7 @@ function renderParts(state, submitted = false) {
2612
2615
  });
2613
2616
  }
2614
2617
  function renderOutput$4(leadingSymbol, trailingSymbol, parts, options) {
2615
- const annotateLine = (line) => annotate(text$7(line), bold);
2618
+ const annotateLine = (line) => annotate(text$7(line), bold$1);
2616
2619
  const prefix = cat(leadingSymbol, space$1);
2617
2620
  return Arr.match(options.message.split(/\r?\n/), {
2618
2621
  onEmpty: () => hsep([
@@ -2642,7 +2645,7 @@ function renderSubmission$5(state, options) {
2642
2645
  return Effect$1.gen(function* () {
2643
2646
  const columns = yield* (yield* Terminal).columns;
2644
2647
  const figures$11 = yield* figures;
2645
- return renderOutput$4(annotate(figures$11.tick, green), annotate(figures$11.ellipsis, blackBright), renderParts(state, true), options).pipe(cat(hardLine), optimize(Deep), render$1({
2648
+ return renderOutput$4(annotate(figures$11.tick, green$1), annotate(figures$11.ellipsis, blackBright), renderParts(state, true), options).pipe(cat(hardLine), optimize(Deep), render$1({
2646
2649
  style: "pretty",
2647
2650
  options: { lineWidth: columns }
2648
2651
  }));
@@ -3106,7 +3109,7 @@ function handleClear$4(options) {
3106
3109
  }
3107
3110
  const NEWLINE_REGEX$2 = /\r?\n/;
3108
3111
  function renderPrompt(confirm, message, leadingSymbol, trailingSymbol) {
3109
- const annotateLine = (line) => annotate(text$7(line), bold);
3112
+ const annotateLine = (line) => annotate(text$7(line), bold$1);
3110
3113
  const prefix = cat(leadingSymbol, space$1);
3111
3114
  return Arr.match(message.split(NEWLINE_REGEX$2), {
3112
3115
  onEmpty: () => hsep([
@@ -3159,7 +3162,7 @@ function renderNextFrame$4(state, options) {
3159
3162
  options: { lineWidth: columns }
3160
3163
  }));
3161
3164
  }
3162
- const leadingSymbol = annotate(figures$10.tick, green);
3165
+ const leadingSymbol = annotate(figures$10.tick, green$1);
3163
3166
  const trailingSymbol = annotate(figures$10.ellipsis, blackBright);
3164
3167
  const promptMsg = renderPrompt(empty$3, options.message, leadingSymbol, trailingSymbol);
3165
3168
  const files = renderFiles(state, state.files, figures$10, options);
@@ -3173,7 +3176,7 @@ function renderSubmission$4(value, options) {
3173
3176
  return Effect$1.gen(function* () {
3174
3177
  const columns = yield* (yield* Terminal).columns;
3175
3178
  const figures$9 = yield* figures;
3176
- const leadingSymbol = annotate(figures$9.tick, green);
3179
+ const leadingSymbol = annotate(figures$9.tick, green$1);
3177
3180
  const trailingSymbol = annotate(figures$9.ellipsis, blackBright);
3178
3181
  return renderPrompt(empty$3, options.message, leadingSymbol, trailingSymbol).pipe(cat(space$1), cat(annotate(text$7(value), white$1)), cat(hardLine), render$1({
3179
3182
  style: "pretty",
@@ -3316,7 +3319,7 @@ function handleClear$3(options) {
3316
3319
  function renderInput$1(state, submitted) {
3317
3320
  const annotation = Option$1.match(state.error, {
3318
3321
  onNone: () => combine(underlined, cyanBright),
3319
- onSome: () => red$1
3322
+ onSome: () => red$2
3320
3323
  });
3321
3324
  const value = state.value === "" ? empty$3 : text$7(`${state.value}`);
3322
3325
  return submitted ? value : annotate(value, annotation);
@@ -3328,8 +3331,8 @@ function renderError$1(state, pointer) {
3328
3331
  onSome: (error) => Arr.match(error.split(NEWLINE_REGEX$1), {
3329
3332
  onEmpty: () => empty$3,
3330
3333
  onNonEmpty: (errorLines) => {
3331
- const annotateLine = (line) => annotate(text$7(line), combine(italicized, red$1));
3332
- const prefix = cat(annotate(pointer, red$1), space$1);
3334
+ const annotateLine = (line) => annotate(text$7(line), combine(italicized, red$2));
3335
+ const prefix = cat(annotate(pointer, red$2), space$1);
3333
3336
  const lines = Arr.map(errorLines, (str) => annotateLine(str));
3334
3337
  return cursorSavePosition.pipe(cat(hardLine), cat(prefix), cat(align(vsep(lines))), cat(cursorRestorePosition));
3335
3338
  }
@@ -3337,7 +3340,7 @@ function renderError$1(state, pointer) {
3337
3340
  });
3338
3341
  }
3339
3342
  function renderOutput$3(state, leadingSymbol, trailingSymbol, options, submitted = false) {
3340
- const annotateLine = (line) => annotate(text$7(line), bold);
3343
+ const annotateLine = (line) => annotate(text$7(line), bold$1);
3341
3344
  const prefix = cat(leadingSymbol, space$1);
3342
3345
  return Arr.match(options.message.split(/\r?\n/), {
3343
3346
  onEmpty: () => hsep([
@@ -3368,7 +3371,7 @@ function renderSubmission$3(nextState, options) {
3368
3371
  return Effect$1.gen(function* () {
3369
3372
  const columns = yield* (yield* Terminal).columns;
3370
3373
  const figures$7 = yield* figures;
3371
- return renderOutput$3(nextState, annotate(figures$7.tick, green), annotate(figures$7.ellipsis, blackBright), options, true).pipe(cat(hardLine), optimize(Deep), render$1({
3374
+ return renderOutput$3(nextState, annotate(figures$7.tick, green$1), annotate(figures$7.ellipsis, blackBright), options, true).pipe(cat(hardLine), optimize(Deep), render$1({
3372
3375
  style: "pretty",
3373
3376
  options: { lineWidth: columns }
3374
3377
  }));
@@ -3463,7 +3466,7 @@ function handleProcessInteger(options) {
3463
3466
  };
3464
3467
  }
3465
3468
  /** @internal */
3466
- const integer = (options) => {
3469
+ const integer$3 = (options) => {
3467
3470
  const opts = {
3468
3471
  min: Number.NEGATIVE_INFINITY,
3469
3472
  max: Number.POSITIVE_INFINITY,
@@ -3551,7 +3554,7 @@ const float = (options) => {
3551
3554
  const renderBeep$2 = /*#__PURE__*/ render$1(beep, { style: "pretty" });
3552
3555
  const NEWLINE_REGEX = /\r?\n/;
3553
3556
  function renderOutput$2(leadingSymbol, trailingSymbol, options) {
3554
- const annotateLine = (line) => annotate(text$7(line), bold);
3557
+ const annotateLine = (line) => annotate(text$7(line), bold$1);
3555
3558
  const prefix = cat(leadingSymbol, space$1);
3556
3559
  return Arr.match(options.message.split(NEWLINE_REGEX), {
3557
3560
  onEmpty: () => hsep([prefix, trailingSymbol]),
@@ -3566,7 +3569,7 @@ function renderChoicePrefix(state, choices, toDisplay, currentIndex, figures) {
3566
3569
  if (currentIndex === toDisplay.startIndex && toDisplay.startIndex > 0) prefix = figures.arrowUp;
3567
3570
  else if (currentIndex === toDisplay.endIndex - 1 && toDisplay.endIndex < choices.length) prefix = figures.arrowDown;
3568
3571
  if (choices[currentIndex].disabled) {
3569
- const annotation = combine(bold, blackBright);
3572
+ const annotation = combine(bold$1, blackBright);
3570
3573
  return state === currentIndex ? figures.pointer.pipe(annotate(annotation), cat(prefix)) : prefix.pipe(cat(space$1));
3571
3574
  }
3572
3575
  return state === currentIndex ? figures.pointer.pipe(annotate(cyanBright), cat(prefix)) : prefix.pipe(cat(space$1));
@@ -3611,7 +3614,7 @@ function renderSubmission$2(state, options) {
3611
3614
  const columns = yield* (yield* Terminal).columns;
3612
3615
  const figures$6 = yield* figures;
3613
3616
  const selected = text$7(options.choices[state].title);
3614
- return renderOutput$2(annotate(figures$6.tick, green), annotate(figures$6.ellipsis, blackBright), options).pipe(cat(space$1), cat(annotate(selected, white$1)), cat(hardLine), render$1({
3617
+ return renderOutput$2(annotate(figures$6.tick, green$1), annotate(figures$6.ellipsis, blackBright), options).pipe(cat(space$1), cat(annotate(selected, white$1)), cat(hardLine), render$1({
3615
3618
  style: "pretty",
3616
3619
  options: { lineWidth: columns }
3617
3620
  }));
@@ -3714,7 +3717,7 @@ function renderInput(nextState, options, submitted) {
3714
3717
  if (nextState.value.length === 0) return blackBright;
3715
3718
  return combine(underlined, cyanBright);
3716
3719
  },
3717
- onSome: () => red$1
3720
+ onSome: () => red$2
3718
3721
  });
3719
3722
  switch (options.type) {
3720
3723
  case "hidden": return empty$3;
@@ -3728,8 +3731,8 @@ function renderError(nextState, pointer) {
3728
3731
  onSome: (error) => Arr.match(error.split(/\r?\n/), {
3729
3732
  onEmpty: () => empty$3,
3730
3733
  onNonEmpty: (errorLines) => {
3731
- const annotateLine = (line) => text$7(line).pipe(annotate(combine(italicized, red$1)));
3732
- const prefix = cat(annotate(pointer, red$1), space$1);
3734
+ const annotateLine = (line) => text$7(line).pipe(annotate(combine(italicized, red$2)));
3735
+ const prefix = cat(annotate(pointer, red$2), space$1);
3733
3736
  const lines = Arr.map(errorLines, (str) => annotateLine(str));
3734
3737
  return cursorSavePosition.pipe(cat(hardLine), cat(prefix), cat(align(vsep(lines))), cat(cursorRestorePosition));
3735
3738
  }
@@ -3737,7 +3740,7 @@ function renderError(nextState, pointer) {
3737
3740
  });
3738
3741
  }
3739
3742
  function renderOutput$1(nextState, leadingSymbol, trailingSymbol, options, submitted = false) {
3740
- const annotateLine = (line) => annotate(text$7(line), bold);
3743
+ const annotateLine = (line) => annotate(text$7(line), bold$1);
3741
3744
  const promptLines = options.message.split(/\r?\n/);
3742
3745
  const prefix = cat(leadingSymbol, space$1);
3743
3746
  if (Arr.isNonEmptyReadonlyArray(promptLines)) {
@@ -3767,7 +3770,7 @@ function renderSubmission$1(state, options) {
3767
3770
  return Effect$1.gen(function* () {
3768
3771
  const columns = yield* (yield* Terminal).columns;
3769
3772
  const figures$4 = yield* figures;
3770
- return renderOutput$1(state, annotate(figures$4.tick, green), annotate(figures$4.ellipsis, blackBright), options, true).pipe(cat(hardLine), optimize(Deep), render$1({
3773
+ return renderOutput$1(state, annotate(figures$4.tick, green$1), annotate(figures$4.ellipsis, blackBright), options, true).pipe(cat(hardLine), optimize(Deep), render$1({
3771
3774
  style: "pretty",
3772
3775
  options: { lineWidth: columns }
3773
3776
  }));
@@ -3909,7 +3912,7 @@ function renderToggle(value, options, submitted = false) {
3909
3912
  ]);
3910
3913
  }
3911
3914
  function renderOutput(toggle, leadingSymbol, trailingSymbol, options) {
3912
- const annotateLine = (line) => annotate(text$7(line), bold);
3915
+ const annotateLine = (line) => annotate(text$7(line), bold$1);
3913
3916
  const promptLines = options.message.split(/\r?\n/);
3914
3917
  const prefix = cat(leadingSymbol, space$1);
3915
3918
  if (Arr.isNonEmptyReadonlyArray(promptLines)) {
@@ -3941,7 +3944,7 @@ function renderSubmission(value, options) {
3941
3944
  const terminal = yield* Terminal;
3942
3945
  const figures$2 = yield* figures;
3943
3946
  const columns = yield* terminal.columns;
3944
- const leadingSymbol = annotate(figures$2.tick, green);
3947
+ const leadingSymbol = annotate(figures$2.tick, green$1);
3945
3948
  const trailingSymbol = annotate(figures$2.ellipsis, blackBright);
3946
3949
  return renderOutput(renderToggle(value, options, true), leadingSymbol, trailingSymbol, options).pipe(cat(hardLine), optimize(Deep), render$1({
3947
3950
  style: "pretty",
@@ -4029,6 +4032,12 @@ const choice = (alternatives) => {
4029
4032
  return op;
4030
4033
  };
4031
4034
  /** @internal */
4035
+ const integer$2 = /*#__PURE__*/ (() => {
4036
+ const op = /*#__PURE__*/ Object.create(proto$5);
4037
+ op._tag = "Integer";
4038
+ return op;
4039
+ })();
4040
+ /** @internal */
4032
4041
  const text$4 = /*#__PURE__*/ (() => {
4033
4042
  const op = /*#__PURE__*/ Object.create(proto$5);
4034
4043
  op._tag = "Text";
@@ -4184,7 +4193,7 @@ const wizardInternal$3 = (self, help) => {
4184
4193
  case "Integer": {
4185
4194
  const primitiveHelp = p("Enter an integer");
4186
4195
  const message = sequence(help, primitiveHelp);
4187
- return integer({ message: toAnsiText(message).trimEnd() }).pipe(map$3((value) => `${value}`));
4196
+ return integer$3({ message: toAnsiText(message).trimEnd() }).pipe(map$3((value) => `${value}`));
4188
4197
  }
4189
4198
  case "Path": {
4190
4199
  const primitiveHelp = p("Select a file system path");
@@ -4689,7 +4698,7 @@ const wizardInternal$2 = (self, config) => {
4689
4698
  case "Variadic": {
4690
4699
  const repeatHelp = p("How many times should this argument should be repeated?");
4691
4700
  const message = pipe(getHelpInternal$2(self), sequence(repeatHelp));
4692
- return integer({
4701
+ return integer$3({
4693
4702
  message: toAnsiText(message).trimEnd(),
4694
4703
  min: getMinSizeInternal$1(self),
4695
4704
  max: getMaxSizeInternal$1(self)
@@ -4874,13 +4883,15 @@ const boolean$1 = (name, options) => {
4874
4883
  const head = Arr.headNonEmpty(negationNames);
4875
4884
  const tail = Arr.tailNonEmpty(negationNames);
4876
4885
  const negationOption = makeSingle(head, tail, boolean$2(Option$1.some(!ifPresent)));
4877
- return withDefault(orElse(option, negationOption), !ifPresent);
4886
+ return withDefault$1(orElse(option, negationOption), !ifPresent);
4878
4887
  }
4879
- return withDefault(option, !ifPresent);
4888
+ return withDefault$1(option, !ifPresent);
4880
4889
  };
4881
4890
  /** @internal */
4882
4891
  const choiceWithValue = (name, choices) => makeSingle(name, Arr.empty(), choice(choices));
4883
4892
  /** @internal */
4893
+ const integer$1 = (name) => makeSingle(name, Arr.empty(), integer$2);
4894
+ /** @internal */
4884
4895
  const none = /*#__PURE__*/ (() => {
4885
4896
  const op = /*#__PURE__*/ Object.create(proto$2);
4886
4897
  op._tag = "Empty";
@@ -4895,7 +4906,7 @@ const getUsage$1 = (self) => getUsageInternal$1(self);
4895
4906
  /** @internal */
4896
4907
  const map$1 = /*#__PURE__*/ dual(2, (self, f) => makeMap(self, (a) => Either$1.right(f(a))));
4897
4908
  /** @internal */
4898
- const optional$1 = (self) => withDefault(map$1(self, Option$1.some), Option$1.none());
4909
+ const optional$1 = (self) => withDefault$1(map$1(self, Option$1.some), Option$1.none());
4899
4910
  /** @internal */
4900
4911
  const orElse = /*#__PURE__*/ dual(2, (self, that) => orElseEither(self, that).pipe(map$1(Either$1.merge)));
4901
4912
  /** @internal */
@@ -4917,7 +4928,7 @@ const withAlias = /*#__PURE__*/ dual(2, (self, alias) => modifySingle(self, (sin
4917
4928
  return makeSingle(single.name, aliases, single.primitiveType, single.description, single.pseudoName);
4918
4929
  }));
4919
4930
  /** @internal */
4920
- const withDefault = /*#__PURE__*/ dual(2, (self, fallback) => makeWithDefault(self, fallback));
4931
+ const withDefault$1 = /*#__PURE__*/ dual(2, (self, fallback) => makeWithDefault(self, fallback));
4921
4932
  /** @internal */
4922
4933
  const withDescription = /*#__PURE__*/ dual(2, (self, desc) => modifySingle(self, (single) => {
4923
4934
  const description = sequence(single.description, p(desc));
@@ -5305,7 +5316,7 @@ const wizardInternal$1 = (self, config) => {
5305
5316
  case "Variadic": {
5306
5317
  const repeatHelp = p("How many times should this argument be repeated?");
5307
5318
  const message = pipe(getHelpInternal$1(self), sequence(repeatHelp));
5308
- return integer({
5319
+ return integer$3({
5309
5320
  message: toAnsiText(message).trimEnd(),
5310
5321
  min: getMinSizeInternal(self),
5311
5322
  max: getMaxSizeInternal(self)
@@ -5919,7 +5930,7 @@ const completionsOptions = /*#__PURE__*/ choiceWithValue("completions", [
5919
5930
  ["zsh", "zsh"]
5920
5931
  ]).pipe(optional$1, /*#__PURE__*/ withDescription("Generate a completion script for a specific shell."));
5921
5932
  /** @internal */
5922
- const logLevelOptions = /*#__PURE__*/ choiceWithValue("log-level", LogLevel.allLevels.map((level) => [level._tag.toLowerCase(), level])).pipe(optional$1, /*#__PURE__*/ withDescription("Sets the minimum log level for a command."));
5933
+ const logLevelOptions = /*#__PURE__*/ choiceWithValue("log-level", LogLevel$1.allLevels.map((level) => [level._tag.toLowerCase(), level])).pipe(optional$1, /*#__PURE__*/ withDescription("Sets the minimum log level for a command."));
5923
5934
  /** @internal */
5924
5935
  const helpOptions = /*#__PURE__*/ boolean$1("help").pipe(/*#__PURE__*/ withAlias("h"), /*#__PURE__*/ withDescription("Show the help documentation for a command."));
5925
5936
  /** @internal */
@@ -5957,6 +5968,11 @@ const boolean = boolean$1;
5957
5968
  * @since 1.0.0
5958
5969
  * @category constructors
5959
5970
  */
5971
+ const integer = integer$1;
5972
+ /**
5973
+ * @since 1.0.0
5974
+ * @category constructors
5975
+ */
5960
5976
  const text = text$1;
5961
5977
  /**
5962
5978
  * @since 1.0.0
@@ -5974,6 +5990,11 @@ const optional = optional$1;
5974
5990
  * @category combinators
5975
5991
  */
5976
5992
  const repeated = repeated$1;
5993
+ /**
5994
+ * @since 1.0.0
5995
+ * @category combinators
5996
+ */
5997
+ const withDefault = withDefault$1;
5977
5998
  //#endregion
5978
5999
  //#region ../../node_modules/.bun/@effect+cli@0.77.2+74e017304edf4bf0/node_modules/@effect/cli/dist/esm/internal/commandDirective.js
5979
6000
  /** @internal */
@@ -6294,7 +6315,7 @@ const wizardInternal = (self, prefix, config) => {
6294
6315
  case "GetUserInput":
6295
6316
  case "Standard": return Effect$1.gen(function* () {
6296
6317
  const logCurrentCommand = Ref.get(commandLineRef).pipe(Effect$1.flatMap((commandLine) => {
6297
- const currentCommand = p(pipe(strong(highlight("COMMAND:", cyan)), concat$1(space), concat$1(highlight(Arr.join(commandLine, " "), magenta))));
6318
+ const currentCommand = p(pipe(strong(highlight("COMMAND:", cyan$1)), concat$1(space), concat$1(highlight(Arr.join(commandLine, " "), magenta))));
6298
6319
  return Console.log(toAnsiText(currentCommand));
6299
6320
  }));
6300
6321
  if (isStandard(self)) {
@@ -6634,7 +6655,7 @@ const handleBuiltInOption = (self, executable, args, builtIn, execute, config, o
6634
6655
  filteredArgs.push(args[i]);
6635
6656
  }
6636
6657
  const nextArgs = Arr.appendAll(baseArgs, filteredArgs);
6637
- return run$2(self, nextArgs, execute).pipe(Logger.withMinimumLogLevel(builtIn.level));
6658
+ return run$2(self, nextArgs, execute).pipe(Logger$1.withMinimumLogLevel(builtIn.level));
6638
6659
  }
6639
6660
  case "ShowHelp": {
6640
6661
  const banner = h1(code(self.name));
@@ -6740,7 +6761,7 @@ const renderWizardArgs = (args) => {
6740
6761
  return blocks([
6741
6762
  p(strong(code("Wizard Mode Complete!"))),
6742
6763
  p(executeMsg),
6743
- p(concat$1(text$6(" "), highlight(params, cyan)))
6764
+ p(concat$1(text$6(" "), highlight(params, cyan$1)))
6744
6765
  ]);
6745
6766
  };
6746
6767
  const isLogLevelArg = (arg) => {
@@ -8050,6 +8071,8 @@ const layer = /*#__PURE__*/ pipe(/*#__PURE__*/ Layer.mergeAll(layer$3, layer$5,
8050
8071
  //#endregion
8051
8072
  //#region ../cli/src/dashboard-reports.ts
8052
8073
  const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
8074
+ /** Written into a run directory by `evalkit tidy`; deleting it restores the run. */
8075
+ const ARCHIVE_MARKER = "archived.json";
8053
8076
  function isMissing(error) {
8054
8077
  return error.code === "ENOENT";
8055
8078
  }
@@ -8086,15 +8109,35 @@ function fileContentType(file) {
8086
8109
  ".js": "text/javascript"
8087
8110
  }[extname(file)] ?? "application/octet-stream";
8088
8111
  }
8089
- /** Read saved local reports. A missing summary means "running"; malformed reports are errors. */
8112
+ /**
8113
+ * Read saved local reports. A missing summary means "running"; malformed reports are errors.
8114
+ * Runs written by an older EvalKit (manifest `schemaVersion` < 2) are skipped and counted.
8115
+ */
8090
8116
  function createDashboardReportReader(root, registry) {
8117
+ const legacyRuns = /* @__PURE__ */ new Set();
8091
8118
  async function listRuns() {
8092
8119
  const entries = await ifPresent(() => readdir(root, { withFileTypes: true }));
8093
8120
  if (!entries) return [];
8094
8121
  const ids = entries.filter((entry) => entry.isDirectory() && UUID_PATTERN.test(entry.name)).map((entry) => entry.name);
8095
8122
  return (await Promise.all(ids.map(readRun))).filter((run) => run !== void 0);
8096
8123
  }
8124
+ /** True for a run from a report format this version cannot read; never for corrupt JSON. */
8125
+ async function isLegacyRun(id) {
8126
+ const text = await ifPresent(() => readFile(resolve(root, id, "manifest.json"), "utf8"));
8127
+ if (text === void 0) return false;
8128
+ let version;
8129
+ try {
8130
+ version = JSON.parse(text).schemaVersion;
8131
+ } catch {
8132
+ return false;
8133
+ }
8134
+ return typeof version === "number" && version < 2;
8135
+ }
8097
8136
  async function readRun(id) {
8137
+ if (await isLegacyRun(id)) {
8138
+ legacyRuns.add(id);
8139
+ return;
8140
+ }
8098
8141
  const manifest = await ifPresent(() => readRunManifest(root, id));
8099
8142
  if (!manifest) return void 0;
8100
8143
  const summary = await ifPresent(() => readRunSummary(root, id));
@@ -8103,6 +8146,8 @@ function createDashboardReportReader(root, registry) {
8103
8146
  const scores = trials.flatMap((trial) => trial?.scoring?.overall === void 0 ? [] : [trial.scoring.overall]);
8104
8147
  return {
8105
8148
  id,
8149
+ ...await ifPresent(() => stat(resolve(root, id, "archived.json"))) ? { archived: true } : {},
8150
+ ...manifest.source ? { source: manifest.source } : {},
8106
8151
  evalId: manifest.evalId,
8107
8152
  ...manifest.suiteId ? { suiteId: manifest.suiteId } : {},
8108
8153
  ...manifest.matrix ? {
@@ -8171,6 +8216,8 @@ function createDashboardReportReader(root, registry) {
8171
8216
  }
8172
8217
  return {
8173
8218
  listRuns,
8219
+ /** Run IDs skipped by the last `listRuns()` because an older EvalKit wrote them. */
8220
+ legacyRunIds: () => [...legacyRuns],
8174
8221
  listTrials,
8175
8222
  trialDetail,
8176
8223
  listArtifacts,
@@ -8205,67 +8252,172 @@ async function walkArtifacts(root) {
8205
8252
  //#region ../cli/src/dashboard-build.ts
8206
8253
  /** Marks a directory as EvalKit-owned so a rebuild may safely replace it. */
8207
8254
  const MARKER = ".evalkit-dashboard";
8208
- /**
8209
- * Write a read-only static dashboard: the bundled UI plus JSON snapshots of
8210
- * the catalog and local reports, and each trial's candidate artifact files.
8211
- * Evaluator workspaces are never copied; reports may still contain sensitive
8212
- * messages, scores, or evidence, so review the output before sharing it.
8213
- */
8214
- async function buildDashboard(options) {
8255
+ const quietStep = (_label, work) => work(() => {});
8256
+ async function buildDashboard(options, step = quietStep) {
8215
8257
  const { registry, reportRoot, dashboardRoot, outDir } = options;
8216
- await prepareOutDir(outDir);
8217
- await cp(dashboardRoot, outDir, { recursive: true });
8218
- await writeFile(resolve(outDir, MARKER), "Generated by `evalkit build`.\n");
8219
- await writeFile(resolve(outDir, "index.html"), staticIndex(await readFile(resolve(dashboardRoot, "index.html"), "utf8")));
8258
+ const template = await readFile(resolve(dashboardRoot, "index.html"), "utf8");
8220
8259
  const data = resolve(outDir, "data");
8221
8260
  const write = async (path, value) => {
8222
8261
  const file = resolve(data, path);
8223
8262
  await mkdir(dirname(file), { recursive: true });
8224
8263
  await writeFile(file, `${JSON.stringify(value)}\n`);
8225
8264
  };
8265
+ await step("Dashboard assets", async () => {
8266
+ await prepareOutDir(outDir);
8267
+ await cp(dashboardRoot, outDir, { recursive: true });
8268
+ await writeFile(resolve(outDir, MARKER), "Generated by `evalkit build`.\n");
8269
+ await writeFile(resolve(outDir, "index.html"), dashboardIndex(template, { base: "./" }));
8270
+ }, () => "read-only UI with relative assets");
8271
+ await step("Eval catalog", async () => {
8272
+ await write("catalog.json", { evals: registry.catalog() });
8273
+ await write("evals.json", { evals: registry.metadata() });
8274
+ await write("suites.json", { suites: registry.suiteMetadata() });
8275
+ return {
8276
+ evals: registry.evals.length,
8277
+ suites: registry.suites.length
8278
+ };
8279
+ }, ({ evals, suites }) => `${count(evals, "eval")} · ${count(suites, "suite")}`);
8226
8280
  const reports = createDashboardReportReader(reportRoot, registry);
8227
- const runs = await reports.listRuns();
8228
- await write("catalog.json", { evals: registry.catalog() });
8229
- await write("evals.json", { evals: registry.metadata() });
8230
- await write("suites.json", { suites: registry.suiteMetadata() });
8231
- await write("runs.json", { runs });
8232
- let trials = 0;
8281
+ const snapshot = await step("Run reports", async (progress) => {
8282
+ const all = await reports.listRuns();
8283
+ const runs = all.filter((run) => !run.archived);
8284
+ await write("runs.json", { runs });
8285
+ let trials = 0;
8286
+ let files = 0;
8287
+ const trialIds = [];
8288
+ for (const [index, run] of runs.entries()) {
8289
+ progress(`run ${index + 1}/${runs.length} · ${run.id.slice(0, 8)}`);
8290
+ const written = await snapshotRun(reports, run.id, directorySink(resolve(data, "runs", run.id)));
8291
+ trials += written.trialIds.length;
8292
+ files += written.files;
8293
+ trialIds.push(...written.trialIds);
8294
+ }
8295
+ return {
8296
+ runs: runs.length,
8297
+ trials,
8298
+ files,
8299
+ skipped: reports.legacyRunIds().length,
8300
+ archived: all.length - runs.length,
8301
+ runIds: runs.map((run) => run.id),
8302
+ trialIds
8303
+ };
8304
+ }, ({ runs, trials, files, skipped, archived }) => `${count(runs, "run")} · ${count(trials, "trial")} · ${count(files, "candidate file")}` + (archived ? ` · ${archived} archived excluded` : "") + (skipped ? ` · ${skipped} older-format skipped` : ""));
8305
+ await step("Route pages", async () => {
8306
+ const routes = staticRoutes({
8307
+ suites: registry.suites.map(authoringId),
8308
+ evals: registry.catalog().map((entry) => entry.path),
8309
+ runs: snapshot.runIds,
8310
+ trials: snapshot.trialIds
8311
+ });
8312
+ for (const route of routes) {
8313
+ const file = resolve(outDir, `${route}.html`);
8314
+ await mkdir(dirname(file), { recursive: true });
8315
+ await writeFile(file, dashboardIndex(template, { base: baseFor(route) }));
8316
+ }
8317
+ await writeFile(resolve(outDir, "404.html"), dashboardIndex(template, { base: "/" }));
8318
+ return routes.length;
8319
+ }, (pages) => `${count(pages, "page")} with clean /path URLs`);
8320
+ const { runIds: _runs, trialIds: _trials, ...summary } = snapshot;
8321
+ return {
8322
+ outDir,
8323
+ ...summary
8324
+ };
8325
+ }
8326
+ /**
8327
+ * Every dashboard URL known at build time, as file paths without `.html`.
8328
+ * Static hosts serve `/runs/abc` from `runs/abc.html`, so links and reloads
8329
+ * work without SPA rewrite rules. IDs that cannot be a safe file name are
8330
+ * skipped; those pages still open by navigating inside the app.
8331
+ */
8332
+ function staticRoutes(ids) {
8333
+ const safe = (id) => id.length > 0 && !/[/\\\0]/.test(id) && id !== "." && id !== "..";
8334
+ const under = (prefix, values) => values.filter(safe).map((id) => `${prefix}/${id}`);
8335
+ return [
8336
+ "suites",
8337
+ "agents",
8338
+ "fixtures",
8339
+ "runs",
8340
+ ...under("suites", ids.suites),
8341
+ ...under("evals", ids.evals),
8342
+ ...under("runs", ids.runs),
8343
+ ...under("trials", ids.trials),
8344
+ ...under("workspaces", ids.trials)
8345
+ ];
8346
+ }
8347
+ /** `runs/abc.html` is served at `<mount>/runs/abc`; its base is one level up. */
8348
+ function baseFor(route) {
8349
+ const depth = route.split("/").length - 1;
8350
+ return depth ? "../".repeat(depth) : "./";
8351
+ }
8352
+ function count(value, noun) {
8353
+ return `${value} ${noun}${value === 1 ? "" : "s"}`;
8354
+ }
8355
+ /** Writes a run's snapshot below `root` (static builds). */
8356
+ function directorySink(root) {
8357
+ const target = (path) => {
8358
+ const file = resolve(root, path);
8359
+ if (!file.startsWith(`${resolve(root)}${sep}`)) throw new Error(`Invalid snapshot path: ${path}`);
8360
+ return file;
8361
+ };
8362
+ return {
8363
+ async json(path, value) {
8364
+ const file = target(path);
8365
+ await mkdir(dirname(file), { recursive: true });
8366
+ await writeFile(file, `${JSON.stringify(value)}\n`);
8367
+ },
8368
+ async bytes(path, data) {
8369
+ const file = target(path);
8370
+ await mkdir(dirname(file), { recursive: true });
8371
+ await writeFile(file, data);
8372
+ }
8373
+ };
8374
+ }
8375
+ /**
8376
+ * One run's snapshot, shared by `evalkit build` (files on disk) and
8377
+ * `evalkit share` (an upload bundle): `trials.json`, then per trial
8378
+ * `detail.json`, `events.json`, `artifacts.json`, `workspace.json`, and the
8379
+ * candidate files under `workspace/`. Evaluator workspaces are never read.
8380
+ */
8381
+ async function snapshotRun(reports, runId, sink, options = {}) {
8382
+ const trials = await reports.listTrials(runId);
8383
+ await sink.json("trials.json", { trials });
8233
8384
  let files = 0;
8234
- for (const run of runs) {
8235
- const runTrials = await reports.listTrials(run.id);
8236
- await write(`runs/${run.id}/trials.json`, { trials: runTrials });
8237
- for (const trial of runTrials) {
8238
- trials += 1;
8239
- const base = `runs/${run.id}/trials/${trial.id}`;
8240
- await write(`${base}/detail.json`, await reports.trialDetail(run.id, trial.id));
8241
- await write(`${base}/events.json`, { events: await reports.trajectory(run.id, trial.id) });
8242
- await write(`${base}/artifacts.json`, { artifacts: await reports.listArtifacts(run.id, trial.id) });
8243
- const workspace = await reports.listCandidateWorkspace(run.id, trial.id);
8244
- await write(`${base}/workspace.json`, workspace);
8245
- for (const entry of workspace) {
8246
- if (entry.kind !== "file") continue;
8247
- const response = await reports.candidateFile(run.id, trial.id, entry.path);
8248
- const target = resolve(data, base, "workspace", entry.path);
8249
- const root = resolve(data, base, "workspace");
8250
- if (!target.startsWith(`${root}${sep}`)) throw new Error(`Invalid workspace path: ${entry.path}`);
8251
- await mkdir(dirname(target), { recursive: true });
8252
- await writeFile(target, new Uint8Array(await response.arrayBuffer()));
8253
- files += 1;
8254
- }
8385
+ for (const trial of trials) {
8386
+ const base = `trials/${trial.id}`;
8387
+ await sink.json(`${base}/detail.json`, await reports.trialDetail(runId, trial.id));
8388
+ await sink.json(`${base}/events.json`, { events: await reports.trajectory(runId, trial.id) });
8389
+ await sink.json(`${base}/artifacts.json`, { artifacts: await reports.listArtifacts(runId, trial.id) });
8390
+ const workspace = await reports.listCandidateWorkspace(runId, trial.id);
8391
+ await sink.json(`${base}/workspace.json`, workspace);
8392
+ if (options.workspaceFiles === false) continue;
8393
+ for (const entry of workspace) {
8394
+ if (entry.kind !== "file") continue;
8395
+ const response = await reports.candidateFile(runId, trial.id, entry.path);
8396
+ await sink.bytes(`${base}/workspace/${entry.path}`, new Uint8Array(await response.arrayBuffer()));
8397
+ files += 1;
8255
8398
  }
8256
8399
  }
8257
8400
  return {
8258
- outDir,
8259
- runs: runs.length,
8260
- trials,
8401
+ trialIds: trials.map((trial) => trial.id),
8261
8402
  files
8262
8403
  };
8263
8404
  }
8264
- /** Use relative asset URLs and select the static API in the browser bundle. */
8265
- function staticIndex(html) {
8266
- const result = html.replace("<html lang=\"en\">", "<html lang=\"en\" data-evalkit-snapshot=\"static\">").replaceAll("\"/assets/", "\"./assets/");
8267
- if (!result.includes("data-evalkit-snapshot=\"static\"")) throw new Error("Bundled dashboard index.html has an unexpected shape");
8268
- return result;
8405
+ /** Total size and file count of a directory, for the build summary. */
8406
+ async function directorySize(root) {
8407
+ let bytes = 0;
8408
+ let files = 0;
8409
+ for (const entry of await readdir(root, {
8410
+ withFileTypes: true,
8411
+ recursive: true
8412
+ })) {
8413
+ if (!entry.isFile()) continue;
8414
+ bytes += (await stat(resolve(entry.parentPath, entry.name))).size;
8415
+ files += 1;
8416
+ }
8417
+ return {
8418
+ bytes,
8419
+ files
8420
+ };
8269
8421
  }
8270
8422
  async function prepareOutDir(outDir) {
8271
8423
  let entries;
@@ -8460,6 +8612,231 @@ async function loadProject(cwd, configPath) {
8460
8612
  };
8461
8613
  }
8462
8614
  //#endregion
8615
+ //#region ../cli/src/terminal.ts
8616
+ /** Human CLI output. Colors and live progress only appear on an interactive terminal. */
8617
+ const interactive = Boolean(process.stdout.isTTY);
8618
+ const colored = interactive && !process.env.NO_COLOR && process.env.TERM !== "dumb";
8619
+ const paint = (code, reset) => (text) => colored ? `\x1b[${code}m${text}\x1b[${reset}m` : text;
8620
+ const bold = paint(1, 22);
8621
+ const dim = paint(2, 22);
8622
+ const red = paint(31, 39);
8623
+ const green = paint(32, 39);
8624
+ const yellow = paint(33, 39);
8625
+ const cyan = paint(36, 39);
8626
+ function banner(command, version) {
8627
+ console.log(`\n ${bold(cyan("EvalKit"))} ${bold(command)} ${dim(`v${version}`)}\n`);
8628
+ }
8629
+ function duration(ms) {
8630
+ return ms < 1e3 ? `${Math.round(ms)}ms` : `${(ms / 1e3).toFixed(2)}s`;
8631
+ }
8632
+ function bytes(size) {
8633
+ if (size < 1024) return `${size} B`;
8634
+ if (size < 1024 ** 2) return `${(size / 1024).toFixed(1)} kB`;
8635
+ return `${(size / 1024 ** 2).toFixed(1)} MB`;
8636
+ }
8637
+ function plural(count, noun) {
8638
+ return `${count} ${noun}${count === 1 ? "" : "s"}`;
8639
+ }
8640
+ /** Times one build step, showing a live line while it runs. */
8641
+ async function step(label, work, describe) {
8642
+ const started = performance.now();
8643
+ const progress = (detail) => {
8644
+ if (interactive) process.stdout.write(`\r\x1b[2K ${cyan("◇")} ${label.padEnd(20)} ${dim(detail)}`);
8645
+ };
8646
+ progress("…");
8647
+ try {
8648
+ const value = await work(progress);
8649
+ const elapsed = duration(performance.now() - started);
8650
+ if (interactive) process.stdout.write("\r\x1B[2K");
8651
+ console.log(` ${green("✓")} ${label.padEnd(20)} ${describe(value)} ${dim(elapsed)}`);
8652
+ return value;
8653
+ } catch (error) {
8654
+ if (interactive) process.stdout.write("\r\x1B[2K");
8655
+ console.log(` ${red("✗")} ${label}`);
8656
+ throw error;
8657
+ }
8658
+ }
8659
+ function clock() {
8660
+ return dim((/* @__PURE__ */ new Date()).toTimeString().slice(0, 8));
8661
+ }
8662
+ function cellLabel(name, parameters) {
8663
+ const shown = Object.entries(parameters).map(([key, value]) => `${key}=${typeof value === "string" ? value : JSON.stringify(value)}`).join(" ");
8664
+ return shown ? `${bold(name)} ${dim(shown)}` : bold(name);
8665
+ }
8666
+ function passed(result) {
8667
+ return result.status === "completed" && (result.aggregateScoring?.passRate === 1 || Boolean(result.scoring?.passed));
8668
+ }
8669
+ /** Live log for runs launched from the `evalkit dev` dashboard. */
8670
+ const runLog = {
8671
+ started(name, parameters = {}, suiteId) {
8672
+ const suite = suiteId ? dim(` (suite ${suiteId})`) : "";
8673
+ console.log(`${clock()} ${cyan("▶ run ")} ${cellLabel(name, parameters)}${suite}`);
8674
+ },
8675
+ finished(name, parameters, result, ms) {
8676
+ const ok = passed(result);
8677
+ const trials = result.trials?.length ?? 1;
8678
+ const passedTrials = result.aggregateScoring?.passed ?? (ok ? trials : 0);
8679
+ const status = result.status === "completed" ? ok ? green("✓ pass") : red("✗ fail") : red(`✗ ${result.status}`);
8680
+ console.log(`${clock()} ${status} ${cellLabel(name, parameters)} ${dim(`${passedTrials}/${trials} trials · ${duration(ms)}`)}`);
8681
+ console.log(` ${dim(`report ${result.reportLocation}`)}`);
8682
+ },
8683
+ cell(cell, result, ms) {
8684
+ runLog.finished(cell.eval.name ?? authoringId(cell.eval), cell.parameters, result, ms);
8685
+ },
8686
+ failed(name, error) {
8687
+ console.log(`${clock()} ${red("✗ error")} ${bold(name)} ${error instanceof Error ? error.message : String(error)}`);
8688
+ }
8689
+ };
8690
+ /**
8691
+ * The runner logs structured Effect lines; the CLI prints its own summaries, so
8692
+ * only warnings show by default. `EVALKIT_LOG_LEVEL=info` (or `debug`) restores them.
8693
+ */
8694
+ function runnerLogs(effect) {
8695
+ const requested = process.env.EVALKIT_LOG_LEVEL?.toLowerCase();
8696
+ const level = LogLevel.allLevels.find((item) => item.label.toLowerCase() === requested) ?? LogLevel.Warning;
8697
+ return effect.pipe(Logger.withMinimumLogLevel(level));
8698
+ }
8699
+ /** True when a person can answer a prompt (not CI, pipes, or redirected input). */
8700
+ function canPrompt() {
8701
+ return Boolean(process.stdin.isTTY && process.stdout.isTTY);
8702
+ }
8703
+ async function confirm(question) {
8704
+ const { createInterface } = await import("node:readline/promises");
8705
+ const prompt = createInterface({
8706
+ input: process.stdin,
8707
+ output: process.stdout
8708
+ });
8709
+ try {
8710
+ const answer = await prompt.question(` ${cyan("?")} ${question} ${dim("(y/N)")} `);
8711
+ return /^y(es)?$/i.test(answer.trim());
8712
+ } catch {
8713
+ console.log("");
8714
+ return false;
8715
+ } finally {
8716
+ prompt.close();
8717
+ }
8718
+ }
8719
+ //#endregion
8720
+ //#region ../cli/src/source.ts
8721
+ const exec = promisify(execFile);
8722
+ /**
8723
+ * Normalize a git remote to a stable repository key: lowercase host, no
8724
+ * credentials, port, scheme, `.git`, or trailing slash.
8725
+ *
8726
+ * git@GitLab.example.com:group/sub/repo.git → gitlab.example.com/group/sub/repo
8727
+ * https://user:token@github.com/o/r/ → github.com/o/r
8728
+ * /srv/repos/thing.git (no host) → local/thing
8729
+ */
8730
+ function normalizeRepoUrl(url) {
8731
+ const trimmed = url.trim().replace(/\/+$/, "").replace(/\.git$/, "");
8732
+ const scp = /^(?:[^@/]+@)?([^:/]+):(?!\/)(.+)$/.exec(trimmed);
8733
+ if (scp && !/^[a-z]+:\/\//i.test(trimmed)) return `${scp[1].toLowerCase()}/${scp[2].replace(/^\/+/, "")}`;
8734
+ try {
8735
+ const parsed = new URL(trimmed);
8736
+ if (parsed.protocol !== "file:" && parsed.hostname) return `${parsed.hostname.toLowerCase()}${parsed.pathname.replace(/\/+$/, "")}`;
8737
+ } catch {}
8738
+ return `local/${basename(trimmed)}`;
8739
+ }
8740
+ async function git(cwd, ...args) {
8741
+ try {
8742
+ const { stdout } = await exec("git", [
8743
+ "-C",
8744
+ cwd,
8745
+ ...args
8746
+ ], { timeout: 1e4 });
8747
+ return stdout.trim() || void 0;
8748
+ } catch {
8749
+ return;
8750
+ }
8751
+ }
8752
+ const posix = (path) => path.split(sep).join("/");
8753
+ const projectPath = (root, projectRoot) => root ? posix(relative(root, projectRoot)) : "";
8754
+ function ciSource(env, projectRoot) {
8755
+ if (env.EVALKIT_SOURCE_REPO && env.EVALKIT_SOURCE_COMMIT) return {
8756
+ repo: normalizeRepoUrl(env.EVALKIT_SOURCE_REPO),
8757
+ project: env.EVALKIT_SOURCE_PROJECT ?? "",
8758
+ commit: env.EVALKIT_SOURCE_COMMIT,
8759
+ ...env.EVALKIT_SOURCE_REF ? { ref: env.EVALKIT_SOURCE_REF } : {},
8760
+ ...env.EVALKIT_SOURCE_COMMITTED_AT ? { committedAt: env.EVALKIT_SOURCE_COMMITTED_AT } : {}
8761
+ };
8762
+ if (env.GITLAB_CI === "true" && env.CI_PROJECT_URL && env.CI_COMMIT_SHA) return {
8763
+ repo: normalizeRepoUrl(env.CI_PROJECT_URL),
8764
+ project: projectPath(env.CI_PROJECT_DIR, projectRoot),
8765
+ commit: env.CI_COMMIT_SHA,
8766
+ ...env.CI_MERGE_REQUEST_SOURCE_BRANCH_NAME ?? env.CI_COMMIT_REF_NAME ? { ref: env.CI_MERGE_REQUEST_SOURCE_BRANCH_NAME ?? env.CI_COMMIT_REF_NAME } : {},
8767
+ ...env.CI_COMMIT_TIMESTAMP ? { committedAt: env.CI_COMMIT_TIMESTAMP } : {},
8768
+ ci: {
8769
+ provider: "gitlab",
8770
+ ...env.CI_PIPELINE_ID ? { pipeline: env.CI_PIPELINE_ID } : {},
8771
+ ...env.CI_JOB_ID ? { job: env.CI_JOB_ID } : {},
8772
+ ...env.CI_JOB_URL ? { url: env.CI_JOB_URL } : {}
8773
+ }
8774
+ };
8775
+ if (env.GITHUB_ACTIONS === "true" && env.GITHUB_REPOSITORY && env.GITHUB_SHA) {
8776
+ const server = env.GITHUB_SERVER_URL ?? "https://github.com";
8777
+ return {
8778
+ repo: normalizeRepoUrl(`${server}/${env.GITHUB_REPOSITORY}`),
8779
+ project: projectPath(env.GITHUB_WORKSPACE, projectRoot),
8780
+ commit: env.GITHUB_SHA,
8781
+ ...env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME ? { ref: env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME } : {},
8782
+ ci: {
8783
+ provider: "github",
8784
+ ...env.GITHUB_RUN_ID ? { pipeline: env.GITHUB_RUN_ID } : {},
8785
+ ...env.GITHUB_JOB ? { job: env.GITHUB_JOB } : {},
8786
+ ...env.GITHUB_RUN_ID ? { url: `${server}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}` } : {}
8787
+ }
8788
+ };
8789
+ }
8790
+ }
8791
+ /**
8792
+ * Determine which repository and commit this project's eval code comes from.
8793
+ * Order: EVALKIT_SOURCE_* variables, GitLab/GitHub CI variables, then git.
8794
+ * Returns undefined outside git with no CI variables ("untracked" results).
8795
+ *
8796
+ * `outputDirs` (results, sandbox, snapshot) never make the tree dirty, even
8797
+ * when a project forgot to ignore them.
8798
+ */
8799
+ async function detectSource(givenRoot, options = {}) {
8800
+ const env = options.env ?? process.env;
8801
+ const projectRoot = await realpath(givenRoot).catch(() => givenRoot);
8802
+ const top = await git(projectRoot, "rev-parse", "--show-toplevel");
8803
+ const excludes = top ? (options.outputDirs ?? []).map((dir) => join(projectRoot, relative(givenRoot, dir))).map((dir) => posix(relative(top, dir))).filter((dir) => dir && !dir.startsWith("..")).map((dir) => `:(exclude)${dir}`) : [];
8804
+ const dirty = top ? Boolean(await git(top, "status", "--porcelain", "--", ".", ...excludes)) : false;
8805
+ const fromCi = ciSource(env, projectRoot);
8806
+ if (fromCi) return {
8807
+ ...fromCi,
8808
+ ...top && !env.EVALKIT_SOURCE_PROJECT ? { project: projectPath(top, projectRoot) } : {},
8809
+ dirty
8810
+ };
8811
+ if (!top) return void 0;
8812
+ const commit = await git(top, "rev-parse", "HEAD");
8813
+ if (!commit) return void 0;
8814
+ const remote = await git(top, "config", "--get", "remote.origin.url") ?? (await git(top, "remote"))?.split("\n").map((name) => name.trim()).find(Boolean);
8815
+ const remoteUrl = remote && !remote.includes("/") && !remote.includes(":") ? await git(top, "config", "--get", `remote.${remote}.url`) : remote;
8816
+ const ref = await git(top, "symbolic-ref", "--short", "-q", "HEAD");
8817
+ const committedAt = await git(top, "show", "-s", "--format=%cI", "HEAD");
8818
+ return {
8819
+ repo: normalizeRepoUrl(remoteUrl ?? top),
8820
+ project: projectPath(top, projectRoot),
8821
+ commit,
8822
+ ...ref ? { ref } : {},
8823
+ ...committedAt ? { committedAt } : {},
8824
+ dirty
8825
+ };
8826
+ }
8827
+ /** The project's evals and suites, saved with each run (`catalog.json`). */
8828
+ function runCatalog(registry) {
8829
+ return {
8830
+ catalog: registry.catalog(),
8831
+ evals: registry.metadata(),
8832
+ suites: registry.suiteMetadata()
8833
+ };
8834
+ }
8835
+ /** `abc1234` or `abc1234+dirty`. */
8836
+ function shortSource(source) {
8837
+ return `${source.commit.slice(0, 7)}${source.dirty ? "+dirty" : ""}`;
8838
+ }
8839
+ //#endregion
8463
8840
  //#region ../cli/src/run-command.ts
8464
8841
  function parseRunArgs(args) {
8465
8842
  const { values, positionals } = parseArgs({
@@ -8606,15 +8983,24 @@ async function runProjectCommand(args, cwd = process.cwd(), parsed = parseRunArg
8606
8983
  return;
8607
8984
  }
8608
8985
  if (cells > (project.config.execution?.maxCells ?? 100) && !values.all) throw new Error(`Selected ${cells} cells. Narrow the selection, inspect --dry-run, or pass --all.`);
8609
- if (!values.json) console.log(`Running ${cells} cell(s) from ${plan.matrix}`);
8610
- if ((await Effect.runPromise(runMatrix(matrix, {
8986
+ const reportDir = resolve(project.root, project.config.reportDir ?? "_evalkit-results");
8987
+ const sandboxDir = resolve(project.root, project.config.sandboxDir ?? "_evalkit-sandbox");
8988
+ const source = await detectSource(project.root, { outputDirs: [
8989
+ reportDir,
8990
+ sandboxDir,
8991
+ resolve(project.root, "_evalkit-dashboard")
8992
+ ] });
8993
+ if (!values.json) console.log(`Running ${cells} cell(s) from ${plan.matrix}` + (source ? ` at ${shortSource(source)}${source.ref ? ` (${source.ref})` : ""}` : " (untracked: not in a git repository)"));
8994
+ if ((await Effect.runPromise(runnerLogs(runMatrix(matrix, {
8611
8995
  selection,
8612
8996
  concurrency: parsed.concurrency ?? project.config.execution?.concurrency ?? 4,
8613
8997
  trials,
8614
8998
  ...suiteId ? { suiteId } : {},
8615
8999
  ...values.local ? { runtime: "local" } : {},
8616
- report: localReportStore(resolve(project.root, project.config.reportDir ?? "_evalkit-results")),
8617
- workspaceRoot: resolve(project.root, project.config.sandboxDir ?? "_evalkit-sandbox"),
9000
+ ...source ? { source } : {},
9001
+ catalog: runCatalog(registry),
9002
+ report: localReportStore(reportDir),
9003
+ workspaceRoot: sandboxDir,
8618
9004
  onResult: (cell, result) => {
8619
9005
  if (values.json) console.log(JSON.stringify({
8620
9006
  cell: {
@@ -8633,7 +9019,211 @@ async function runProjectCommand(args, cwd = process.cwd(), parsed = parseRunArg
8633
9019
  }
8634
9020
  }
8635
9021
  }
8636
- }))).failed) process.exitCode = 1;
9022
+ })))).failed) process.exitCode = 1;
9023
+ }
9024
+ //#endregion
9025
+ //#region ../cli/src/tidy.ts
9026
+ /**
9027
+ * Every eval the project can run: standalone, in suites, or only in matrices.
9028
+ * A run is archived only when its eval is in none of these.
9029
+ */
9030
+ function projectEvalIds(registry) {
9031
+ return new Set([
9032
+ ...registry.evals,
9033
+ ...registry.suites.flatMap((suite) => suite.evals),
9034
+ ...registry.matrices.flatMap((matrix) => matrix.evals)
9035
+ ].map(authoringId));
9036
+ }
9037
+ /**
9038
+ * Mark runs of removed evals as archived, and restore runs whose eval returned.
9039
+ * Reports are never deleted: archiving writes a marker file beside the manifest.
9040
+ */
9041
+ async function tidyReports(options) {
9042
+ const known = projectEvalIds(options.registry);
9043
+ const reports = createDashboardReportReader(options.reportRoot, options.registry);
9044
+ const result = {
9045
+ archived: [],
9046
+ restored: [],
9047
+ stillArchived: 0,
9048
+ current: 0
9049
+ };
9050
+ for (const run of await reports.listRuns()) {
9051
+ const marker = resolve(options.reportRoot, run.id, ARCHIVE_MARKER);
9052
+ const entry = {
9053
+ runId: run.id,
9054
+ evalId: run.evalId
9055
+ };
9056
+ if (known.has(run.evalId)) {
9057
+ result.current += 1;
9058
+ if (run.archived) {
9059
+ await rm(marker, { force: true });
9060
+ result.restored.push(entry);
9061
+ }
9062
+ } else if (run.archived) result.stillArchived += 1;
9063
+ else {
9064
+ await writeFile(marker, `${JSON.stringify({
9065
+ archivedAt: (/* @__PURE__ */ new Date()).toISOString(),
9066
+ reason: `Eval "${run.evalId}" is no longer in the project`
9067
+ })}\n`);
9068
+ result.archived.push(entry);
9069
+ }
9070
+ }
9071
+ return result;
9072
+ }
9073
+ //#endregion
9074
+ //#region ../cli/src/share-command.ts
9075
+ /**
9076
+ * Which local runs `evalkit share` uploads: finished runs with a source, at
9077
+ * HEAD (or any commit with `all`), not dirty (unless `allowDirty`), not archived.
9078
+ */
9079
+ function selectRuns(runs, head, options) {
9080
+ const result = {
9081
+ selected: [],
9082
+ untracked: 0,
9083
+ otherCommits: 0,
9084
+ dirty: 0,
9085
+ archived: 0,
9086
+ running: 0
9087
+ };
9088
+ for (const run of runs) {
9089
+ const { source } = run;
9090
+ if (!source) result.untracked += 1;
9091
+ else if (!options.all && (source.commit !== head.commit || repoKey(source) !== repoKey(head))) result.otherCommits += 1;
9092
+ else if (source.dirty && !options.allowDirty) result.dirty += 1;
9093
+ else if (run.archived) result.archived += 1;
9094
+ else if (run.status === "running") result.running += 1;
9095
+ else result.selected.push({
9096
+ ...run,
9097
+ source
9098
+ });
9099
+ }
9100
+ return result;
9101
+ }
9102
+ /** One line for the Runs step: what will be shared and what was skipped. */
9103
+ function selectionSummary(selection, options) {
9104
+ return [
9105
+ `${plural(selection.selected.length, "run")} to share`,
9106
+ selection.otherCommits && !options.all ? `${selection.otherCommits} from other commits` : "",
9107
+ selection.running ? `${selection.running} still running skipped` : "",
9108
+ selection.dirty ? `${selection.dirty} dirty skipped` : "",
9109
+ selection.untracked ? `${selection.untracked} untracked skipped` : "",
9110
+ selection.archived ? `${selection.archived} archived skipped` : ""
9111
+ ].filter(Boolean).join(" · ");
9112
+ }
9113
+ /** Why nothing was selected, and what to do about it. */
9114
+ function noRunsMessage(selection, head, options) {
9115
+ const at = shortSource(head);
9116
+ if (selection.dirty) return `No committed runs for ${at}: ${plural(selection.dirty, "run")} came from uncommitted changes. Commit and rerun \`evalkit run\`, or pass --allow-dirty.`;
9117
+ if (selection.running) return `No finished runs for ${at}: ${plural(selection.running, "run")} still running. Share again when they finish.`;
9118
+ return `No runs for ${at}. Run \`evalkit run\` first${options.all ? "" : ", or pass --all to share runs from other commits"}.`;
9119
+ }
9120
+ /** Where to look after sharing: the commit, or the repository if several commits were shared. */
9121
+ function shareTarget(serverUrl, head, runs) {
9122
+ const commits = new Set(runs.map((run) => run.source.commit));
9123
+ const commit = commits.size === 1 ? `/-/${[...commits][0]}` : "";
9124
+ return new URL(`/${repoKey(head)}${commit}`, serverUrl).href;
9125
+ }
9126
+ /** Build one run's upload bundle: run.json, catalog.json, and its snapshot. */
9127
+ async function runBundle(options) {
9128
+ const files = [];
9129
+ const encode = (value) => new TextEncoder().encode(`${JSON.stringify(value)}\n`);
9130
+ files.push({
9131
+ path: "run.json",
9132
+ data: encode(options.run)
9133
+ });
9134
+ const saved = await readFile(resolve(options.reportRoot, options.run.id, "catalog.json")).catch(() => void 0);
9135
+ files.push({
9136
+ path: "catalog.json",
9137
+ data: saved ? new Uint8Array(saved) : encode(runCatalog(options.registry))
9138
+ });
9139
+ await snapshotRun(options.reports, options.run.id, {
9140
+ async json(path, value) {
9141
+ files.push({
9142
+ path,
9143
+ data: encode(value)
9144
+ });
9145
+ },
9146
+ async bytes(path, data) {
9147
+ files.push({
9148
+ path,
9149
+ data
9150
+ });
9151
+ }
9152
+ }, { workspaceFiles: options.workspaceFiles });
9153
+ return encodeBundle(files);
9154
+ }
9155
+ /** The server refused the bundle (4xx): retrying the same bundle cannot help. */
9156
+ var UploadRejected = class extends Error {};
9157
+ /** POST one bundle, retrying network errors and 5xx responses. */
9158
+ async function uploadBundle(serverUrl, bundle, options = {}) {
9159
+ const attempts = options.attempts ?? 3;
9160
+ let lastError;
9161
+ for (let attempt = 1; attempt <= attempts; attempt++) {
9162
+ try {
9163
+ return await postBundle(serverUrl, bundle);
9164
+ } catch (error) {
9165
+ if (error instanceof UploadRejected) throw error;
9166
+ lastError = error;
9167
+ }
9168
+ if (attempt < attempts) await Bun.sleep((options.delayMs ?? 500) * 2 ** (attempt - 1));
9169
+ }
9170
+ throw lastError instanceof Error ? lastError : /* @__PURE__ */ new Error(`Upload failed: ${String(lastError)}`);
9171
+ }
9172
+ async function postBundle(serverUrl, bundle) {
9173
+ const response = await fetch(new URL("/api/v1/uploads", serverUrl), {
9174
+ method: "POST",
9175
+ headers: { "content-type": "application/octet-stream" },
9176
+ body: bundle
9177
+ });
9178
+ if (response.ok) return parseUploadResult(await response.json());
9179
+ const reason = await response.text();
9180
+ if (response.status < 500) throw new UploadRejected(`Share server rejected the upload: ${reason}`);
9181
+ throw new Error(`Share server error ${response.status}: ${reason}`);
9182
+ }
9183
+ function parseUploadResult(value) {
9184
+ const result = value;
9185
+ if ((result?.status === "created" || result?.status === "exists") && typeof result.url === "string") return {
9186
+ status: result.status,
9187
+ url: result.url
9188
+ };
9189
+ throw new UploadRejected(`Unexpected share server response: ${JSON.stringify(value)}`);
9190
+ }
9191
+ async function shareRuns(options) {
9192
+ const reports = createDashboardReportReader(options.reportRoot, options.registry);
9193
+ return options.step("Uploading", async (progress) => {
9194
+ const result = {
9195
+ created: 0,
9196
+ exists: 0,
9197
+ failed: [],
9198
+ bytes: 0
9199
+ };
9200
+ for (const [index, run] of options.runs.entries()) {
9201
+ progress(`run ${index + 1}/${options.runs.length} · ${formatBytes(result.bytes)}`);
9202
+ try {
9203
+ const bundle = await runBundle({
9204
+ reports,
9205
+ reportRoot: options.reportRoot,
9206
+ registry: options.registry,
9207
+ run,
9208
+ workspaceFiles: options.workspaceFiles
9209
+ });
9210
+ const uploaded = await uploadBundle(options.serverUrl, bundle);
9211
+ result.bytes += bundle.byteLength;
9212
+ result[uploaded.status] += 1;
9213
+ } catch (error) {
9214
+ result.failed.push({
9215
+ runId: run.id,
9216
+ error: error instanceof Error ? error.message : String(error)
9217
+ });
9218
+ }
9219
+ }
9220
+ return result;
9221
+ }, (result) => `${result.created} new · ${result.exists} already shared` + (result.failed.length ? ` · ${result.failed.length} failed` : "") + ` · ${formatBytes(result.bytes)}`);
9222
+ }
9223
+ function formatBytes(size) {
9224
+ if (size < 1024) return `${size} B`;
9225
+ if (size < 1024 ** 2) return `${(size / 1024).toFixed(1)} kB`;
9226
+ return `${(size / 1024 ** 2).toFixed(1)} MB`;
8637
9227
  }
8638
9228
  //#endregion
8639
9229
  //#region ../cli/src/new-project-template.ts
@@ -8735,12 +9325,11 @@ Replace these examples with your own agent, tasks, and checks.
8735
9325
 
8736
9326
  \`\`\`sh
8737
9327
  bun install
8738
- bun run check # type-check your eval, agent, scorer, and config
8739
- bun run evals # run the example across both styles
8740
- bun run matrix:plan # inspect the matrix without running it
8741
- bun run matrix # run the configured matrix
8742
- bun run dashboard # inspect local reports and launch runs
8743
- bun run dashboard:build # write a read-only, shareable snapshot
9328
+ bun run check # type-check your eval, agent, scorer, and config
9329
+ bun run evalkit run # run the example across both styles
9330
+ bun run evalkit run --matrix styles --dry-run # inspect the matrix without running it
9331
+ bun run evalkit dev # inspect local reports and launch runs
9332
+ bun run evalkit build # write a read-only, shareable snapshot
8744
9333
  \`\`\`
8745
9334
 
8746
9335
  EvalKit installs from npm. Reports, trial workspaces, and dashboard snapshots stay local
@@ -8750,12 +9339,13 @@ scores, evidence, and candidate files (never evaluator workspaces): review it be
8750
9339
  }
8751
9340
  //#endregion
8752
9341
  //#region ../cli/src/new-project.ts
8753
- const npmDependency = "^0.2.0";
9342
+ const npmDependency = "^0.4.0";
8754
9343
  const ignored = [
8755
9344
  "node_modules/",
8756
9345
  "_evalkit-results/",
8757
9346
  "_evalkit-sandbox/",
8758
- "_evalkit-dashboard/"
9347
+ "_evalkit-dashboard/",
9348
+ "_evalkit-share/"
8759
9349
  ];
8760
9350
  const readIfExists = async (path) => {
8761
9351
  try {
@@ -8834,11 +9424,6 @@ async function preparePackage(packagePath, name) {
8834
9424
  const devDependencies = pkg.devDependencies ?? {};
8835
9425
  if (!scripts || typeof scripts !== "object" || Array.isArray(scripts) || !dependencies || typeof dependencies !== "object" || Array.isArray(dependencies) || !devDependencies || typeof devDependencies !== "object" || Array.isArray(devDependencies)) throw new Error(`Invalid package.json scripts/dependencies: ${packagePath}`);
8836
9426
  scripts.check ??= "tsc --noEmit";
8837
- scripts.evals ??= "evalkit run greeting --local";
8838
- scripts.matrix ??= "evalkit run --matrix styles --local";
8839
- scripts["matrix:plan"] ??= "evalkit run --matrix styles --dry-run";
8840
- scripts.dashboard ??= "evalkit dev";
8841
- scripts["dashboard:build"] ??= "evalkit build";
8842
9427
  if (!dependencies["@leostera/evalkit"] && !devDependencies["@leostera/evalkit"]) dependencies["@leostera/evalkit"] = npmDependency;
8843
9428
  if (!dependencies.typescript && !devDependencies.typescript) devDependencies.typescript = "^6.0.0";
8844
9429
  if (!dependencies["@types/bun"] && !devDependencies["@types/bun"]) devDependencies["@types/bun"] = "^1.4.0";
@@ -8867,6 +9452,7 @@ async function writeStarterReadme(root, name, inPlace) {
8867
9452
  }
8868
9453
  //#endregion
8869
9454
  //#region ../cli/src/index.ts
9455
+ const VERSION = "0.4.0";
8870
9456
  let projectRoot = process.cwd();
8871
9457
  let reportRoot = resolve(projectRoot, "_evalkit-results");
8872
9458
  const sandboxRoot = resolve(projectRoot, "_evalkit-sandbox");
@@ -8886,13 +9472,31 @@ async function runEvals(options) {
8886
9472
  const registry = await loadRegistry();
8887
9473
  const evaluations = options.evalIds.map((id) => registry.get(id));
8888
9474
  if (evaluations.some((e) => !e)) throw new Error("Unknown eval");
8889
- await Effect.runPromise(Effect.all(evaluations.map((evaluation) => runEval(evaluation, {
8890
- report: localReportStore(reportRoot),
8891
- workspaceRoot: sandboxRoot,
8892
- suiteId: options.suiteId,
8893
- runtime: runtimeFor(evaluation),
8894
- concurrency: options.concurrency
8895
- })), { concurrency: options.concurrency }));
9475
+ const provenance = await runProvenance(registry);
9476
+ await Effect.runPromise(runnerLogs(Effect.all(evaluations.map((evaluation) => {
9477
+ const name = evaluation.name ?? authoringId(evaluation);
9478
+ const started = performance.now();
9479
+ return Effect.sync(() => runLog.started(name, {}, options.suiteId)).pipe(Effect.andThen(runEval(evaluation, {
9480
+ report: localReportStore(reportRoot),
9481
+ workspaceRoot: sandboxRoot,
9482
+ suiteId: options.suiteId,
9483
+ runtime: runtimeFor(evaluation),
9484
+ concurrency: options.concurrency,
9485
+ ...provenance
9486
+ })), Effect.tap((result) => runLog.finished(name, {}, result, performance.now() - started)), Effect.tapError((error) => Effect.sync(() => runLog.failed(name, error))));
9487
+ }), { concurrency: options.concurrency })));
9488
+ }
9489
+ /** Detected per launch: `evalkit dev` keeps running while commits change. */
9490
+ async function runProvenance(registry) {
9491
+ const source = await detectSource(projectRoot, { outputDirs: [
9492
+ reportRoot,
9493
+ sandboxRoot,
9494
+ resolve(projectRoot, "_evalkit-dashboard")
9495
+ ] });
9496
+ return {
9497
+ ...source ? { source } : {},
9498
+ catalog: runCatalog(registry)
9499
+ };
8896
9500
  }
8897
9501
  function registerMatrixRoutes(app, registry, dashboardMatrix, trials) {
8898
9502
  let matrixRunning = false;
@@ -8935,14 +9539,19 @@ function registerMatrixRoutes(app, registry, dashboardMatrix, trials) {
8935
9539
  }
8936
9540
  if (matrixRunning) return context.text("A dashboard matrix cell is already running", 409);
8937
9541
  matrixRunning = true;
8938
- Effect.runPromise(runMatrix(dashboardMatrix, {
9542
+ const name = evaluation.name ?? authoringId(evaluation);
9543
+ const started = performance.now();
9544
+ runLog.started(name, body.parameters ?? {}, suiteId);
9545
+ runProvenance(registry).then((provenance) => Effect.runPromise(runnerLogs(runMatrix(dashboardMatrix, {
8939
9546
  selection,
8940
9547
  concurrency: 1,
8941
9548
  trials,
8942
9549
  ...suiteId ? { suiteId } : {},
8943
9550
  report: localReportStore(reportRoot),
8944
- workspaceRoot: sandboxRoot
8945
- })).catch((error) => console.error("Dashboard matrix run failed:", error)).finally(() => {
9551
+ workspaceRoot: sandboxRoot,
9552
+ ...provenance,
9553
+ onResult: (cell, result) => runLog.cell(cell, result, performance.now() - started)
9554
+ })))).catch((error) => runLog.failed(name, error)).finally(() => {
8946
9555
  matrixRunning = false;
8947
9556
  });
8948
9557
  } else {
@@ -8951,7 +9560,7 @@ function registerMatrixRoutes(app, registry, dashboardMatrix, trials) {
8951
9560
  evalIds: [authoringId(evaluation)],
8952
9561
  suiteId,
8953
9562
  concurrency: 32
8954
- }).catch((error) => console.error("Dashboard run failed:", error));
9563
+ }).catch(() => {});
8955
9564
  }
8956
9565
  return context.json({ accepted: true }, 202);
8957
9566
  });
@@ -8963,7 +9572,7 @@ function registerMatrixRoutes(app, registry, dashboardMatrix, trials) {
8963
9572
  evalIds: suite.evals.map(authoringId),
8964
9573
  suiteId: authoringId(suite),
8965
9574
  concurrency: 32
8966
- }).catch((error) => console.error("Dashboard suite run failed:", error));
9575
+ }).catch(() => {});
8967
9576
  return context.json({ accepted: true }, 202);
8968
9577
  });
8969
9578
  }
@@ -9047,14 +9656,32 @@ async function serveDashboard() {
9047
9656
  const registry = await loadRegistry();
9048
9657
  const dashboardRoot = dashboardAssets();
9049
9658
  const app = new Hono();
9050
- registerMatrixRoutes(app, registry, project?.config.matrix ? registry.matrices.at(-1) : void 0, project?.config.execution?.trials);
9659
+ const dashboardMatrix = project?.config.matrix ? registry.matrices.at(-1) : void 0;
9660
+ registerMatrixRoutes(app, registry, dashboardMatrix, project?.config.execution?.trials);
9051
9661
  registerReportRoutes(app, registry, createDashboardReportReader(reportRoot, registry));
9052
9662
  registerDashboardAssets(app, dashboardRoot);
9053
- const url = `http://localhost:${Bun.serve({
9054
- port: Number(process.env.PORT ?? 4317),
9055
- fetch: app.fetch
9056
- }).port}`;
9057
- console.log(`EvalKit dashboard: ${url}`);
9663
+ const port = Number(process.env.PORT ?? 4317);
9664
+ let server;
9665
+ try {
9666
+ server = Bun.serve({
9667
+ port,
9668
+ fetch: app.fetch
9669
+ });
9670
+ } catch (error) {
9671
+ if (error.code === "EADDRINUSE") throw new Error(`Port ${port} is already in use. Stop the other server or choose another port: PORT=4318 evalkit dev`);
9672
+ throw error;
9673
+ }
9674
+ const url = `http://localhost:${server.port}`;
9675
+ const reader = createDashboardReportReader(reportRoot, registry);
9676
+ const runs = (await reader.listRuns()).length;
9677
+ const legacy = reader.legacyRunIds().length;
9678
+ const matrixCells = dashboardMatrix?.count({ parameters: {} });
9679
+ const row = (label, value) => console.log(` ${cyan("➜")} ${bold(label.padEnd(9))}${value}`);
9680
+ banner("dev", VERSION);
9681
+ row("Local:", cyan(url));
9682
+ row("Project:", `${plural(registry.evals.length, "eval")} · ${plural(registry.suites.length, "suite")}` + (dashboardMatrix ? ` · matrix ${authoringId(dashboardMatrix)} (${plural(matrixCells, "cell")})` : ""));
9683
+ row("Reports:", `${relative(process.cwd(), reportRoot) || "."} ${dim(`(${plural(runs, "run")}${legacy ? `, ${legacy} from an older EvalKit hidden` : ""})`)}`);
9684
+ console.log(`\n ${dim("Runs started from the dashboard are logged below. Press Ctrl+C to stop.")}\n`);
9058
9685
  if (process.env.EVALKIT_NO_OPEN !== "1") {
9059
9686
  const opener = process.platform === "darwin" ? "open" : process.platform === "win32" ? "start" : "xdg-open";
9060
9687
  try {
@@ -9119,7 +9746,7 @@ const runCommand = make$1("run", {
9119
9746
  const newCommand = make$1("new", { directory: text$2({ name: "directory" }) }, ({ directory }) => Effect.tryPromise({
9120
9747
  try: async () => {
9121
9748
  const root = await newProject([directory]);
9122
- console.log(`Created EvalKit project at ${root}\nRun cd ${root} && bun install && bun run evals.`);
9749
+ console.log(`Created EvalKit project at ${root}\nRun cd ${root} && bun install && bun run evalkit run.`);
9123
9750
  },
9124
9751
  catch: (error) => error
9125
9752
  }));
@@ -9132,29 +9759,162 @@ const devCommand = make$1("dev", { config }, ({ config }) => Effect.tryPromise({
9132
9759
  }));
9133
9760
  const buildCommand = make$1("build", {
9134
9761
  config,
9135
- out: text("out").pipe(optional)
9136
- }, ({ config, out }) => Effect.tryPromise({
9762
+ out: text("out").pipe(optional),
9763
+ run: boolean("run"),
9764
+ noRun: boolean("no-run")
9765
+ }, ({ config, out, run, noRun }) => Effect.tryPromise({
9137
9766
  try: async () => {
9138
- await useProject(Option.getOrUndefined(config));
9767
+ if (run && noRun) throw new Error("Choose either --run or --no-run");
9768
+ const started = performance.now();
9769
+ const configPath = Option.getOrUndefined(config);
9770
+ banner("build", VERSION);
9771
+ await loadProjectStep(configPath);
9772
+ if (await shouldRunEvals({
9773
+ run,
9774
+ noRun
9775
+ })) await runAllEvals(configPath);
9139
9776
  const result = await buildDashboard({
9140
9777
  registry: project.registry,
9141
9778
  reportRoot,
9142
9779
  dashboardRoot: dashboardAssets(),
9143
9780
  outDir: resolve(projectRoot, Option.getOrElse(out, () => "_evalkit-dashboard"))
9781
+ }, step);
9782
+ const size = await directorySize(result.outDir);
9783
+ const shown = relative(process.cwd(), result.outDir) || ".";
9784
+ console.log(`\n ${bold(`${shown}/`)} ${dim(`${bytes(size.bytes)} · ${plural(size.files, "file")}`)}`);
9785
+ console.log(`\n ${bold("Built read-only dashboard")} in ${duration(performance.now() - started)}\n`);
9786
+ console.log(` ${cyan("➜")} ${bold("Preview:")} bunx serve ${shown}`);
9787
+ console.log(` ${yellow("⚠")} Contains transcripts, scores, evidence, and candidate files\n (never evaluator workspaces). Review before sharing.\n`);
9788
+ },
9789
+ catch: (error) => error
9790
+ }));
9791
+ async function loadProjectStep(configPath) {
9792
+ await step("Eval project", () => useProject(configPath), () => {
9793
+ const where = relative(process.cwd(), projectRoot);
9794
+ return `${plural(project.registry.evals.length, "eval")} loaded${where ? ` from ${where}` : ""}`;
9795
+ });
9796
+ }
9797
+ /**
9798
+ * A snapshot needs data: with no current results, run the evals. Otherwise ask,
9799
+ * or keep the existing results when no one can answer (CI, pipes).
9800
+ */
9801
+ async function shouldRunEvals(flags) {
9802
+ if (flags.run) return true;
9803
+ if (flags.noRun) return false;
9804
+ const runs = (await createDashboardReportReader(reportRoot, project.registry).listRuns()).filter((entry) => !entry.archived).length;
9805
+ if (!runs) {
9806
+ console.log(` ${cyan("◇")} No results yet, so running the evals to have data to build.`);
9807
+ return true;
9808
+ }
9809
+ if (!canPrompt()) {
9810
+ console.log(` ${dim(`Using ${plural(runs, "existing run")}; pass --run to run the evals first.`)}`);
9811
+ return false;
9812
+ }
9813
+ return confirm(`Run the evals before building? ${dim(`(${plural(runs, "existing run")})`)}`);
9814
+ }
9815
+ async function runAllEvals(configPath) {
9816
+ const started = performance.now();
9817
+ console.log(`\n ${cyan("▶")} ${bold("Running evals")}\n`);
9818
+ await runProjectCommand([], process.cwd(), parseRunArgs(configPath ? ["--config", configPath] : []));
9819
+ const failed = process.exitCode === 1;
9820
+ console.log(`\n ${failed ? yellow("⚠") : cyan("✓")} ${bold("Evals finished")} in ${duration(performance.now() - started)}` + (failed ? dim(" (some failed; building with their results)") : "") + "\n");
9821
+ }
9822
+ const tidyCommand = make$1("tidy", { config }, ({ config }) => Effect.tryPromise({
9823
+ try: async () => {
9824
+ banner("tidy", VERSION);
9825
+ await loadProjectStep(Option.getOrUndefined(config));
9826
+ const result = await step("Run reports", () => tidyReports({
9827
+ registry: project.registry,
9828
+ reportRoot
9829
+ }), (value) => `${plural(value.current + value.archived.length + value.stillArchived, "run")} checked`);
9830
+ const byEval = (entries) => [...entries.reduce((groups, entry) => groups.set(entry.evalId, (groups.get(entry.evalId) ?? 0) + 1), /* @__PURE__ */ new Map()).entries()].map(([id, total]) => ` ${bold(id)} ${dim(`(${plural(total, "run")})`)}`).join("\n");
9831
+ console.log("");
9832
+ if (result.archived.length) console.log(` ${yellow("◆")} Archived ${plural(result.archived.length, "run")} of evals no longer in the project:\n${byEval(result.archived)}`);
9833
+ if (result.restored.length) console.log(` ${cyan("◆")} Restored ${plural(result.restored.length, "run")} whose eval is back:\n${byEval(result.restored)}`);
9834
+ if (!result.archived.length && !result.restored.length) console.log(` ${cyan("✓")} Nothing new to tidy.`);
9835
+ const totalArchived = result.archived.length + result.stillArchived;
9836
+ if (totalArchived) console.log(`\n ${dim(`Archived runs (${totalArchived}) are hidden in \`evalkit dev\` (filter "Show archived")`)}\n ${dim("and left out of `evalkit build`. Nothing was deleted.")}`);
9837
+ console.log("");
9838
+ },
9839
+ catch: (error) => error
9840
+ }));
9841
+ const shareServeCommand = make$1("serve", {
9842
+ dir: text("dir").pipe(withDefault("_evalkit-share")),
9843
+ port: integer("port").pipe(withDefault(4318))
9844
+ }, ({ dir, port }) => Effect.tryPromise({
9845
+ try: async () => {
9846
+ const root = resolve(process.cwd(), dir);
9847
+ const app = await createDirectoryShareApp({
9848
+ dir: root,
9849
+ dashboardRoot: dashboardAssets()
9144
9850
  });
9145
- console.log(`Built a read-only dashboard snapshot at ${result.outDir}\n ${result.runs} run(s), ${result.trials} trial(s), ${result.files} candidate file(s)\nServe it with any static file server. It contains report messages, scores, evidence,
9146
- and candidate files (never evaluator workspaces): review it before sharing.`);
9851
+ let server;
9852
+ try {
9853
+ server = Bun.serve({
9854
+ port,
9855
+ fetch: app.fetch
9856
+ });
9857
+ } catch (error) {
9858
+ if (error.code === "EADDRINUSE") throw new Error(`Port ${port} is already in use. Choose another: evalkit share serve --port 4319`);
9859
+ throw error;
9860
+ }
9861
+ banner("share serve", VERSION);
9862
+ console.log(` ${cyan("➜")} ${bold("Local:".padEnd(9))}${cyan(`http://localhost:${server.port}`)}`);
9863
+ console.log(` ${cyan("➜")} ${bold("Storage:".padEnd(9))}${relative(process.cwd(), root) || "."}`);
9864
+ console.log(`\n ${dim(`Upload with share.url = "http://localhost:${server.port}". No authentication: keep it private.`)}\n`);
9147
9865
  },
9148
9866
  catch: (error) => error
9149
9867
  }));
9868
+ const shareCommand = make$1("share", {
9869
+ config,
9870
+ all: boolean("all"),
9871
+ allowDirty: boolean("allow-dirty"),
9872
+ noWorkspace: boolean("no-workspace")
9873
+ }, ({ config, all, allowDirty, noWorkspace }) => Effect.tryPromise({
9874
+ try: async () => {
9875
+ const started = performance.now();
9876
+ banner("share", VERSION);
9877
+ await loadProjectStep(Option.getOrUndefined(config));
9878
+ const serverUrl = process.env.EVALKIT_SHARE_URL ?? project.config.share?.url;
9879
+ if (!serverUrl) throw new Error("No share server configured. Set share: { url } in evalkit.config.ts or EVALKIT_SHARE_URL.");
9880
+ const head = await step("Source", () => detectSource(projectRoot, { outputDirs: [
9881
+ reportRoot,
9882
+ sandboxRoot,
9883
+ resolve(projectRoot, "_evalkit-dashboard")
9884
+ ] }), (source) => source ? `${repoKey(source)} @ ${shortSource(source)}${source.ref ? ` (${source.ref})` : ""}` : "not a git repository");
9885
+ if (!head) throw new Error("evalkit share needs to know the commit: run it inside a git repository or set EVALKIT_SOURCE_REPO and EVALKIT_SOURCE_COMMIT. `evalkit build` works without one.");
9886
+ const registry = project.registry;
9887
+ const selection = await step("Runs", async () => selectRuns(await createDashboardReportReader(reportRoot, registry).listRuns(), head, {
9888
+ all,
9889
+ allowDirty
9890
+ }), (value) => selectionSummary(value, { all }));
9891
+ if (!selection.selected.length) throw new Error(noRunsMessage(selection, head, { all }));
9892
+ const result = await shareRuns({
9893
+ serverUrl,
9894
+ reportRoot,
9895
+ registry,
9896
+ runs: selection.selected,
9897
+ workspaceFiles: !noWorkspace,
9898
+ step
9899
+ });
9900
+ for (const failure of result.failed) console.log(` ${yellow("✗")} ${failure.runId.slice(0, 8)} ${dim(failure.error)}`);
9901
+ const target = shareTarget(serverUrl, head, selection.selected);
9902
+ console.log(`\n ${bold(result.failed.length ? "Shared with errors" : "Shared")} in ${duration(performance.now() - started)}\n`);
9903
+ console.log(` ${cyan("➜")} ${cyan(target)}\n`);
9904
+ if (result.failed.length) process.exitCode = 1;
9905
+ },
9906
+ catch: (error) => error
9907
+ })).pipe(withSubcommands([shareServeCommand]));
9150
9908
  const cli = run(make$1("evalkit").pipe(withSubcommands([
9151
9909
  newCommand,
9152
9910
  runCommand,
9153
9911
  devCommand,
9154
- buildCommand
9912
+ buildCommand,
9913
+ tidyCommand,
9914
+ shareCommand
9155
9915
  ])), {
9156
9916
  name: "EvalKit",
9157
- version: "0.2.0"
9917
+ version: VERSION
9158
9918
  });
9159
9919
  const argv = process.argv.slice();
9160
9920
  if (argv[2] === "help") argv.splice(2, 1, "--help");