@leostera/evalkit 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/cli.mjs +641 -172
- package/dist/cli.mjs.map +1 -1
- package/dist/dashboard/assets/index-CMr4aulR.css +2 -0
- package/dist/dashboard/assets/{index-BYSEeNaZ.js → index-CVd_Np_w.js} +7 -7
- package/dist/dashboard/index.html +2 -2
- package/package.json +1 -1
- package/dist/dashboard/assets/index-fUDsY6z-.css +0 -2
package/dist/cli.mjs
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
import { F as defineEvalMatrix, I as canonicalParameters, m as registerEvals, n as authoringId } from "./src-DrwwgaPn.mjs";
|
|
3
3
|
import { a as readTrialEvents, c as readTrialSummary, i as readRunSummary, l as localReportStore, n as runMatrix, o as readTrialManifest, r as readRunManifest, t as runEval } from "./src-DluldGKn.mjs";
|
|
4
4
|
import * as Schema from "effect/Schema";
|
|
5
|
-
import { Effect, Option } from "effect";
|
|
6
|
-
import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises";
|
|
5
|
+
import { Effect, LogLevel, Logger, Option } from "effect";
|
|
6
|
+
import { cp, mkdir, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
|
|
7
7
|
import * as Path from "node:path";
|
|
8
8
|
import { basename, dirname, extname, relative, resolve, sep } from "node:path";
|
|
9
9
|
import * as OS from "node:os";
|
|
@@ -43,11 +43,11 @@ import * as EffectSecret from "effect/Secret";
|
|
|
43
43
|
import * as Effectable from "effect/Effectable";
|
|
44
44
|
import * as Match from "effect/Match";
|
|
45
45
|
import * as EffectNumber from "effect/Number";
|
|
46
|
-
import * as LogLevel from "effect/LogLevel";
|
|
46
|
+
import * as LogLevel$1 from "effect/LogLevel";
|
|
47
47
|
import * as Config from "effect/Config";
|
|
48
48
|
import * as HashMap from "effect/HashMap";
|
|
49
49
|
import * as Order from "effect/Order";
|
|
50
|
-
import * as Logger from "effect/Logger";
|
|
50
|
+
import * as Logger$1 from "effect/Logger";
|
|
51
51
|
import * as Unify from "effect/Unify";
|
|
52
52
|
import * as HashSet from "effect/HashSet";
|
|
53
53
|
import * as SynchronizedRef from "effect/SynchronizedRef";
|
|
@@ -392,13 +392,13 @@ const struct = (fields) => {
|
|
|
392
392
|
/** @internal */
|
|
393
393
|
const black = { _tag: "Black" };
|
|
394
394
|
/** @internal */
|
|
395
|
-
const red$
|
|
395
|
+
const red$4 = { _tag: "Red" };
|
|
396
396
|
/** @internal */
|
|
397
|
-
const green$
|
|
397
|
+
const green$3 = { _tag: "Green" };
|
|
398
398
|
/** @internal */
|
|
399
399
|
const magenta$1 = { _tag: "Magenta" };
|
|
400
400
|
/** @internal */
|
|
401
|
-
const cyan$
|
|
401
|
+
const cyan$2 = { _tag: "Cyan" };
|
|
402
402
|
/** @internal */
|
|
403
403
|
const white$3 = { _tag: "White" };
|
|
404
404
|
/** @internal */
|
|
@@ -500,7 +500,7 @@ const ESC = "\x1B[";
|
|
|
500
500
|
const BEL = "\x07";
|
|
501
501
|
const SEP = ";";
|
|
502
502
|
/** @internal */
|
|
503
|
-
const bold$
|
|
503
|
+
const bold$2 = /*#__PURE__*/ make$5({ bold: /*#__PURE__*/ Option$1.some(/*#__PURE__*/ setBold(true)) });
|
|
504
504
|
/** @internal */
|
|
505
505
|
const italicized$1 = /*#__PURE__*/ make$5({ italicized: /*#__PURE__*/ Option$1.some(/*#__PURE__*/ setItalicized(true)) });
|
|
506
506
|
/** @internal */
|
|
@@ -512,15 +512,15 @@ const brightColor = (color) => make$5({ foreground: Option$1.some(setColor(color
|
|
|
512
512
|
/** @internal */
|
|
513
513
|
const color$1 = (color) => make$5({ foreground: Option$1.some(setColor(color, false, "foreground")) });
|
|
514
514
|
/** @internal */
|
|
515
|
-
const red$
|
|
515
|
+
const red$3 = /*#__PURE__*/ color$1(red$4);
|
|
516
516
|
/** @internal */
|
|
517
|
-
const green$
|
|
517
|
+
const green$2 = /*#__PURE__*/ color$1(green$3);
|
|
518
518
|
/** @internal */
|
|
519
519
|
const white$2 = /*#__PURE__*/ color$1(white$3);
|
|
520
520
|
/** @internal */
|
|
521
521
|
const blackBright$1 = /*#__PURE__*/ brightColor(black);
|
|
522
522
|
/** @internal */
|
|
523
|
-
const cyanBright$1 = /*#__PURE__*/ brightColor(cyan$
|
|
523
|
+
const cyanBright$1 = /*#__PURE__*/ brightColor(cyan$2);
|
|
524
524
|
/** @internal */
|
|
525
525
|
const beep$2 = /*#__PURE__*/ make$5({ commands: /*#__PURE__*/ Arr.of(BEL) });
|
|
526
526
|
/** @internal */
|
|
@@ -587,7 +587,7 @@ const stringifyInternal = (self) => {
|
|
|
587
587
|
* @since 1.0.0
|
|
588
588
|
* @category constructors
|
|
589
589
|
*/
|
|
590
|
-
const bold = bold$
|
|
590
|
+
const bold$1 = bold$2;
|
|
591
591
|
/**
|
|
592
592
|
* @since 1.0.0
|
|
593
593
|
* @category constructors
|
|
@@ -612,12 +612,12 @@ const color = color$1;
|
|
|
612
612
|
* @since 1.0.0
|
|
613
613
|
* @category colors
|
|
614
614
|
*/
|
|
615
|
-
const red$
|
|
615
|
+
const red$2 = red$3;
|
|
616
616
|
/**
|
|
617
617
|
* @since 1.0.0
|
|
618
618
|
* @category colors
|
|
619
619
|
*/
|
|
620
|
-
const green = green$
|
|
620
|
+
const green$1 = green$2;
|
|
621
621
|
/**
|
|
622
622
|
* @since 1.0.0
|
|
623
623
|
* @category colors
|
|
@@ -2218,7 +2218,7 @@ const optimize = optimize$1;
|
|
|
2218
2218
|
* @since 1.0.0
|
|
2219
2219
|
* @category constructors
|
|
2220
2220
|
*/
|
|
2221
|
-
const red = red$
|
|
2221
|
+
const red$1 = red$4;
|
|
2222
2222
|
/**
|
|
2223
2223
|
* @since 1.0.0
|
|
2224
2224
|
* @category constructors
|
|
@@ -2228,7 +2228,7 @@ const magenta = magenta$1;
|
|
|
2228
2228
|
* @since 1.0.0
|
|
2229
2229
|
* @category constructors
|
|
2230
2230
|
*/
|
|
2231
|
-
const cyan = cyan$
|
|
2231
|
+
const cyan$1 = cyan$2;
|
|
2232
2232
|
/**
|
|
2233
2233
|
* @since 1.0.0
|
|
2234
2234
|
* @category constructors
|
|
@@ -2248,7 +2248,7 @@ const space = /*#__PURE__*/ text$6(" ");
|
|
|
2248
2248
|
/** @internal */
|
|
2249
2249
|
const code = (value) => highlight(value, white);
|
|
2250
2250
|
/** @internal */
|
|
2251
|
-
const error = (value) => highlight(value, red);
|
|
2251
|
+
const error = (value) => highlight(value, red$1);
|
|
2252
2252
|
/** @internal */
|
|
2253
2253
|
const highlight = (value, color) => ({
|
|
2254
2254
|
_tag: "Highlight",
|
|
@@ -2307,7 +2307,7 @@ const toAnsiDoc$1 = (self) => {
|
|
|
2307
2307
|
switch (self._tag) {
|
|
2308
2308
|
case "Highlight": return annotate(toAnsiDoc$1(self.value), color(self.color));
|
|
2309
2309
|
case "Sequence": return cat(toAnsiDoc$1(self.left), toAnsiDoc$1(self.right));
|
|
2310
|
-
case "Strong": return annotate(toAnsiDoc$1(self.value), bold);
|
|
2310
|
+
case "Strong": return annotate(toAnsiDoc$1(self.value), bold$1);
|
|
2311
2311
|
case "Text": return text$7(self.value);
|
|
2312
2312
|
case "URI": return annotate(text$7(self.value), underlined);
|
|
2313
2313
|
case "Weak": return annotate(toAnsiDoc$1(self.value), blackBright);
|
|
@@ -2373,11 +2373,11 @@ const toAnsiText = (self) => render$1(toAnsiDoc(self), { style: "pretty" });
|
|
|
2373
2373
|
const toAnsiDocInternal = (self) => {
|
|
2374
2374
|
switch (self._tag) {
|
|
2375
2375
|
case "Empty": return empty$3;
|
|
2376
|
-
case "Header": return pipe(annotate(toAnsiDoc$1(self.value), bold), cat(hardLine));
|
|
2376
|
+
case "Header": return pipe(annotate(toAnsiDoc$1(self.value), bold$1), cat(hardLine));
|
|
2377
2377
|
case "Paragraph": return pipe(toAnsiDoc$1(self.value), cat(hardLine));
|
|
2378
2378
|
case "DescriptionList": {
|
|
2379
2379
|
const definitions = self.definitions.map(([span, doc]) => cats([
|
|
2380
|
-
annotate(toAnsiDoc$1(span), bold),
|
|
2380
|
+
annotate(toAnsiDoc$1(span), bold$1),
|
|
2381
2381
|
empty$3,
|
|
2382
2382
|
indent(toAnsiDocInternal(doc), 2)
|
|
2383
2383
|
]));
|
|
@@ -2592,8 +2592,8 @@ function renderError$2(state, pointer) {
|
|
|
2592
2592
|
onSome: (error) => {
|
|
2593
2593
|
const errorLines = error.split(NEWLINE_REGEX$3);
|
|
2594
2594
|
if (Arr.isNonEmptyReadonlyArray(errorLines)) {
|
|
2595
|
-
const annotateLine = (line) => annotate(text$7(line), combine(italicized, red$
|
|
2596
|
-
const prefix = cat(annotate(pointer, red$
|
|
2595
|
+
const annotateLine = (line) => annotate(text$7(line), combine(italicized, red$2));
|
|
2596
|
+
const prefix = cat(annotate(pointer, red$2), space$1);
|
|
2597
2597
|
const lines = Arr.map(errorLines, (str) => annotateLine(str));
|
|
2598
2598
|
return cursorSavePosition.pipe(cat(hardLine), cat(prefix), cat(align(vsep(lines))), cat(cursorRestorePosition));
|
|
2599
2599
|
}
|
|
@@ -2612,7 +2612,7 @@ function renderParts(state, submitted = false) {
|
|
|
2612
2612
|
});
|
|
2613
2613
|
}
|
|
2614
2614
|
function renderOutput$4(leadingSymbol, trailingSymbol, parts, options) {
|
|
2615
|
-
const annotateLine = (line) => annotate(text$7(line), bold);
|
|
2615
|
+
const annotateLine = (line) => annotate(text$7(line), bold$1);
|
|
2616
2616
|
const prefix = cat(leadingSymbol, space$1);
|
|
2617
2617
|
return Arr.match(options.message.split(/\r?\n/), {
|
|
2618
2618
|
onEmpty: () => hsep([
|
|
@@ -2642,7 +2642,7 @@ function renderSubmission$5(state, options) {
|
|
|
2642
2642
|
return Effect$1.gen(function* () {
|
|
2643
2643
|
const columns = yield* (yield* Terminal).columns;
|
|
2644
2644
|
const figures$11 = yield* figures;
|
|
2645
|
-
return renderOutput$4(annotate(figures$11.tick, green), annotate(figures$11.ellipsis, blackBright), renderParts(state, true), options).pipe(cat(hardLine), optimize(Deep), render$1({
|
|
2645
|
+
return renderOutput$4(annotate(figures$11.tick, green$1), annotate(figures$11.ellipsis, blackBright), renderParts(state, true), options).pipe(cat(hardLine), optimize(Deep), render$1({
|
|
2646
2646
|
style: "pretty",
|
|
2647
2647
|
options: { lineWidth: columns }
|
|
2648
2648
|
}));
|
|
@@ -3106,7 +3106,7 @@ function handleClear$4(options) {
|
|
|
3106
3106
|
}
|
|
3107
3107
|
const NEWLINE_REGEX$2 = /\r?\n/;
|
|
3108
3108
|
function renderPrompt(confirm, message, leadingSymbol, trailingSymbol) {
|
|
3109
|
-
const annotateLine = (line) => annotate(text$7(line), bold);
|
|
3109
|
+
const annotateLine = (line) => annotate(text$7(line), bold$1);
|
|
3110
3110
|
const prefix = cat(leadingSymbol, space$1);
|
|
3111
3111
|
return Arr.match(message.split(NEWLINE_REGEX$2), {
|
|
3112
3112
|
onEmpty: () => hsep([
|
|
@@ -3159,7 +3159,7 @@ function renderNextFrame$4(state, options) {
|
|
|
3159
3159
|
options: { lineWidth: columns }
|
|
3160
3160
|
}));
|
|
3161
3161
|
}
|
|
3162
|
-
const leadingSymbol = annotate(figures$10.tick, green);
|
|
3162
|
+
const leadingSymbol = annotate(figures$10.tick, green$1);
|
|
3163
3163
|
const trailingSymbol = annotate(figures$10.ellipsis, blackBright);
|
|
3164
3164
|
const promptMsg = renderPrompt(empty$3, options.message, leadingSymbol, trailingSymbol);
|
|
3165
3165
|
const files = renderFiles(state, state.files, figures$10, options);
|
|
@@ -3173,7 +3173,7 @@ function renderSubmission$4(value, options) {
|
|
|
3173
3173
|
return Effect$1.gen(function* () {
|
|
3174
3174
|
const columns = yield* (yield* Terminal).columns;
|
|
3175
3175
|
const figures$9 = yield* figures;
|
|
3176
|
-
const leadingSymbol = annotate(figures$9.tick, green);
|
|
3176
|
+
const leadingSymbol = annotate(figures$9.tick, green$1);
|
|
3177
3177
|
const trailingSymbol = annotate(figures$9.ellipsis, blackBright);
|
|
3178
3178
|
return renderPrompt(empty$3, options.message, leadingSymbol, trailingSymbol).pipe(cat(space$1), cat(annotate(text$7(value), white$1)), cat(hardLine), render$1({
|
|
3179
3179
|
style: "pretty",
|
|
@@ -3316,7 +3316,7 @@ function handleClear$3(options) {
|
|
|
3316
3316
|
function renderInput$1(state, submitted) {
|
|
3317
3317
|
const annotation = Option$1.match(state.error, {
|
|
3318
3318
|
onNone: () => combine(underlined, cyanBright),
|
|
3319
|
-
onSome: () => red$
|
|
3319
|
+
onSome: () => red$2
|
|
3320
3320
|
});
|
|
3321
3321
|
const value = state.value === "" ? empty$3 : text$7(`${state.value}`);
|
|
3322
3322
|
return submitted ? value : annotate(value, annotation);
|
|
@@ -3328,8 +3328,8 @@ function renderError$1(state, pointer) {
|
|
|
3328
3328
|
onSome: (error) => Arr.match(error.split(NEWLINE_REGEX$1), {
|
|
3329
3329
|
onEmpty: () => empty$3,
|
|
3330
3330
|
onNonEmpty: (errorLines) => {
|
|
3331
|
-
const annotateLine = (line) => annotate(text$7(line), combine(italicized, red$
|
|
3332
|
-
const prefix = cat(annotate(pointer, red$
|
|
3331
|
+
const annotateLine = (line) => annotate(text$7(line), combine(italicized, red$2));
|
|
3332
|
+
const prefix = cat(annotate(pointer, red$2), space$1);
|
|
3333
3333
|
const lines = Arr.map(errorLines, (str) => annotateLine(str));
|
|
3334
3334
|
return cursorSavePosition.pipe(cat(hardLine), cat(prefix), cat(align(vsep(lines))), cat(cursorRestorePosition));
|
|
3335
3335
|
}
|
|
@@ -3337,7 +3337,7 @@ function renderError$1(state, pointer) {
|
|
|
3337
3337
|
});
|
|
3338
3338
|
}
|
|
3339
3339
|
function renderOutput$3(state, leadingSymbol, trailingSymbol, options, submitted = false) {
|
|
3340
|
-
const annotateLine = (line) => annotate(text$7(line), bold);
|
|
3340
|
+
const annotateLine = (line) => annotate(text$7(line), bold$1);
|
|
3341
3341
|
const prefix = cat(leadingSymbol, space$1);
|
|
3342
3342
|
return Arr.match(options.message.split(/\r?\n/), {
|
|
3343
3343
|
onEmpty: () => hsep([
|
|
@@ -3368,7 +3368,7 @@ function renderSubmission$3(nextState, options) {
|
|
|
3368
3368
|
return Effect$1.gen(function* () {
|
|
3369
3369
|
const columns = yield* (yield* Terminal).columns;
|
|
3370
3370
|
const figures$7 = yield* figures;
|
|
3371
|
-
return renderOutput$3(nextState, annotate(figures$7.tick, green), annotate(figures$7.ellipsis, blackBright), options, true).pipe(cat(hardLine), optimize(Deep), render$1({
|
|
3371
|
+
return renderOutput$3(nextState, annotate(figures$7.tick, green$1), annotate(figures$7.ellipsis, blackBright), options, true).pipe(cat(hardLine), optimize(Deep), render$1({
|
|
3372
3372
|
style: "pretty",
|
|
3373
3373
|
options: { lineWidth: columns }
|
|
3374
3374
|
}));
|
|
@@ -3551,7 +3551,7 @@ const float = (options) => {
|
|
|
3551
3551
|
const renderBeep$2 = /*#__PURE__*/ render$1(beep, { style: "pretty" });
|
|
3552
3552
|
const NEWLINE_REGEX = /\r?\n/;
|
|
3553
3553
|
function renderOutput$2(leadingSymbol, trailingSymbol, options) {
|
|
3554
|
-
const annotateLine = (line) => annotate(text$7(line), bold);
|
|
3554
|
+
const annotateLine = (line) => annotate(text$7(line), bold$1);
|
|
3555
3555
|
const prefix = cat(leadingSymbol, space$1);
|
|
3556
3556
|
return Arr.match(options.message.split(NEWLINE_REGEX), {
|
|
3557
3557
|
onEmpty: () => hsep([prefix, trailingSymbol]),
|
|
@@ -3566,7 +3566,7 @@ function renderChoicePrefix(state, choices, toDisplay, currentIndex, figures) {
|
|
|
3566
3566
|
if (currentIndex === toDisplay.startIndex && toDisplay.startIndex > 0) prefix = figures.arrowUp;
|
|
3567
3567
|
else if (currentIndex === toDisplay.endIndex - 1 && toDisplay.endIndex < choices.length) prefix = figures.arrowDown;
|
|
3568
3568
|
if (choices[currentIndex].disabled) {
|
|
3569
|
-
const annotation = combine(bold, blackBright);
|
|
3569
|
+
const annotation = combine(bold$1, blackBright);
|
|
3570
3570
|
return state === currentIndex ? figures.pointer.pipe(annotate(annotation), cat(prefix)) : prefix.pipe(cat(space$1));
|
|
3571
3571
|
}
|
|
3572
3572
|
return state === currentIndex ? figures.pointer.pipe(annotate(cyanBright), cat(prefix)) : prefix.pipe(cat(space$1));
|
|
@@ -3611,7 +3611,7 @@ function renderSubmission$2(state, options) {
|
|
|
3611
3611
|
const columns = yield* (yield* Terminal).columns;
|
|
3612
3612
|
const figures$6 = yield* figures;
|
|
3613
3613
|
const selected = text$7(options.choices[state].title);
|
|
3614
|
-
return renderOutput$2(annotate(figures$6.tick, green), annotate(figures$6.ellipsis, blackBright), options).pipe(cat(space$1), cat(annotate(selected, white$1)), cat(hardLine), render$1({
|
|
3614
|
+
return renderOutput$2(annotate(figures$6.tick, green$1), annotate(figures$6.ellipsis, blackBright), options).pipe(cat(space$1), cat(annotate(selected, white$1)), cat(hardLine), render$1({
|
|
3615
3615
|
style: "pretty",
|
|
3616
3616
|
options: { lineWidth: columns }
|
|
3617
3617
|
}));
|
|
@@ -3714,7 +3714,7 @@ function renderInput(nextState, options, submitted) {
|
|
|
3714
3714
|
if (nextState.value.length === 0) return blackBright;
|
|
3715
3715
|
return combine(underlined, cyanBright);
|
|
3716
3716
|
},
|
|
3717
|
-
onSome: () => red$
|
|
3717
|
+
onSome: () => red$2
|
|
3718
3718
|
});
|
|
3719
3719
|
switch (options.type) {
|
|
3720
3720
|
case "hidden": return empty$3;
|
|
@@ -3728,8 +3728,8 @@ function renderError(nextState, pointer) {
|
|
|
3728
3728
|
onSome: (error) => Arr.match(error.split(/\r?\n/), {
|
|
3729
3729
|
onEmpty: () => empty$3,
|
|
3730
3730
|
onNonEmpty: (errorLines) => {
|
|
3731
|
-
const annotateLine = (line) => text$7(line).pipe(annotate(combine(italicized, red$
|
|
3732
|
-
const prefix = cat(annotate(pointer, red$
|
|
3731
|
+
const annotateLine = (line) => text$7(line).pipe(annotate(combine(italicized, red$2)));
|
|
3732
|
+
const prefix = cat(annotate(pointer, red$2), space$1);
|
|
3733
3733
|
const lines = Arr.map(errorLines, (str) => annotateLine(str));
|
|
3734
3734
|
return cursorSavePosition.pipe(cat(hardLine), cat(prefix), cat(align(vsep(lines))), cat(cursorRestorePosition));
|
|
3735
3735
|
}
|
|
@@ -3737,7 +3737,7 @@ function renderError(nextState, pointer) {
|
|
|
3737
3737
|
});
|
|
3738
3738
|
}
|
|
3739
3739
|
function renderOutput$1(nextState, leadingSymbol, trailingSymbol, options, submitted = false) {
|
|
3740
|
-
const annotateLine = (line) => annotate(text$7(line), bold);
|
|
3740
|
+
const annotateLine = (line) => annotate(text$7(line), bold$1);
|
|
3741
3741
|
const promptLines = options.message.split(/\r?\n/);
|
|
3742
3742
|
const prefix = cat(leadingSymbol, space$1);
|
|
3743
3743
|
if (Arr.isNonEmptyReadonlyArray(promptLines)) {
|
|
@@ -3767,7 +3767,7 @@ function renderSubmission$1(state, options) {
|
|
|
3767
3767
|
return Effect$1.gen(function* () {
|
|
3768
3768
|
const columns = yield* (yield* Terminal).columns;
|
|
3769
3769
|
const figures$4 = yield* figures;
|
|
3770
|
-
return renderOutput$1(state, annotate(figures$4.tick, green), annotate(figures$4.ellipsis, blackBright), options, true).pipe(cat(hardLine), optimize(Deep), render$1({
|
|
3770
|
+
return renderOutput$1(state, annotate(figures$4.tick, green$1), annotate(figures$4.ellipsis, blackBright), options, true).pipe(cat(hardLine), optimize(Deep), render$1({
|
|
3771
3771
|
style: "pretty",
|
|
3772
3772
|
options: { lineWidth: columns }
|
|
3773
3773
|
}));
|
|
@@ -3909,7 +3909,7 @@ function renderToggle(value, options, submitted = false) {
|
|
|
3909
3909
|
]);
|
|
3910
3910
|
}
|
|
3911
3911
|
function renderOutput(toggle, leadingSymbol, trailingSymbol, options) {
|
|
3912
|
-
const annotateLine = (line) => annotate(text$7(line), bold);
|
|
3912
|
+
const annotateLine = (line) => annotate(text$7(line), bold$1);
|
|
3913
3913
|
const promptLines = options.message.split(/\r?\n/);
|
|
3914
3914
|
const prefix = cat(leadingSymbol, space$1);
|
|
3915
3915
|
if (Arr.isNonEmptyReadonlyArray(promptLines)) {
|
|
@@ -3941,7 +3941,7 @@ function renderSubmission(value, options) {
|
|
|
3941
3941
|
const terminal = yield* Terminal;
|
|
3942
3942
|
const figures$2 = yield* figures;
|
|
3943
3943
|
const columns = yield* terminal.columns;
|
|
3944
|
-
const leadingSymbol = annotate(figures$2.tick, green);
|
|
3944
|
+
const leadingSymbol = annotate(figures$2.tick, green$1);
|
|
3945
3945
|
const trailingSymbol = annotate(figures$2.ellipsis, blackBright);
|
|
3946
3946
|
return renderOutput(renderToggle(value, options, true), leadingSymbol, trailingSymbol, options).pipe(cat(hardLine), optimize(Deep), render$1({
|
|
3947
3947
|
style: "pretty",
|
|
@@ -5919,7 +5919,7 @@ const completionsOptions = /*#__PURE__*/ choiceWithValue("completions", [
|
|
|
5919
5919
|
["zsh", "zsh"]
|
|
5920
5920
|
]).pipe(optional$1, /*#__PURE__*/ withDescription("Generate a completion script for a specific shell."));
|
|
5921
5921
|
/** @internal */
|
|
5922
|
-
const logLevelOptions = /*#__PURE__*/ choiceWithValue("log-level", LogLevel.allLevels.map((level) => [level._tag.toLowerCase(), level])).pipe(optional$1, /*#__PURE__*/ withDescription("Sets the minimum log level for a command."));
|
|
5922
|
+
const logLevelOptions = /*#__PURE__*/ choiceWithValue("log-level", LogLevel$1.allLevels.map((level) => [level._tag.toLowerCase(), level])).pipe(optional$1, /*#__PURE__*/ withDescription("Sets the minimum log level for a command."));
|
|
5923
5923
|
/** @internal */
|
|
5924
5924
|
const helpOptions = /*#__PURE__*/ boolean$1("help").pipe(/*#__PURE__*/ withAlias("h"), /*#__PURE__*/ withDescription("Show the help documentation for a command."));
|
|
5925
5925
|
/** @internal */
|
|
@@ -6294,7 +6294,7 @@ const wizardInternal = (self, prefix, config) => {
|
|
|
6294
6294
|
case "GetUserInput":
|
|
6295
6295
|
case "Standard": return Effect$1.gen(function* () {
|
|
6296
6296
|
const logCurrentCommand = Ref.get(commandLineRef).pipe(Effect$1.flatMap((commandLine) => {
|
|
6297
|
-
const currentCommand = p(pipe(strong(highlight("COMMAND:", cyan)), concat$1(space), concat$1(highlight(Arr.join(commandLine, " "), magenta))));
|
|
6297
|
+
const currentCommand = p(pipe(strong(highlight("COMMAND:", cyan$1)), concat$1(space), concat$1(highlight(Arr.join(commandLine, " "), magenta))));
|
|
6298
6298
|
return Console.log(toAnsiText(currentCommand));
|
|
6299
6299
|
}));
|
|
6300
6300
|
if (isStandard(self)) {
|
|
@@ -6634,7 +6634,7 @@ const handleBuiltInOption = (self, executable, args, builtIn, execute, config, o
|
|
|
6634
6634
|
filteredArgs.push(args[i]);
|
|
6635
6635
|
}
|
|
6636
6636
|
const nextArgs = Arr.appendAll(baseArgs, filteredArgs);
|
|
6637
|
-
return run$2(self, nextArgs, execute).pipe(Logger.withMinimumLogLevel(builtIn.level));
|
|
6637
|
+
return run$2(self, nextArgs, execute).pipe(Logger$1.withMinimumLogLevel(builtIn.level));
|
|
6638
6638
|
}
|
|
6639
6639
|
case "ShowHelp": {
|
|
6640
6640
|
const banner = h1(code(self.name));
|
|
@@ -6740,7 +6740,7 @@ const renderWizardArgs = (args) => {
|
|
|
6740
6740
|
return blocks([
|
|
6741
6741
|
p(strong(code("Wizard Mode Complete!"))),
|
|
6742
6742
|
p(executeMsg),
|
|
6743
|
-
p(concat$1(text$6(" "), highlight(params, cyan)))
|
|
6743
|
+
p(concat$1(text$6(" "), highlight(params, cyan$1)))
|
|
6744
6744
|
]);
|
|
6745
6745
|
};
|
|
6746
6746
|
const isLogLevelArg = (arg) => {
|
|
@@ -7775,7 +7775,7 @@ const ProcessProto = {
|
|
|
7775
7775
|
};
|
|
7776
7776
|
}
|
|
7777
7777
|
};
|
|
7778
|
-
const runCommand = (fileSystem) => (command) => {
|
|
7778
|
+
const runCommand$1 = (fileSystem) => (command) => {
|
|
7779
7779
|
switch (command._tag) {
|
|
7780
7780
|
case "StandardCommand": {
|
|
7781
7781
|
const spawn = Effect$1.flatMap(Deferred.make(), (exitCode) => Effect$1.async((resume) => {
|
|
@@ -7851,13 +7851,13 @@ const runCommand = (fileSystem) => (command) => {
|
|
|
7851
7851
|
}
|
|
7852
7852
|
case "PipedCommand": {
|
|
7853
7853
|
const flattened = flatten(command);
|
|
7854
|
-
if (flattened.length === 1) return pipe(flattened[0], runCommand(fileSystem));
|
|
7854
|
+
if (flattened.length === 1) return pipe(flattened[0], runCommand$1(fileSystem));
|
|
7855
7855
|
const head = flattened[0];
|
|
7856
7856
|
const tail = flattened.slice(1);
|
|
7857
7857
|
const initial = tail.slice(0, tail.length - 1);
|
|
7858
7858
|
const last = tail[tail.length - 1];
|
|
7859
|
-
const stream = initial.reduce((stdin$2, command) => pipe(stdin(command, stdin$2), runCommand(fileSystem), Effect$1.map((process) => process.stdout), Stream.unwrapScoped), pipe(runCommand(fileSystem)(head), Effect$1.map((process) => process.stdout), Stream.unwrapScoped));
|
|
7860
|
-
return pipe(stdin(last, stream), runCommand(fileSystem));
|
|
7859
|
+
const stream = initial.reduce((stdin$2, command) => pipe(stdin(command, stdin$2), runCommand$1(fileSystem), Effect$1.map((process) => process.stdout), Stream.unwrapScoped), pipe(runCommand$1(fileSystem)(head), Effect$1.map((process) => process.stdout), Stream.unwrapScoped));
|
|
7860
|
+
return pipe(stdin(last, stream), runCommand$1(fileSystem));
|
|
7861
7861
|
}
|
|
7862
7862
|
}
|
|
7863
7863
|
};
|
|
@@ -7867,7 +7867,7 @@ const runCommand = (fileSystem) => (command) => {
|
|
|
7867
7867
|
* @since 1.0.0
|
|
7868
7868
|
* @category layer
|
|
7869
7869
|
*/
|
|
7870
|
-
const layer$5 = /* @__PURE__ */ Layer.effect(CommandExecutor, /*#__PURE__*/ pipe(FileSystem, /*#__PURE__*/ Effect$1.map((fileSystem) => makeExecutor(runCommand(fileSystem)))));
|
|
7870
|
+
const layer$5 = /* @__PURE__ */ Layer.effect(CommandExecutor, /*#__PURE__*/ pipe(FileSystem, /*#__PURE__*/ Effect$1.map((fileSystem) => makeExecutor(runCommand$1(fileSystem)))));
|
|
7871
7871
|
//#endregion
|
|
7872
7872
|
//#region ../../node_modules/.bun/@effect+platform-node-shared@0.61.1+1edbab697a2570fc/node_modules/@effect/platform-node-shared/dist/esm/internal/path.js
|
|
7873
7873
|
const fromFileUrl = (url) => Effect$1.try({
|
|
@@ -8048,65 +8048,10 @@ const layerManager = /* @__PURE__ */ Layer.provide(layerManager$2, layerWorker);
|
|
|
8048
8048
|
*/
|
|
8049
8049
|
const layer = /*#__PURE__*/ pipe(/*#__PURE__*/ Layer.mergeAll(layer$3, layer$5, layer$1, layerManager), /*#__PURE__*/ Layer.provideMerge(layer$7));
|
|
8050
8050
|
//#endregion
|
|
8051
|
-
//#region ../cli/src/dashboard-cells.ts
|
|
8052
|
-
const MAX_SCAN = 1e5;
|
|
8053
|
-
function display(value) {
|
|
8054
|
-
return typeof value === "string" ? value : JSON.stringify(value) ?? String(value);
|
|
8055
|
-
}
|
|
8056
|
-
/** Bounded report projection; both count and rows come from the core eligible-cell iterator. */
|
|
8057
|
-
function listMatrixCells(matrix, input) {
|
|
8058
|
-
if (!Number.isSafeInteger(input.offset) || input.offset < 0 || !Number.isSafeInteger(input.limit) || input.limit < 1 || input.limit > 100) throw new Error("Matrix page requires offset >= 0 and limit between 1 and 100");
|
|
8059
|
-
if (![
|
|
8060
|
-
"eval",
|
|
8061
|
-
"agent",
|
|
8062
|
-
"trials",
|
|
8063
|
-
...[...Object.keys(matrix.cases?.[0] ?? {}), ...Object.keys(matrix.parameters)].map((key) => `axis:${key}`)
|
|
8064
|
-
].includes(input.sort)) throw new Error(`Unknown matrix sort: ${input.sort}`);
|
|
8065
|
-
const selection = input.evalIds.length ? { evals: input.evalIds } : {};
|
|
8066
|
-
const planned = matrix.count(selection);
|
|
8067
|
-
if (planned > MAX_SCAN) throw new Error(`Matrix has ${planned} eligible cells; narrow the selection in the CLI (dashboard limit ${MAX_SCAN})`);
|
|
8068
|
-
const terms = input.query.trim().toLowerCase().split(/\s+/).filter(Boolean);
|
|
8069
|
-
const cells = [...matrix.cells(selection)].filter((cell) => {
|
|
8070
|
-
const haystack = [
|
|
8071
|
-
cell.eval.id,
|
|
8072
|
-
cell.eval.name,
|
|
8073
|
-
cell.eval.agent?.identity?.name,
|
|
8074
|
-
cell.eval.agent?.identity?.kind,
|
|
8075
|
-
...Object.entries(cell.parameters).flatMap(([key, value]) => [key, display(value)])
|
|
8076
|
-
].map(display).join(" ").toLowerCase();
|
|
8077
|
-
return terms.every((term) => haystack.includes(term));
|
|
8078
|
-
});
|
|
8079
|
-
function value(cell) {
|
|
8080
|
-
if (input.sort.startsWith("axis:")) {
|
|
8081
|
-
const parameter = cell.parameters[input.sort.slice(5)];
|
|
8082
|
-
return typeof parameter === "number" ? parameter : canonicalParameters(parameter);
|
|
8083
|
-
}
|
|
8084
|
-
if (input.sort === "eval") return cell.eval.name ?? cell.eval.id;
|
|
8085
|
-
if (input.sort === "agent") return cell.eval.agent?.identity?.name ?? cell.eval.agent?.identity?.kind ?? "";
|
|
8086
|
-
return cell.eval.policy?.trials ?? 1;
|
|
8087
|
-
}
|
|
8088
|
-
cells.sort((left, right) => {
|
|
8089
|
-
const a = value(left);
|
|
8090
|
-
const b = value(right);
|
|
8091
|
-
const comparison = typeof a === "number" && typeof b === "number" ? a - b : String(a).localeCompare(String(b), void 0, {
|
|
8092
|
-
numeric: true,
|
|
8093
|
-
sensitivity: "base"
|
|
8094
|
-
});
|
|
8095
|
-
return (input.direction === "asc" ? comparison : -comparison) || left.key.localeCompare(right.key);
|
|
8096
|
-
});
|
|
8097
|
-
return {
|
|
8098
|
-
total: cells.length,
|
|
8099
|
-
planned,
|
|
8100
|
-
cells: cells.slice(input.offset, input.offset + input.limit).map((cell) => ({
|
|
8101
|
-
evalId: cell.eval.id,
|
|
8102
|
-
key: cell.key,
|
|
8103
|
-
parameters: cell.parameters
|
|
8104
|
-
}))
|
|
8105
|
-
};
|
|
8106
|
-
}
|
|
8107
|
-
//#endregion
|
|
8108
8051
|
//#region ../cli/src/dashboard-reports.ts
|
|
8109
8052
|
const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
8053
|
+
/** Written into a run directory by `evalkit tidy`; deleting it restores the run. */
|
|
8054
|
+
const ARCHIVE_MARKER = "archived.json";
|
|
8110
8055
|
function isMissing(error) {
|
|
8111
8056
|
return error.code === "ENOENT";
|
|
8112
8057
|
}
|
|
@@ -8143,15 +8088,35 @@ function fileContentType(file) {
|
|
|
8143
8088
|
".js": "text/javascript"
|
|
8144
8089
|
}[extname(file)] ?? "application/octet-stream";
|
|
8145
8090
|
}
|
|
8146
|
-
/**
|
|
8091
|
+
/**
|
|
8092
|
+
* Read saved local reports. A missing summary means "running"; malformed reports are errors.
|
|
8093
|
+
* Runs written by an older EvalKit (manifest `schemaVersion` < 2) are skipped and counted.
|
|
8094
|
+
*/
|
|
8147
8095
|
function createDashboardReportReader(root, registry) {
|
|
8096
|
+
const legacyRuns = /* @__PURE__ */ new Set();
|
|
8148
8097
|
async function listRuns() {
|
|
8149
8098
|
const entries = await ifPresent(() => readdir(root, { withFileTypes: true }));
|
|
8150
8099
|
if (!entries) return [];
|
|
8151
8100
|
const ids = entries.filter((entry) => entry.isDirectory() && UUID_PATTERN.test(entry.name)).map((entry) => entry.name);
|
|
8152
8101
|
return (await Promise.all(ids.map(readRun))).filter((run) => run !== void 0);
|
|
8153
8102
|
}
|
|
8103
|
+
/** True for a run from a report format this version cannot read; never for corrupt JSON. */
|
|
8104
|
+
async function isLegacyRun(id) {
|
|
8105
|
+
const text = await ifPresent(() => readFile(resolve(root, id, "manifest.json"), "utf8"));
|
|
8106
|
+
if (text === void 0) return false;
|
|
8107
|
+
let version;
|
|
8108
|
+
try {
|
|
8109
|
+
version = JSON.parse(text).schemaVersion;
|
|
8110
|
+
} catch {
|
|
8111
|
+
return false;
|
|
8112
|
+
}
|
|
8113
|
+
return typeof version === "number" && version < 2;
|
|
8114
|
+
}
|
|
8154
8115
|
async function readRun(id) {
|
|
8116
|
+
if (await isLegacyRun(id)) {
|
|
8117
|
+
legacyRuns.add(id);
|
|
8118
|
+
return;
|
|
8119
|
+
}
|
|
8155
8120
|
const manifest = await ifPresent(() => readRunManifest(root, id));
|
|
8156
8121
|
if (!manifest) return void 0;
|
|
8157
8122
|
const summary = await ifPresent(() => readRunSummary(root, id));
|
|
@@ -8160,6 +8125,7 @@ function createDashboardReportReader(root, registry) {
|
|
|
8160
8125
|
const scores = trials.flatMap((trial) => trial?.scoring?.overall === void 0 ? [] : [trial.scoring.overall]);
|
|
8161
8126
|
return {
|
|
8162
8127
|
id,
|
|
8128
|
+
...await ifPresent(() => stat(resolve(root, id, "archived.json"))) ? { archived: true } : {},
|
|
8163
8129
|
evalId: manifest.evalId,
|
|
8164
8130
|
...manifest.suiteId ? { suiteId: manifest.suiteId } : {},
|
|
8165
8131
|
...manifest.matrix ? {
|
|
@@ -8228,6 +8194,8 @@ function createDashboardReportReader(root, registry) {
|
|
|
8228
8194
|
}
|
|
8229
8195
|
return {
|
|
8230
8196
|
listRuns,
|
|
8197
|
+
/** Run IDs skipped by the last `listRuns()` because an older EvalKit wrote them. */
|
|
8198
|
+
legacyRunIds: () => [...legacyRuns],
|
|
8231
8199
|
listTrials,
|
|
8232
8200
|
trialDetail,
|
|
8233
8201
|
listArtifacts,
|
|
@@ -8259,6 +8227,235 @@ async function walkArtifacts(root) {
|
|
|
8259
8227
|
return result;
|
|
8260
8228
|
}
|
|
8261
8229
|
//#endregion
|
|
8230
|
+
//#region ../cli/src/dashboard-build.ts
|
|
8231
|
+
/** Marks a directory as EvalKit-owned so a rebuild may safely replace it. */
|
|
8232
|
+
const MARKER = ".evalkit-dashboard";
|
|
8233
|
+
const quietStep = (_label, work) => work(() => {});
|
|
8234
|
+
async function buildDashboard(options, step = quietStep) {
|
|
8235
|
+
const { registry, reportRoot, dashboardRoot, outDir } = options;
|
|
8236
|
+
const template = await readFile(resolve(dashboardRoot, "index.html"), "utf8");
|
|
8237
|
+
const data = resolve(outDir, "data");
|
|
8238
|
+
const write = async (path, value) => {
|
|
8239
|
+
const file = resolve(data, path);
|
|
8240
|
+
await mkdir(dirname(file), { recursive: true });
|
|
8241
|
+
await writeFile(file, `${JSON.stringify(value)}\n`);
|
|
8242
|
+
};
|
|
8243
|
+
await step("Dashboard assets", async () => {
|
|
8244
|
+
await prepareOutDir(outDir);
|
|
8245
|
+
await cp(dashboardRoot, outDir, { recursive: true });
|
|
8246
|
+
await writeFile(resolve(outDir, MARKER), "Generated by `evalkit build`.\n");
|
|
8247
|
+
await writeFile(resolve(outDir, "index.html"), staticIndex(template));
|
|
8248
|
+
}, () => "read-only UI with relative assets");
|
|
8249
|
+
await step("Eval catalog", async () => {
|
|
8250
|
+
await write("catalog.json", { evals: registry.catalog() });
|
|
8251
|
+
await write("evals.json", { evals: registry.metadata() });
|
|
8252
|
+
await write("suites.json", { suites: registry.suiteMetadata() });
|
|
8253
|
+
return {
|
|
8254
|
+
evals: registry.evals.length,
|
|
8255
|
+
suites: registry.suites.length
|
|
8256
|
+
};
|
|
8257
|
+
}, ({ evals, suites }) => `${count(evals, "eval")} · ${count(suites, "suite")}`);
|
|
8258
|
+
const reports = createDashboardReportReader(reportRoot, registry);
|
|
8259
|
+
const snapshot = await step("Run reports", async (progress) => {
|
|
8260
|
+
const all = await reports.listRuns();
|
|
8261
|
+
const runs = all.filter((run) => !run.archived);
|
|
8262
|
+
await write("runs.json", { runs });
|
|
8263
|
+
let trials = 0;
|
|
8264
|
+
let files = 0;
|
|
8265
|
+
const trialIds = [];
|
|
8266
|
+
for (const [index, run] of runs.entries()) {
|
|
8267
|
+
progress(`run ${index + 1}/${runs.length} · ${run.id.slice(0, 8)}`);
|
|
8268
|
+
const runTrials = await reports.listTrials(run.id);
|
|
8269
|
+
await write(`runs/${run.id}/trials.json`, { trials: runTrials });
|
|
8270
|
+
for (const trial of runTrials) {
|
|
8271
|
+
trials += 1;
|
|
8272
|
+
trialIds.push(trial.id);
|
|
8273
|
+
files += await snapshotTrial(reports, write, data, run.id, trial.id);
|
|
8274
|
+
}
|
|
8275
|
+
}
|
|
8276
|
+
return {
|
|
8277
|
+
runs: runs.length,
|
|
8278
|
+
trials,
|
|
8279
|
+
files,
|
|
8280
|
+
skipped: reports.legacyRunIds().length,
|
|
8281
|
+
archived: all.length - runs.length,
|
|
8282
|
+
runIds: runs.map((run) => run.id),
|
|
8283
|
+
trialIds
|
|
8284
|
+
};
|
|
8285
|
+
}, ({ runs, trials, files, skipped, archived }) => `${count(runs, "run")} · ${count(trials, "trial")} · ${count(files, "candidate file")}` + (archived ? ` · ${archived} archived excluded` : "") + (skipped ? ` · ${skipped} older-format skipped` : ""));
|
|
8286
|
+
await step("Route pages", async () => {
|
|
8287
|
+
const routes = staticRoutes({
|
|
8288
|
+
suites: registry.suites.map(authoringId),
|
|
8289
|
+
evals: registry.catalog().map((entry) => entry.path),
|
|
8290
|
+
runs: snapshot.runIds,
|
|
8291
|
+
trials: snapshot.trialIds
|
|
8292
|
+
});
|
|
8293
|
+
for (const route of routes) {
|
|
8294
|
+
const file = resolve(outDir, `${route}.html`);
|
|
8295
|
+
await mkdir(dirname(file), { recursive: true });
|
|
8296
|
+
await writeFile(file, staticIndex(template, baseFor(route)));
|
|
8297
|
+
}
|
|
8298
|
+
await writeFile(resolve(outDir, "404.html"), staticIndex(template, "/"));
|
|
8299
|
+
return routes.length;
|
|
8300
|
+
}, (pages) => `${count(pages, "page")} with clean /path URLs`);
|
|
8301
|
+
const { runIds: _runs, trialIds: _trials, ...summary } = snapshot;
|
|
8302
|
+
return {
|
|
8303
|
+
outDir,
|
|
8304
|
+
...summary
|
|
8305
|
+
};
|
|
8306
|
+
}
|
|
8307
|
+
/**
|
|
8308
|
+
* Every dashboard URL known at build time, as file paths without `.html`.
|
|
8309
|
+
* Static hosts serve `/runs/abc` from `runs/abc.html`, so links and reloads
|
|
8310
|
+
* work without SPA rewrite rules. IDs that cannot be a safe file name are
|
|
8311
|
+
* skipped; those pages still open by navigating inside the app.
|
|
8312
|
+
*/
|
|
8313
|
+
function staticRoutes(ids) {
|
|
8314
|
+
const safe = (id) => id.length > 0 && !/[/\\\0]/.test(id) && id !== "." && id !== "..";
|
|
8315
|
+
const under = (prefix, values) => values.filter(safe).map((id) => `${prefix}/${id}`);
|
|
8316
|
+
return [
|
|
8317
|
+
"suites",
|
|
8318
|
+
"agents",
|
|
8319
|
+
"fixtures",
|
|
8320
|
+
"runs",
|
|
8321
|
+
...under("suites", ids.suites),
|
|
8322
|
+
...under("evals", ids.evals),
|
|
8323
|
+
...under("runs", ids.runs),
|
|
8324
|
+
...under("trials", ids.trials),
|
|
8325
|
+
...under("workspaces", ids.trials)
|
|
8326
|
+
];
|
|
8327
|
+
}
|
|
8328
|
+
/** `runs/abc.html` is served at `<mount>/runs/abc`; its base is one level up. */
|
|
8329
|
+
function baseFor(route) {
|
|
8330
|
+
const depth = route.split("/").length - 1;
|
|
8331
|
+
return depth ? "../".repeat(depth) : "./";
|
|
8332
|
+
}
|
|
8333
|
+
function count(value, noun) {
|
|
8334
|
+
return `${value} ${noun}${value === 1 ? "" : "s"}`;
|
|
8335
|
+
}
|
|
8336
|
+
/** Writes one trial's JSON views and candidate files; returns the file count. */
|
|
8337
|
+
async function snapshotTrial(reports, write, data, runId, trialId) {
|
|
8338
|
+
const base = `runs/${runId}/trials/${trialId}`;
|
|
8339
|
+
await write(`${base}/detail.json`, await reports.trialDetail(runId, trialId));
|
|
8340
|
+
await write(`${base}/events.json`, { events: await reports.trajectory(runId, trialId) });
|
|
8341
|
+
await write(`${base}/artifacts.json`, { artifacts: await reports.listArtifacts(runId, trialId) });
|
|
8342
|
+
const workspace = await reports.listCandidateWorkspace(runId, trialId);
|
|
8343
|
+
await write(`${base}/workspace.json`, workspace);
|
|
8344
|
+
const root = resolve(data, base, "workspace");
|
|
8345
|
+
let files = 0;
|
|
8346
|
+
for (const entry of workspace) {
|
|
8347
|
+
if (entry.kind !== "file") continue;
|
|
8348
|
+
const target = resolve(root, entry.path);
|
|
8349
|
+
if (!target.startsWith(`${root}${sep}`)) throw new Error(`Invalid workspace path: ${entry.path}`);
|
|
8350
|
+
const response = await reports.candidateFile(runId, trialId, entry.path);
|
|
8351
|
+
await mkdir(dirname(target), { recursive: true });
|
|
8352
|
+
await writeFile(target, new Uint8Array(await response.arrayBuffer()));
|
|
8353
|
+
files += 1;
|
|
8354
|
+
}
|
|
8355
|
+
return files;
|
|
8356
|
+
}
|
|
8357
|
+
/** Total size and file count of a directory, for the build summary. */
|
|
8358
|
+
async function directorySize(root) {
|
|
8359
|
+
let bytes = 0;
|
|
8360
|
+
let files = 0;
|
|
8361
|
+
for (const entry of await readdir(root, {
|
|
8362
|
+
withFileTypes: true,
|
|
8363
|
+
recursive: true
|
|
8364
|
+
})) {
|
|
8365
|
+
if (!entry.isFile()) continue;
|
|
8366
|
+
bytes += (await stat(resolve(entry.parentPath, entry.name))).size;
|
|
8367
|
+
files += 1;
|
|
8368
|
+
}
|
|
8369
|
+
return {
|
|
8370
|
+
bytes,
|
|
8371
|
+
files
|
|
8372
|
+
};
|
|
8373
|
+
}
|
|
8374
|
+
/**
|
|
8375
|
+
* Select the static API and make every URL relative to `<base href>`, which
|
|
8376
|
+
* points at the snapshot root. The app derives its router basename from it.
|
|
8377
|
+
*/
|
|
8378
|
+
function staticIndex(html, base = "./") {
|
|
8379
|
+
const result = html.replace("<html lang=\"en\">", "<html lang=\"en\" data-evalkit-snapshot=\"static\">").replace("<head>", `<head>\n <base href="${base}" />`).replaceAll("\"/assets/", "\"./assets/");
|
|
8380
|
+
if (!result.includes("data-evalkit-snapshot=\"static\"") || !result.includes("<base href=")) throw new Error("Bundled dashboard index.html has an unexpected shape");
|
|
8381
|
+
return result;
|
|
8382
|
+
}
|
|
8383
|
+
async function prepareOutDir(outDir) {
|
|
8384
|
+
let entries;
|
|
8385
|
+
try {
|
|
8386
|
+
if (!(await stat(outDir)).isDirectory()) throw new Error(`Output path is not a directory: ${outDir}`);
|
|
8387
|
+
entries = await readdir(outDir);
|
|
8388
|
+
} catch (error) {
|
|
8389
|
+
if (error.code !== "ENOENT") throw error;
|
|
8390
|
+
await mkdir(outDir, { recursive: true });
|
|
8391
|
+
return;
|
|
8392
|
+
}
|
|
8393
|
+
if (!entries.length) return;
|
|
8394
|
+
if (!entries.includes(MARKER)) throw new Error(`Refusing to overwrite ${outDir}: it is not empty and was not created by evalkit build`);
|
|
8395
|
+
await rm(outDir, {
|
|
8396
|
+
recursive: true,
|
|
8397
|
+
force: true
|
|
8398
|
+
});
|
|
8399
|
+
await mkdir(outDir, { recursive: true });
|
|
8400
|
+
}
|
|
8401
|
+
//#endregion
|
|
8402
|
+
//#region ../cli/src/dashboard-cells.ts
|
|
8403
|
+
const MAX_SCAN = 1e5;
|
|
8404
|
+
function display(value) {
|
|
8405
|
+
return typeof value === "string" ? value : JSON.stringify(value) ?? String(value);
|
|
8406
|
+
}
|
|
8407
|
+
/** Bounded report projection; both count and rows come from the core eligible-cell iterator. */
|
|
8408
|
+
function listMatrixCells(matrix, input) {
|
|
8409
|
+
if (!Number.isSafeInteger(input.offset) || input.offset < 0 || !Number.isSafeInteger(input.limit) || input.limit < 1 || input.limit > 100) throw new Error("Matrix page requires offset >= 0 and limit between 1 and 100");
|
|
8410
|
+
if (![
|
|
8411
|
+
"eval",
|
|
8412
|
+
"agent",
|
|
8413
|
+
"trials",
|
|
8414
|
+
...[...Object.keys(matrix.cases?.[0] ?? {}), ...Object.keys(matrix.parameters)].map((key) => `axis:${key}`)
|
|
8415
|
+
].includes(input.sort)) throw new Error(`Unknown matrix sort: ${input.sort}`);
|
|
8416
|
+
const selection = input.evalIds.length ? { evals: input.evalIds } : {};
|
|
8417
|
+
const planned = matrix.count(selection);
|
|
8418
|
+
if (planned > MAX_SCAN) throw new Error(`Matrix has ${planned} eligible cells; narrow the selection in the CLI (dashboard limit ${MAX_SCAN})`);
|
|
8419
|
+
const terms = input.query.trim().toLowerCase().split(/\s+/).filter(Boolean);
|
|
8420
|
+
const cells = [...matrix.cells(selection)].filter((cell) => {
|
|
8421
|
+
const haystack = [
|
|
8422
|
+
cell.eval.id,
|
|
8423
|
+
cell.eval.name,
|
|
8424
|
+
cell.eval.agent?.identity?.name,
|
|
8425
|
+
cell.eval.agent?.identity?.kind,
|
|
8426
|
+
...Object.entries(cell.parameters).flatMap(([key, value]) => [key, display(value)])
|
|
8427
|
+
].map(display).join(" ").toLowerCase();
|
|
8428
|
+
return terms.every((term) => haystack.includes(term));
|
|
8429
|
+
});
|
|
8430
|
+
function value(cell) {
|
|
8431
|
+
if (input.sort.startsWith("axis:")) {
|
|
8432
|
+
const parameter = cell.parameters[input.sort.slice(5)];
|
|
8433
|
+
return typeof parameter === "number" ? parameter : canonicalParameters(parameter);
|
|
8434
|
+
}
|
|
8435
|
+
if (input.sort === "eval") return cell.eval.name ?? cell.eval.id;
|
|
8436
|
+
if (input.sort === "agent") return cell.eval.agent?.identity?.name ?? cell.eval.agent?.identity?.kind ?? "";
|
|
8437
|
+
return cell.eval.policy?.trials ?? 1;
|
|
8438
|
+
}
|
|
8439
|
+
cells.sort((left, right) => {
|
|
8440
|
+
const a = value(left);
|
|
8441
|
+
const b = value(right);
|
|
8442
|
+
const comparison = typeof a === "number" && typeof b === "number" ? a - b : String(a).localeCompare(String(b), void 0, {
|
|
8443
|
+
numeric: true,
|
|
8444
|
+
sensitivity: "base"
|
|
8445
|
+
});
|
|
8446
|
+
return (input.direction === "asc" ? comparison : -comparison) || left.key.localeCompare(right.key);
|
|
8447
|
+
});
|
|
8448
|
+
return {
|
|
8449
|
+
total: cells.length,
|
|
8450
|
+
planned,
|
|
8451
|
+
cells: cells.slice(input.offset, input.offset + input.limit).map((cell) => ({
|
|
8452
|
+
evalId: cell.eval.id,
|
|
8453
|
+
key: cell.key,
|
|
8454
|
+
parameters: cell.parameters
|
|
8455
|
+
}))
|
|
8456
|
+
};
|
|
8457
|
+
}
|
|
8458
|
+
//#endregion
|
|
8262
8459
|
//#region ../cli/src/dashboard-matrix.ts
|
|
8263
8460
|
/** Dashboard submissions are intentionally one explicit, configured cell at a time. */
|
|
8264
8461
|
function selectDashboardCell(matrix, evalId, parameters) {
|
|
@@ -8376,6 +8573,111 @@ async function loadProject(cwd, configPath) {
|
|
|
8376
8573
|
};
|
|
8377
8574
|
}
|
|
8378
8575
|
//#endregion
|
|
8576
|
+
//#region ../cli/src/terminal.ts
|
|
8577
|
+
/** Human CLI output. Colors and live progress only appear on an interactive terminal. */
|
|
8578
|
+
const interactive = Boolean(process.stdout.isTTY);
|
|
8579
|
+
const colored = interactive && !process.env.NO_COLOR && process.env.TERM !== "dumb";
|
|
8580
|
+
const paint = (code, reset) => (text) => colored ? `\x1b[${code}m${text}\x1b[${reset}m` : text;
|
|
8581
|
+
const bold = paint(1, 22);
|
|
8582
|
+
const dim = paint(2, 22);
|
|
8583
|
+
const red = paint(31, 39);
|
|
8584
|
+
const green = paint(32, 39);
|
|
8585
|
+
const yellow = paint(33, 39);
|
|
8586
|
+
const cyan = paint(36, 39);
|
|
8587
|
+
function banner(command, version) {
|
|
8588
|
+
console.log(`\n ${bold(cyan("EvalKit"))} ${bold(command)} ${dim(`v${version}`)}\n`);
|
|
8589
|
+
}
|
|
8590
|
+
function duration(ms) {
|
|
8591
|
+
return ms < 1e3 ? `${Math.round(ms)}ms` : `${(ms / 1e3).toFixed(2)}s`;
|
|
8592
|
+
}
|
|
8593
|
+
function bytes(size) {
|
|
8594
|
+
if (size < 1024) return `${size} B`;
|
|
8595
|
+
if (size < 1024 ** 2) return `${(size / 1024).toFixed(1)} kB`;
|
|
8596
|
+
return `${(size / 1024 ** 2).toFixed(1)} MB`;
|
|
8597
|
+
}
|
|
8598
|
+
function plural(count, noun) {
|
|
8599
|
+
return `${count} ${noun}${count === 1 ? "" : "s"}`;
|
|
8600
|
+
}
|
|
8601
|
+
/** Times one build step, showing a live line while it runs. */
|
|
8602
|
+
async function step(label, work, describe) {
|
|
8603
|
+
const started = performance.now();
|
|
8604
|
+
const progress = (detail) => {
|
|
8605
|
+
if (interactive) process.stdout.write(`\r\x1b[2K ${cyan("◇")} ${label.padEnd(20)} ${dim(detail)}`);
|
|
8606
|
+
};
|
|
8607
|
+
progress("…");
|
|
8608
|
+
try {
|
|
8609
|
+
const value = await work(progress);
|
|
8610
|
+
const elapsed = duration(performance.now() - started);
|
|
8611
|
+
if (interactive) process.stdout.write("\r\x1B[2K");
|
|
8612
|
+
console.log(` ${green("✓")} ${label.padEnd(20)} ${describe(value)} ${dim(elapsed)}`);
|
|
8613
|
+
return value;
|
|
8614
|
+
} catch (error) {
|
|
8615
|
+
if (interactive) process.stdout.write("\r\x1B[2K");
|
|
8616
|
+
console.log(` ${red("✗")} ${label}`);
|
|
8617
|
+
throw error;
|
|
8618
|
+
}
|
|
8619
|
+
}
|
|
8620
|
+
function clock() {
|
|
8621
|
+
return dim((/* @__PURE__ */ new Date()).toTimeString().slice(0, 8));
|
|
8622
|
+
}
|
|
8623
|
+
function cellLabel(name, parameters) {
|
|
8624
|
+
const shown = Object.entries(parameters).map(([key, value]) => `${key}=${typeof value === "string" ? value : JSON.stringify(value)}`).join(" ");
|
|
8625
|
+
return shown ? `${bold(name)} ${dim(shown)}` : bold(name);
|
|
8626
|
+
}
|
|
8627
|
+
function passed(result) {
|
|
8628
|
+
return result.status === "completed" && (result.aggregateScoring?.passRate === 1 || Boolean(result.scoring?.passed));
|
|
8629
|
+
}
|
|
8630
|
+
/** Live log for runs launched from the `evalkit dev` dashboard. */
|
|
8631
|
+
const runLog = {
|
|
8632
|
+
started(name, parameters = {}, suiteId) {
|
|
8633
|
+
const suite = suiteId ? dim(` (suite ${suiteId})`) : "";
|
|
8634
|
+
console.log(`${clock()} ${cyan("▶ run ")} ${cellLabel(name, parameters)}${suite}`);
|
|
8635
|
+
},
|
|
8636
|
+
finished(name, parameters, result, ms) {
|
|
8637
|
+
const ok = passed(result);
|
|
8638
|
+
const trials = result.trials?.length ?? 1;
|
|
8639
|
+
const passedTrials = result.aggregateScoring?.passed ?? (ok ? trials : 0);
|
|
8640
|
+
const status = result.status === "completed" ? ok ? green("✓ pass") : red("✗ fail") : red(`✗ ${result.status}`);
|
|
8641
|
+
console.log(`${clock()} ${status} ${cellLabel(name, parameters)} ${dim(`${passedTrials}/${trials} trials · ${duration(ms)}`)}`);
|
|
8642
|
+
console.log(` ${dim(`report ${result.reportLocation}`)}`);
|
|
8643
|
+
},
|
|
8644
|
+
cell(cell, result, ms) {
|
|
8645
|
+
runLog.finished(cell.eval.name ?? authoringId(cell.eval), cell.parameters, result, ms);
|
|
8646
|
+
},
|
|
8647
|
+
failed(name, error) {
|
|
8648
|
+
console.log(`${clock()} ${red("✗ error")} ${bold(name)} ${error instanceof Error ? error.message : String(error)}`);
|
|
8649
|
+
}
|
|
8650
|
+
};
|
|
8651
|
+
/**
|
|
8652
|
+
* The runner logs structured Effect lines; the CLI prints its own summaries, so
|
|
8653
|
+
* only warnings show by default. `EVALKIT_LOG_LEVEL=info` (or `debug`) restores them.
|
|
8654
|
+
*/
|
|
8655
|
+
function runnerLogs(effect) {
|
|
8656
|
+
const requested = process.env.EVALKIT_LOG_LEVEL?.toLowerCase();
|
|
8657
|
+
const level = LogLevel.allLevels.find((item) => item.label.toLowerCase() === requested) ?? LogLevel.Warning;
|
|
8658
|
+
return effect.pipe(Logger.withMinimumLogLevel(level));
|
|
8659
|
+
}
|
|
8660
|
+
/** True when a person can answer a prompt (not CI, pipes, or redirected input). */
|
|
8661
|
+
function canPrompt() {
|
|
8662
|
+
return Boolean(process.stdin.isTTY && process.stdout.isTTY);
|
|
8663
|
+
}
|
|
8664
|
+
async function confirm(question) {
|
|
8665
|
+
const { createInterface } = await import("node:readline/promises");
|
|
8666
|
+
const prompt = createInterface({
|
|
8667
|
+
input: process.stdin,
|
|
8668
|
+
output: process.stdout
|
|
8669
|
+
});
|
|
8670
|
+
try {
|
|
8671
|
+
const answer = await prompt.question(` ${cyan("?")} ${question} ${dim("(y/N)")} `);
|
|
8672
|
+
return /^y(es)?$/i.test(answer.trim());
|
|
8673
|
+
} catch {
|
|
8674
|
+
console.log("");
|
|
8675
|
+
return false;
|
|
8676
|
+
} finally {
|
|
8677
|
+
prompt.close();
|
|
8678
|
+
}
|
|
8679
|
+
}
|
|
8680
|
+
//#endregion
|
|
8379
8681
|
//#region ../cli/src/run-command.ts
|
|
8380
8682
|
function parseRunArgs(args) {
|
|
8381
8683
|
const { values, positionals } = parseArgs({
|
|
@@ -8388,6 +8690,8 @@ function parseRunArgs(args) {
|
|
|
8388
8690
|
type: "string",
|
|
8389
8691
|
multiple: true
|
|
8390
8692
|
},
|
|
8693
|
+
suite: { type: "string" },
|
|
8694
|
+
matrix: { type: "string" },
|
|
8391
8695
|
model: {
|
|
8392
8696
|
type: "string",
|
|
8393
8697
|
multiple: true
|
|
@@ -8454,6 +8758,8 @@ function normalizeRunOptions(values, positionals) {
|
|
|
8454
8758
|
if (index < 1) throw new Error("--param requires name=JSON");
|
|
8455
8759
|
parameters[value.slice(0, index)] = JSON.parse(value.slice(index + 1));
|
|
8456
8760
|
}
|
|
8761
|
+
if (values.suite && values.matrix) throw new Error("--suite and --matrix cannot be used together");
|
|
8762
|
+
if ((values.suite || values.matrix) && positionals.length) throw new Error("Use --eval to select evals with --suite or --matrix");
|
|
8457
8763
|
return {
|
|
8458
8764
|
values,
|
|
8459
8765
|
positionals,
|
|
@@ -8463,7 +8769,7 @@ function normalizeRunOptions(values, positionals) {
|
|
|
8463
8769
|
trials: integer("trials")
|
|
8464
8770
|
};
|
|
8465
8771
|
}
|
|
8466
|
-
async function runProjectCommand(
|
|
8772
|
+
async function runProjectCommand(args, cwd = process.cwd(), parsed = parseRunArgs(args)) {
|
|
8467
8773
|
const { values, positionals, parameters } = parsed;
|
|
8468
8774
|
const project = await loadProject(cwd, values.config);
|
|
8469
8775
|
process.chdir(project.root);
|
|
@@ -8471,14 +8777,9 @@ async function runProjectCommand(command, args, cwd = process.cwd(), parsed = pa
|
|
|
8471
8777
|
const requestedEvals = values.eval?.flatMap((value) => value.split(",")) ?? [];
|
|
8472
8778
|
let matrix;
|
|
8473
8779
|
let suiteId;
|
|
8474
|
-
if (
|
|
8475
|
-
|
|
8476
|
-
|
|
8477
|
-
if (!matrix) throw new Error(`Unknown matrix: ${positionals[0]}`);
|
|
8478
|
-
} else if (command === "run-suite") {
|
|
8479
|
-
if (positionals.length !== 1) throw new Error("run-suite requires one suite ID");
|
|
8480
|
-
const suite = registry.suites.find((s) => s.id === positionals[0]);
|
|
8481
|
-
if (!suite) throw new Error(`Unknown suite: ${positionals[0]}`);
|
|
8780
|
+
if (values.suite) {
|
|
8781
|
+
const suite = registry.suites.find((item) => item.id === values.suite);
|
|
8782
|
+
if (!suite) throw new Error(`Unknown suite: ${values.suite}`);
|
|
8482
8783
|
suiteId = authoringId(suite);
|
|
8483
8784
|
matrix = defineEvalMatrix({
|
|
8484
8785
|
id: "default",
|
|
@@ -8488,6 +8789,9 @@ async function runProjectCommand(command, args, cwd = process.cwd(), parsed = pa
|
|
|
8488
8789
|
cases: project.config.matrix?.cases,
|
|
8489
8790
|
exclude: project.config.matrix?.exclude
|
|
8490
8791
|
});
|
|
8792
|
+
} else if (values.matrix) {
|
|
8793
|
+
matrix = registry.matrices.find((item) => item.id === values.matrix);
|
|
8794
|
+
if (!matrix) throw new Error(`Unknown matrix: ${values.matrix}`);
|
|
8491
8795
|
} else {
|
|
8492
8796
|
requestedEvals.push(...positionals.flatMap((value) => value.split(",")));
|
|
8493
8797
|
matrix = project.config.matrix ? registry.matrices.at(-1) : defineEvalMatrix({
|
|
@@ -8502,7 +8806,7 @@ async function runProjectCommand(command, args, cwd = process.cwd(), parsed = pa
|
|
|
8502
8806
|
...requestedEvals.length ? { evals: requestedEvals } : {}
|
|
8503
8807
|
};
|
|
8504
8808
|
for (const [axis, choices] of Object.entries(parsed.selection)) if (axis in matrix.parameters || matrix.cases?.some((assignment) => Object.hasOwn(assignment, axis))) selection.parameters[axis] = choices;
|
|
8505
|
-
else if ((axis === "model" || axis === "mode") && choices.length === 1 && !project.config.matrix &&
|
|
8809
|
+
else if ((axis === "model" || axis === "mode") && choices.length === 1 && !project.config.matrix && !values.matrix) selection.overrides[axis] = choices[0];
|
|
8506
8810
|
else throw new Error(`Unknown matrix axis: ${axis}`);
|
|
8507
8811
|
const cells = matrix.count(selection);
|
|
8508
8812
|
if (!cells) throw new Error("Selection contains no eligible cells; check cases and exclusions");
|
|
@@ -8521,7 +8825,7 @@ async function runProjectCommand(command, args, cwd = process.cwd(), parsed = pa
|
|
|
8521
8825
|
}
|
|
8522
8826
|
if (cells > (project.config.execution?.maxCells ?? 100) && !values.all) throw new Error(`Selected ${cells} cells. Narrow the selection, inspect --dry-run, or pass --all.`);
|
|
8523
8827
|
if (!values.json) console.log(`Running ${cells} cell(s) from ${plan.matrix}`);
|
|
8524
|
-
if ((await Effect.runPromise(runMatrix(matrix, {
|
|
8828
|
+
if ((await Effect.runPromise(runnerLogs(runMatrix(matrix, {
|
|
8525
8829
|
selection,
|
|
8526
8830
|
concurrency: parsed.concurrency ?? project.config.execution?.concurrency ?? 4,
|
|
8527
8831
|
trials,
|
|
@@ -8547,7 +8851,56 @@ async function runProjectCommand(command, args, cwd = process.cwd(), parsed = pa
|
|
|
8547
8851
|
}
|
|
8548
8852
|
}
|
|
8549
8853
|
}
|
|
8550
|
-
}))).failed) process.exitCode = 1;
|
|
8854
|
+
})))).failed) process.exitCode = 1;
|
|
8855
|
+
}
|
|
8856
|
+
//#endregion
|
|
8857
|
+
//#region ../cli/src/tidy.ts
|
|
8858
|
+
/**
|
|
8859
|
+
* Every eval the project can run: standalone, in suites, or only in matrices.
|
|
8860
|
+
* A run is archived only when its eval is in none of these.
|
|
8861
|
+
*/
|
|
8862
|
+
function projectEvalIds(registry) {
|
|
8863
|
+
return new Set([
|
|
8864
|
+
...registry.evals,
|
|
8865
|
+
...registry.suites.flatMap((suite) => suite.evals),
|
|
8866
|
+
...registry.matrices.flatMap((matrix) => matrix.evals)
|
|
8867
|
+
].map(authoringId));
|
|
8868
|
+
}
|
|
8869
|
+
/**
|
|
8870
|
+
* Mark runs of removed evals as archived, and restore runs whose eval returned.
|
|
8871
|
+
* Reports are never deleted: archiving writes a marker file beside the manifest.
|
|
8872
|
+
*/
|
|
8873
|
+
async function tidyReports(options) {
|
|
8874
|
+
const known = projectEvalIds(options.registry);
|
|
8875
|
+
const reports = createDashboardReportReader(options.reportRoot, options.registry);
|
|
8876
|
+
const result = {
|
|
8877
|
+
archived: [],
|
|
8878
|
+
restored: [],
|
|
8879
|
+
stillArchived: 0,
|
|
8880
|
+
current: 0
|
|
8881
|
+
};
|
|
8882
|
+
for (const run of await reports.listRuns()) {
|
|
8883
|
+
const marker = resolve(options.reportRoot, run.id, ARCHIVE_MARKER);
|
|
8884
|
+
const entry = {
|
|
8885
|
+
runId: run.id,
|
|
8886
|
+
evalId: run.evalId
|
|
8887
|
+
};
|
|
8888
|
+
if (known.has(run.evalId)) {
|
|
8889
|
+
result.current += 1;
|
|
8890
|
+
if (run.archived) {
|
|
8891
|
+
await rm(marker, { force: true });
|
|
8892
|
+
result.restored.push(entry);
|
|
8893
|
+
}
|
|
8894
|
+
} else if (run.archived) result.stillArchived += 1;
|
|
8895
|
+
else {
|
|
8896
|
+
await writeFile(marker, `${JSON.stringify({
|
|
8897
|
+
archivedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
8898
|
+
reason: `Eval "${run.evalId}" is no longer in the project`
|
|
8899
|
+
})}\n`);
|
|
8900
|
+
result.archived.push(entry);
|
|
8901
|
+
}
|
|
8902
|
+
}
|
|
8903
|
+
return result;
|
|
8551
8904
|
}
|
|
8552
8905
|
//#endregion
|
|
8553
8906
|
//#region ../cli/src/new-project-template.ts
|
|
@@ -8653,21 +9006,23 @@ bun run check # type-check your eval, agent, scorer, and config
|
|
|
8653
9006
|
bun run evals # run the example across both styles
|
|
8654
9007
|
bun run matrix:plan # inspect the matrix without running it
|
|
8655
9008
|
bun run matrix # run the configured matrix
|
|
8656
|
-
bun run dashboard # inspect local reports
|
|
9009
|
+
bun run dashboard # inspect local reports and launch runs
|
|
9010
|
+
bun run dashboard:build # write a read-only, shareable snapshot
|
|
8657
9011
|
\`\`\`
|
|
8658
9012
|
|
|
8659
|
-
EvalKit installs from
|
|
8660
|
-
|
|
8661
|
-
|
|
9013
|
+
EvalKit installs from npm. Reports, trial workspaces, and dashboard snapshots stay local
|
|
9014
|
+
and are ignored by Git. A snapshot in \`_evalkit-dashboard/\` contains report messages,
|
|
9015
|
+
scores, evidence, and candidate files (never evaluator workspaces): review it before sharing.
|
|
8662
9016
|
`;
|
|
8663
9017
|
}
|
|
8664
9018
|
//#endregion
|
|
8665
9019
|
//#region ../cli/src/new-project.ts
|
|
8666
|
-
const npmDependency = "^0.
|
|
9020
|
+
const npmDependency = "^0.3.0";
|
|
8667
9021
|
const ignored = [
|
|
8668
9022
|
"node_modules/",
|
|
8669
9023
|
"_evalkit-results/",
|
|
8670
|
-
"_evalkit-sandbox/"
|
|
9024
|
+
"_evalkit-sandbox/",
|
|
9025
|
+
"_evalkit-dashboard/"
|
|
8671
9026
|
];
|
|
8672
9027
|
const readIfExists = async (path) => {
|
|
8673
9028
|
try {
|
|
@@ -8746,10 +9101,11 @@ async function preparePackage(packagePath, name) {
|
|
|
8746
9101
|
const devDependencies = pkg.devDependencies ?? {};
|
|
8747
9102
|
if (!scripts || typeof scripts !== "object" || Array.isArray(scripts) || !dependencies || typeof dependencies !== "object" || Array.isArray(dependencies) || !devDependencies || typeof devDependencies !== "object" || Array.isArray(devDependencies)) throw new Error(`Invalid package.json scripts/dependencies: ${packagePath}`);
|
|
8748
9103
|
scripts.check ??= "tsc --noEmit";
|
|
8749
|
-
scripts.evals ??= "evalkit run
|
|
8750
|
-
scripts.matrix ??= "evalkit run
|
|
8751
|
-
scripts["matrix:plan"] ??= "evalkit run
|
|
8752
|
-
scripts.dashboard ??= "evalkit
|
|
9104
|
+
scripts.evals ??= "evalkit run greeting --local";
|
|
9105
|
+
scripts.matrix ??= "evalkit run --matrix styles --local";
|
|
9106
|
+
scripts["matrix:plan"] ??= "evalkit run --matrix styles --dry-run";
|
|
9107
|
+
scripts.dashboard ??= "evalkit dev";
|
|
9108
|
+
scripts["dashboard:build"] ??= "evalkit build";
|
|
8753
9109
|
if (!dependencies["@leostera/evalkit"] && !devDependencies["@leostera/evalkit"]) dependencies["@leostera/evalkit"] = npmDependency;
|
|
8754
9110
|
if (!dependencies.typescript && !devDependencies.typescript) devDependencies.typescript = "^6.0.0";
|
|
8755
9111
|
if (!dependencies["@types/bun"] && !devDependencies["@types/bun"]) devDependencies["@types/bun"] = "^1.4.0";
|
|
@@ -8778,6 +9134,7 @@ async function writeStarterReadme(root, name, inPlace) {
|
|
|
8778
9134
|
}
|
|
8779
9135
|
//#endregion
|
|
8780
9136
|
//#region ../cli/src/index.ts
|
|
9137
|
+
const VERSION = "0.3.0";
|
|
8781
9138
|
let projectRoot = process.cwd();
|
|
8782
9139
|
let reportRoot = resolve(projectRoot, "_evalkit-results");
|
|
8783
9140
|
const sandboxRoot = resolve(projectRoot, "_evalkit-sandbox");
|
|
@@ -8797,13 +9154,17 @@ async function runEvals(options) {
|
|
|
8797
9154
|
const registry = await loadRegistry();
|
|
8798
9155
|
const evaluations = options.evalIds.map((id) => registry.get(id));
|
|
8799
9156
|
if (evaluations.some((e) => !e)) throw new Error("Unknown eval");
|
|
8800
|
-
await Effect.runPromise(Effect.all(evaluations.map((evaluation) =>
|
|
8801
|
-
|
|
8802
|
-
|
|
8803
|
-
|
|
8804
|
-
|
|
8805
|
-
|
|
8806
|
-
|
|
9157
|
+
await Effect.runPromise(runnerLogs(Effect.all(evaluations.map((evaluation) => {
|
|
9158
|
+
const name = evaluation.name ?? authoringId(evaluation);
|
|
9159
|
+
const started = performance.now();
|
|
9160
|
+
return Effect.sync(() => runLog.started(name, {}, options.suiteId)).pipe(Effect.andThen(runEval(evaluation, {
|
|
9161
|
+
report: localReportStore(reportRoot),
|
|
9162
|
+
workspaceRoot: sandboxRoot,
|
|
9163
|
+
suiteId: options.suiteId,
|
|
9164
|
+
runtime: runtimeFor(evaluation),
|
|
9165
|
+
concurrency: options.concurrency
|
|
9166
|
+
})), Effect.tap((result) => runLog.finished(name, {}, result, performance.now() - started)), Effect.tapError((error) => Effect.sync(() => runLog.failed(name, error))));
|
|
9167
|
+
}), { concurrency: options.concurrency })));
|
|
8807
9168
|
}
|
|
8808
9169
|
function registerMatrixRoutes(app, registry, dashboardMatrix, trials) {
|
|
8809
9170
|
let matrixRunning = false;
|
|
@@ -8846,14 +9207,18 @@ function registerMatrixRoutes(app, registry, dashboardMatrix, trials) {
|
|
|
8846
9207
|
}
|
|
8847
9208
|
if (matrixRunning) return context.text("A dashboard matrix cell is already running", 409);
|
|
8848
9209
|
matrixRunning = true;
|
|
8849
|
-
|
|
9210
|
+
const name = evaluation.name ?? authoringId(evaluation);
|
|
9211
|
+
const started = performance.now();
|
|
9212
|
+
runLog.started(name, body.parameters ?? {}, suiteId);
|
|
9213
|
+
Effect.runPromise(runnerLogs(runMatrix(dashboardMatrix, {
|
|
8850
9214
|
selection,
|
|
8851
9215
|
concurrency: 1,
|
|
8852
9216
|
trials,
|
|
8853
9217
|
...suiteId ? { suiteId } : {},
|
|
8854
9218
|
report: localReportStore(reportRoot),
|
|
8855
|
-
workspaceRoot: sandboxRoot
|
|
8856
|
-
|
|
9219
|
+
workspaceRoot: sandboxRoot,
|
|
9220
|
+
onResult: (cell, result) => runLog.cell(cell, result, performance.now() - started)
|
|
9221
|
+
}))).catch((error) => runLog.failed(name, error)).finally(() => {
|
|
8857
9222
|
matrixRunning = false;
|
|
8858
9223
|
});
|
|
8859
9224
|
} else {
|
|
@@ -8862,7 +9227,7 @@ function registerMatrixRoutes(app, registry, dashboardMatrix, trials) {
|
|
|
8862
9227
|
evalIds: [authoringId(evaluation)],
|
|
8863
9228
|
suiteId,
|
|
8864
9229
|
concurrency: 32
|
|
8865
|
-
}).catch((
|
|
9230
|
+
}).catch(() => {});
|
|
8866
9231
|
}
|
|
8867
9232
|
return context.json({ accepted: true }, 202);
|
|
8868
9233
|
});
|
|
@@ -8874,7 +9239,7 @@ function registerMatrixRoutes(app, registry, dashboardMatrix, trials) {
|
|
|
8874
9239
|
evalIds: suite.evals.map(authoringId),
|
|
8875
9240
|
suiteId: authoringId(suite),
|
|
8876
9241
|
concurrency: 32
|
|
8877
|
-
}).catch((
|
|
9242
|
+
}).catch(() => {});
|
|
8878
9243
|
return context.json({ accepted: true }, 202);
|
|
8879
9244
|
});
|
|
8880
9245
|
}
|
|
@@ -8942,19 +9307,48 @@ function registerDashboardAssets(app, dashboardRoot) {
|
|
|
8942
9307
|
return new Response(content, { headers: { "content-type": fileContentType(servedFile) } });
|
|
8943
9308
|
});
|
|
8944
9309
|
}
|
|
9310
|
+
/** The npm package bundles the UI beside cli.mjs; the monorepo uses its Vite build. */
|
|
9311
|
+
function dashboardAssets() {
|
|
9312
|
+
const bundledDashboard = new URL("./dashboard/", import.meta.url);
|
|
9313
|
+
const root = fileURLToPath(existsSync(bundledDashboard) ? bundledDashboard : new URL("../../dashboard/dist/", import.meta.url));
|
|
9314
|
+
if (!existsSync(resolve(root, "index.html"))) throw new Error(`Dashboard assets not found at ${root}; run \`bun run build\` in the EvalKit repository`);
|
|
9315
|
+
return root;
|
|
9316
|
+
}
|
|
9317
|
+
async function useProject(configPath) {
|
|
9318
|
+
project = await loadProject(projectRoot, configPath);
|
|
9319
|
+
projectRoot = project.root;
|
|
9320
|
+
reportRoot = resolve(projectRoot, project.config.reportDir ?? "_evalkit-results");
|
|
9321
|
+
}
|
|
8945
9322
|
async function serveDashboard() {
|
|
8946
9323
|
const registry = await loadRegistry();
|
|
8947
|
-
const
|
|
8948
|
-
const dashboardRoot = fileURLToPath(existsSync(bundledDashboard) ? bundledDashboard : new URL("../../dashboard/dist/", import.meta.url));
|
|
9324
|
+
const dashboardRoot = dashboardAssets();
|
|
8949
9325
|
const app = new Hono();
|
|
8950
|
-
|
|
9326
|
+
const dashboardMatrix = project?.config.matrix ? registry.matrices.at(-1) : void 0;
|
|
9327
|
+
registerMatrixRoutes(app, registry, dashboardMatrix, project?.config.execution?.trials);
|
|
8951
9328
|
registerReportRoutes(app, registry, createDashboardReportReader(reportRoot, registry));
|
|
8952
9329
|
registerDashboardAssets(app, dashboardRoot);
|
|
8953
|
-
const
|
|
8954
|
-
|
|
8955
|
-
|
|
8956
|
-
|
|
8957
|
-
|
|
9330
|
+
const port = Number(process.env.PORT ?? 4317);
|
|
9331
|
+
let server;
|
|
9332
|
+
try {
|
|
9333
|
+
server = Bun.serve({
|
|
9334
|
+
port,
|
|
9335
|
+
fetch: app.fetch
|
|
9336
|
+
});
|
|
9337
|
+
} catch (error) {
|
|
9338
|
+
if (error.code === "EADDRINUSE") throw new Error(`Port ${port} is already in use. Stop the other server or choose another port: PORT=4318 evalkit dev`);
|
|
9339
|
+
throw error;
|
|
9340
|
+
}
|
|
9341
|
+
const url = `http://localhost:${server.port}`;
|
|
9342
|
+
const reader = createDashboardReportReader(reportRoot, registry);
|
|
9343
|
+
const runs = (await reader.listRuns()).length;
|
|
9344
|
+
const legacy = reader.legacyRunIds().length;
|
|
9345
|
+
const matrixCells = dashboardMatrix?.count({ parameters: {} });
|
|
9346
|
+
const row = (label, value) => console.log(` ${cyan("➜")} ${bold(label.padEnd(9))}${value}`);
|
|
9347
|
+
banner("dev", VERSION);
|
|
9348
|
+
row("Local:", cyan(url));
|
|
9349
|
+
row("Project:", `${plural(registry.evals.length, "eval")} · ${plural(registry.suites.length, "suite")}` + (dashboardMatrix ? ` · matrix ${authoringId(dashboardMatrix)} (${plural(matrixCells, "cell")})` : ""));
|
|
9350
|
+
row("Reports:", `${relative(process.cwd(), reportRoot) || "."} ${dim(`(${plural(runs, "run")}${legacy ? `, ${legacy} from an older EvalKit hidden` : ""})`)}`);
|
|
9351
|
+
console.log(`\n ${dim("Runs started from the dashboard are logged below. Press Ctrl+C to stop.")}\n`);
|
|
8958
9352
|
if (process.env.EVALKIT_NO_OPEN !== "1") {
|
|
8959
9353
|
const opener = process.platform === "darwin" ? "open" : process.platform === "win32" ? "start" : "xdg-open";
|
|
8960
9354
|
try {
|
|
@@ -8969,6 +9363,8 @@ const config = text("config").pipe(optional);
|
|
|
8969
9363
|
const runOptions = {
|
|
8970
9364
|
config,
|
|
8971
9365
|
eval: text("eval").pipe(repeated),
|
|
9366
|
+
suite: text("suite").pipe(optional),
|
|
9367
|
+
matrix: text("matrix").pipe(optional),
|
|
8972
9368
|
model: text("model").pipe(repeated),
|
|
8973
9369
|
mode: text("mode").pipe(repeated),
|
|
8974
9370
|
select: text("select").pipe(repeated),
|
|
@@ -8989,6 +9385,8 @@ function runInput(options, positionals) {
|
|
|
8989
9385
|
return normalizeRunOptions({
|
|
8990
9386
|
config: Option.getOrUndefined(options.config),
|
|
8991
9387
|
eval: options.eval,
|
|
9388
|
+
suite: Option.getOrUndefined(options.suite),
|
|
9389
|
+
matrix: Option.getOrUndefined(options.matrix),
|
|
8992
9390
|
model: options.model,
|
|
8993
9391
|
mode: options.mode,
|
|
8994
9392
|
select: options.select,
|
|
@@ -9004,22 +9402,14 @@ function runInput(options, positionals) {
|
|
|
9004
9402
|
local: options.local
|
|
9005
9403
|
}, positionals);
|
|
9006
9404
|
}
|
|
9007
|
-
const execute = (
|
|
9008
|
-
try: () => runProjectCommand(
|
|
9405
|
+
const execute = (input) => Effect.tryPromise({
|
|
9406
|
+
try: () => runProjectCommand([], process.cwd(), input),
|
|
9009
9407
|
catch: (error) => error
|
|
9010
9408
|
});
|
|
9011
|
-
const
|
|
9409
|
+
const runCommand = make$1("run", {
|
|
9012
9410
|
...runOptions,
|
|
9013
9411
|
evalIds: text$2({ name: "eval-id" }).pipe(repeated$2)
|
|
9014
|
-
}, ({ evalIds, ...options }) => execute(
|
|
9015
|
-
const matrixCommand = make$1("run-matrix", {
|
|
9016
|
-
...runOptions,
|
|
9017
|
-
matrixId: text$2({ name: "matrix-id" })
|
|
9018
|
-
}, ({ matrixId, ...options }) => execute("run-matrix", runInput(options, [matrixId])));
|
|
9019
|
-
const suiteCommand = make$1("run-suite", {
|
|
9020
|
-
...runOptions,
|
|
9021
|
-
suiteId: text$2({ name: "suite-id" })
|
|
9022
|
-
}, ({ suiteId, ...options }) => execute("run-suite", runInput(options, [suiteId])));
|
|
9412
|
+
}, ({ evalIds, ...options }) => execute(runInput(options, evalIds)));
|
|
9023
9413
|
const newCommand = make$1("new", { directory: text$2({ name: "directory" }) }, ({ directory }) => Effect.tryPromise({
|
|
9024
9414
|
try: async () => {
|
|
9025
9415
|
const root = await newProject([directory]);
|
|
@@ -9027,24 +9417,103 @@ const newCommand = make$1("new", { directory: text$2({ name: "directory" }) }, (
|
|
|
9027
9417
|
},
|
|
9028
9418
|
catch: (error) => error
|
|
9029
9419
|
}));
|
|
9030
|
-
const
|
|
9420
|
+
const devCommand = make$1("dev", { config }, ({ config }) => Effect.tryPromise({
|
|
9031
9421
|
try: async () => {
|
|
9032
|
-
|
|
9033
|
-
projectRoot = project.root;
|
|
9034
|
-
reportRoot = resolve(projectRoot, project.config.reportDir ?? "_evalkit-results");
|
|
9422
|
+
await useProject(Option.getOrUndefined(config));
|
|
9035
9423
|
await serveDashboard();
|
|
9036
9424
|
},
|
|
9037
9425
|
catch: (error) => error
|
|
9038
9426
|
}));
|
|
9427
|
+
const buildCommand = make$1("build", {
|
|
9428
|
+
config,
|
|
9429
|
+
out: text("out").pipe(optional),
|
|
9430
|
+
run: boolean("run"),
|
|
9431
|
+
noRun: boolean("no-run")
|
|
9432
|
+
}, ({ config, out, run, noRun }) => Effect.tryPromise({
|
|
9433
|
+
try: async () => {
|
|
9434
|
+
if (run && noRun) throw new Error("Choose either --run or --no-run");
|
|
9435
|
+
const started = performance.now();
|
|
9436
|
+
const configPath = Option.getOrUndefined(config);
|
|
9437
|
+
banner("build", VERSION);
|
|
9438
|
+
await loadProjectStep(configPath);
|
|
9439
|
+
if (await shouldRunEvals({
|
|
9440
|
+
run,
|
|
9441
|
+
noRun
|
|
9442
|
+
})) await runAllEvals(configPath);
|
|
9443
|
+
const result = await buildDashboard({
|
|
9444
|
+
registry: project.registry,
|
|
9445
|
+
reportRoot,
|
|
9446
|
+
dashboardRoot: dashboardAssets(),
|
|
9447
|
+
outDir: resolve(projectRoot, Option.getOrElse(out, () => "_evalkit-dashboard"))
|
|
9448
|
+
}, step);
|
|
9449
|
+
const size = await directorySize(result.outDir);
|
|
9450
|
+
const shown = relative(process.cwd(), result.outDir) || ".";
|
|
9451
|
+
console.log(`\n ${bold(`${shown}/`)} ${dim(`${bytes(size.bytes)} · ${plural(size.files, "file")}`)}`);
|
|
9452
|
+
console.log(`\n ${bold("Built read-only dashboard")} in ${duration(performance.now() - started)}\n`);
|
|
9453
|
+
console.log(` ${cyan("➜")} ${bold("Preview:")} bunx serve ${shown}`);
|
|
9454
|
+
console.log(` ${yellow("⚠")} Contains transcripts, scores, evidence, and candidate files\n (never evaluator workspaces). Review before sharing.\n`);
|
|
9455
|
+
},
|
|
9456
|
+
catch: (error) => error
|
|
9457
|
+
}));
|
|
9458
|
+
async function loadProjectStep(configPath) {
|
|
9459
|
+
await step("Eval project", () => useProject(configPath), () => {
|
|
9460
|
+
const where = relative(process.cwd(), projectRoot);
|
|
9461
|
+
return `${plural(project.registry.evals.length, "eval")} loaded${where ? ` from ${where}` : ""}`;
|
|
9462
|
+
});
|
|
9463
|
+
}
|
|
9464
|
+
/**
|
|
9465
|
+
* A snapshot needs data: with no current results, run the evals. Otherwise ask,
|
|
9466
|
+
* or keep the existing results when no one can answer (CI, pipes).
|
|
9467
|
+
*/
|
|
9468
|
+
async function shouldRunEvals(flags) {
|
|
9469
|
+
if (flags.run) return true;
|
|
9470
|
+
if (flags.noRun) return false;
|
|
9471
|
+
const runs = (await createDashboardReportReader(reportRoot, project.registry).listRuns()).filter((entry) => !entry.archived).length;
|
|
9472
|
+
if (!runs) {
|
|
9473
|
+
console.log(` ${cyan("◇")} No results yet, so running the evals to have data to build.`);
|
|
9474
|
+
return true;
|
|
9475
|
+
}
|
|
9476
|
+
if (!canPrompt()) {
|
|
9477
|
+
console.log(` ${dim(`Using ${plural(runs, "existing run")}; pass --run to run the evals first.`)}`);
|
|
9478
|
+
return false;
|
|
9479
|
+
}
|
|
9480
|
+
return confirm(`Run the evals before building? ${dim(`(${plural(runs, "existing run")})`)}`);
|
|
9481
|
+
}
|
|
9482
|
+
async function runAllEvals(configPath) {
|
|
9483
|
+
const started = performance.now();
|
|
9484
|
+
console.log(`\n ${cyan("▶")} ${bold("Running evals")}\n`);
|
|
9485
|
+
await runProjectCommand([], process.cwd(), parseRunArgs(configPath ? ["--config", configPath] : []));
|
|
9486
|
+
const failed = process.exitCode === 1;
|
|
9487
|
+
console.log(`\n ${failed ? yellow("⚠") : cyan("✓")} ${bold("Evals finished")} in ${duration(performance.now() - started)}` + (failed ? dim(" (some failed; building with their results)") : "") + "\n");
|
|
9488
|
+
}
|
|
9489
|
+
const tidyCommand = make$1("tidy", { config }, ({ config }) => Effect.tryPromise({
|
|
9490
|
+
try: async () => {
|
|
9491
|
+
banner("tidy", VERSION);
|
|
9492
|
+
await loadProjectStep(Option.getOrUndefined(config));
|
|
9493
|
+
const result = await step("Run reports", () => tidyReports({
|
|
9494
|
+
registry: project.registry,
|
|
9495
|
+
reportRoot
|
|
9496
|
+
}), (value) => `${plural(value.current + value.archived.length + value.stillArchived, "run")} checked`);
|
|
9497
|
+
const byEval = (entries) => [...entries.reduce((groups, entry) => groups.set(entry.evalId, (groups.get(entry.evalId) ?? 0) + 1), /* @__PURE__ */ new Map()).entries()].map(([id, total]) => ` ${bold(id)} ${dim(`(${plural(total, "run")})`)}`).join("\n");
|
|
9498
|
+
console.log("");
|
|
9499
|
+
if (result.archived.length) console.log(` ${yellow("◆")} Archived ${plural(result.archived.length, "run")} of evals no longer in the project:\n${byEval(result.archived)}`);
|
|
9500
|
+
if (result.restored.length) console.log(` ${cyan("◆")} Restored ${plural(result.restored.length, "run")} whose eval is back:\n${byEval(result.restored)}`);
|
|
9501
|
+
if (!result.archived.length && !result.restored.length) console.log(` ${cyan("✓")} Nothing new to tidy.`);
|
|
9502
|
+
const totalArchived = result.archived.length + result.stillArchived;
|
|
9503
|
+
if (totalArchived) console.log(`\n ${dim(`Archived runs (${totalArchived}) are hidden in \`evalkit dev\` (filter "Show archived")`)}\n ${dim("and left out of `evalkit build`. Nothing was deleted.")}`);
|
|
9504
|
+
console.log("");
|
|
9505
|
+
},
|
|
9506
|
+
catch: (error) => error
|
|
9507
|
+
}));
|
|
9039
9508
|
const cli = run(make$1("evalkit").pipe(withSubcommands([
|
|
9040
9509
|
newCommand,
|
|
9041
|
-
|
|
9042
|
-
|
|
9043
|
-
|
|
9044
|
-
|
|
9510
|
+
runCommand,
|
|
9511
|
+
devCommand,
|
|
9512
|
+
buildCommand,
|
|
9513
|
+
tidyCommand
|
|
9045
9514
|
])), {
|
|
9046
9515
|
name: "EvalKit",
|
|
9047
|
-
version:
|
|
9516
|
+
version: VERSION
|
|
9048
9517
|
});
|
|
9049
9518
|
const argv = process.argv.slice();
|
|
9050
9519
|
if (argv[2] === "help") argv.splice(2, 1, "--help");
|