@leostera/evalkit 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.mjs +479 -120
- package/dist/cli.mjs.map +1 -1
- package/dist/dashboard/assets/index-CMr4aulR.css +2 -0
- package/dist/dashboard/assets/index-CVd_Np_w.js +30 -0
- package/dist/dashboard/index.html +2 -2
- package/package.json +1 -1
- package/dist/dashboard/assets/index-Z6raM-uW.js +0 -30
- package/dist/dashboard/assets/index-fUDsY6z-.css +0 -2
package/dist/cli.mjs
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
import { F as defineEvalMatrix, I as canonicalParameters, m as registerEvals, n as authoringId } from "./src-DrwwgaPn.mjs";
|
|
3
3
|
import { a as readTrialEvents, c as readTrialSummary, i as readRunSummary, l as localReportStore, n as runMatrix, o as readTrialManifest, r as readRunManifest, t as runEval } from "./src-DluldGKn.mjs";
|
|
4
4
|
import * as Schema from "effect/Schema";
|
|
5
|
-
import { Effect, Option } from "effect";
|
|
5
|
+
import { Effect, LogLevel, Logger, Option } from "effect";
|
|
6
6
|
import { cp, mkdir, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
|
|
7
7
|
import * as Path from "node:path";
|
|
8
8
|
import { basename, dirname, extname, relative, resolve, sep } from "node:path";
|
|
@@ -43,11 +43,11 @@ import * as EffectSecret from "effect/Secret";
|
|
|
43
43
|
import * as Effectable from "effect/Effectable";
|
|
44
44
|
import * as Match from "effect/Match";
|
|
45
45
|
import * as EffectNumber from "effect/Number";
|
|
46
|
-
import * as LogLevel from "effect/LogLevel";
|
|
46
|
+
import * as LogLevel$1 from "effect/LogLevel";
|
|
47
47
|
import * as Config from "effect/Config";
|
|
48
48
|
import * as HashMap from "effect/HashMap";
|
|
49
49
|
import * as Order from "effect/Order";
|
|
50
|
-
import * as Logger from "effect/Logger";
|
|
50
|
+
import * as Logger$1 from "effect/Logger";
|
|
51
51
|
import * as Unify from "effect/Unify";
|
|
52
52
|
import * as HashSet from "effect/HashSet";
|
|
53
53
|
import * as SynchronizedRef from "effect/SynchronizedRef";
|
|
@@ -392,13 +392,13 @@ const struct = (fields) => {
|
|
|
392
392
|
/** @internal */
|
|
393
393
|
const black = { _tag: "Black" };
|
|
394
394
|
/** @internal */
|
|
395
|
-
const red$
|
|
395
|
+
const red$4 = { _tag: "Red" };
|
|
396
396
|
/** @internal */
|
|
397
|
-
const green$
|
|
397
|
+
const green$3 = { _tag: "Green" };
|
|
398
398
|
/** @internal */
|
|
399
399
|
const magenta$1 = { _tag: "Magenta" };
|
|
400
400
|
/** @internal */
|
|
401
|
-
const cyan$
|
|
401
|
+
const cyan$2 = { _tag: "Cyan" };
|
|
402
402
|
/** @internal */
|
|
403
403
|
const white$3 = { _tag: "White" };
|
|
404
404
|
/** @internal */
|
|
@@ -500,7 +500,7 @@ const ESC = "\x1B[";
|
|
|
500
500
|
const BEL = "\x07";
|
|
501
501
|
const SEP = ";";
|
|
502
502
|
/** @internal */
|
|
503
|
-
const bold$
|
|
503
|
+
const bold$2 = /*#__PURE__*/ make$5({ bold: /*#__PURE__*/ Option$1.some(/*#__PURE__*/ setBold(true)) });
|
|
504
504
|
/** @internal */
|
|
505
505
|
const italicized$1 = /*#__PURE__*/ make$5({ italicized: /*#__PURE__*/ Option$1.some(/*#__PURE__*/ setItalicized(true)) });
|
|
506
506
|
/** @internal */
|
|
@@ -512,15 +512,15 @@ const brightColor = (color) => make$5({ foreground: Option$1.some(setColor(color
|
|
|
512
512
|
/** @internal */
|
|
513
513
|
const color$1 = (color) => make$5({ foreground: Option$1.some(setColor(color, false, "foreground")) });
|
|
514
514
|
/** @internal */
|
|
515
|
-
const red$
|
|
515
|
+
const red$3 = /*#__PURE__*/ color$1(red$4);
|
|
516
516
|
/** @internal */
|
|
517
|
-
const green$
|
|
517
|
+
const green$2 = /*#__PURE__*/ color$1(green$3);
|
|
518
518
|
/** @internal */
|
|
519
519
|
const white$2 = /*#__PURE__*/ color$1(white$3);
|
|
520
520
|
/** @internal */
|
|
521
521
|
const blackBright$1 = /*#__PURE__*/ brightColor(black);
|
|
522
522
|
/** @internal */
|
|
523
|
-
const cyanBright$1 = /*#__PURE__*/ brightColor(cyan$
|
|
523
|
+
const cyanBright$1 = /*#__PURE__*/ brightColor(cyan$2);
|
|
524
524
|
/** @internal */
|
|
525
525
|
const beep$2 = /*#__PURE__*/ make$5({ commands: /*#__PURE__*/ Arr.of(BEL) });
|
|
526
526
|
/** @internal */
|
|
@@ -587,7 +587,7 @@ const stringifyInternal = (self) => {
|
|
|
587
587
|
* @since 1.0.0
|
|
588
588
|
* @category constructors
|
|
589
589
|
*/
|
|
590
|
-
const bold = bold$
|
|
590
|
+
const bold$1 = bold$2;
|
|
591
591
|
/**
|
|
592
592
|
* @since 1.0.0
|
|
593
593
|
* @category constructors
|
|
@@ -612,12 +612,12 @@ const color = color$1;
|
|
|
612
612
|
* @since 1.0.0
|
|
613
613
|
* @category colors
|
|
614
614
|
*/
|
|
615
|
-
const red$
|
|
615
|
+
const red$2 = red$3;
|
|
616
616
|
/**
|
|
617
617
|
* @since 1.0.0
|
|
618
618
|
* @category colors
|
|
619
619
|
*/
|
|
620
|
-
const green = green$
|
|
620
|
+
const green$1 = green$2;
|
|
621
621
|
/**
|
|
622
622
|
* @since 1.0.0
|
|
623
623
|
* @category colors
|
|
@@ -2218,7 +2218,7 @@ const optimize = optimize$1;
|
|
|
2218
2218
|
* @since 1.0.0
|
|
2219
2219
|
* @category constructors
|
|
2220
2220
|
*/
|
|
2221
|
-
const red = red$
|
|
2221
|
+
const red$1 = red$4;
|
|
2222
2222
|
/**
|
|
2223
2223
|
* @since 1.0.0
|
|
2224
2224
|
* @category constructors
|
|
@@ -2228,7 +2228,7 @@ const magenta = magenta$1;
|
|
|
2228
2228
|
* @since 1.0.0
|
|
2229
2229
|
* @category constructors
|
|
2230
2230
|
*/
|
|
2231
|
-
const cyan = cyan$
|
|
2231
|
+
const cyan$1 = cyan$2;
|
|
2232
2232
|
/**
|
|
2233
2233
|
* @since 1.0.0
|
|
2234
2234
|
* @category constructors
|
|
@@ -2248,7 +2248,7 @@ const space = /*#__PURE__*/ text$6(" ");
|
|
|
2248
2248
|
/** @internal */
|
|
2249
2249
|
const code = (value) => highlight(value, white);
|
|
2250
2250
|
/** @internal */
|
|
2251
|
-
const error = (value) => highlight(value, red);
|
|
2251
|
+
const error = (value) => highlight(value, red$1);
|
|
2252
2252
|
/** @internal */
|
|
2253
2253
|
const highlight = (value, color) => ({
|
|
2254
2254
|
_tag: "Highlight",
|
|
@@ -2307,7 +2307,7 @@ const toAnsiDoc$1 = (self) => {
|
|
|
2307
2307
|
switch (self._tag) {
|
|
2308
2308
|
case "Highlight": return annotate(toAnsiDoc$1(self.value), color(self.color));
|
|
2309
2309
|
case "Sequence": return cat(toAnsiDoc$1(self.left), toAnsiDoc$1(self.right));
|
|
2310
|
-
case "Strong": return annotate(toAnsiDoc$1(self.value), bold);
|
|
2310
|
+
case "Strong": return annotate(toAnsiDoc$1(self.value), bold$1);
|
|
2311
2311
|
case "Text": return text$7(self.value);
|
|
2312
2312
|
case "URI": return annotate(text$7(self.value), underlined);
|
|
2313
2313
|
case "Weak": return annotate(toAnsiDoc$1(self.value), blackBright);
|
|
@@ -2373,11 +2373,11 @@ const toAnsiText = (self) => render$1(toAnsiDoc(self), { style: "pretty" });
|
|
|
2373
2373
|
const toAnsiDocInternal = (self) => {
|
|
2374
2374
|
switch (self._tag) {
|
|
2375
2375
|
case "Empty": return empty$3;
|
|
2376
|
-
case "Header": return pipe(annotate(toAnsiDoc$1(self.value), bold), cat(hardLine));
|
|
2376
|
+
case "Header": return pipe(annotate(toAnsiDoc$1(self.value), bold$1), cat(hardLine));
|
|
2377
2377
|
case "Paragraph": return pipe(toAnsiDoc$1(self.value), cat(hardLine));
|
|
2378
2378
|
case "DescriptionList": {
|
|
2379
2379
|
const definitions = self.definitions.map(([span, doc]) => cats([
|
|
2380
|
-
annotate(toAnsiDoc$1(span), bold),
|
|
2380
|
+
annotate(toAnsiDoc$1(span), bold$1),
|
|
2381
2381
|
empty$3,
|
|
2382
2382
|
indent(toAnsiDocInternal(doc), 2)
|
|
2383
2383
|
]));
|
|
@@ -2592,8 +2592,8 @@ function renderError$2(state, pointer) {
|
|
|
2592
2592
|
onSome: (error) => {
|
|
2593
2593
|
const errorLines = error.split(NEWLINE_REGEX$3);
|
|
2594
2594
|
if (Arr.isNonEmptyReadonlyArray(errorLines)) {
|
|
2595
|
-
const annotateLine = (line) => annotate(text$7(line), combine(italicized, red$
|
|
2596
|
-
const prefix = cat(annotate(pointer, red$
|
|
2595
|
+
const annotateLine = (line) => annotate(text$7(line), combine(italicized, red$2));
|
|
2596
|
+
const prefix = cat(annotate(pointer, red$2), space$1);
|
|
2597
2597
|
const lines = Arr.map(errorLines, (str) => annotateLine(str));
|
|
2598
2598
|
return cursorSavePosition.pipe(cat(hardLine), cat(prefix), cat(align(vsep(lines))), cat(cursorRestorePosition));
|
|
2599
2599
|
}
|
|
@@ -2612,7 +2612,7 @@ function renderParts(state, submitted = false) {
|
|
|
2612
2612
|
});
|
|
2613
2613
|
}
|
|
2614
2614
|
function renderOutput$4(leadingSymbol, trailingSymbol, parts, options) {
|
|
2615
|
-
const annotateLine = (line) => annotate(text$7(line), bold);
|
|
2615
|
+
const annotateLine = (line) => annotate(text$7(line), bold$1);
|
|
2616
2616
|
const prefix = cat(leadingSymbol, space$1);
|
|
2617
2617
|
return Arr.match(options.message.split(/\r?\n/), {
|
|
2618
2618
|
onEmpty: () => hsep([
|
|
@@ -2642,7 +2642,7 @@ function renderSubmission$5(state, options) {
|
|
|
2642
2642
|
return Effect$1.gen(function* () {
|
|
2643
2643
|
const columns = yield* (yield* Terminal).columns;
|
|
2644
2644
|
const figures$11 = yield* figures;
|
|
2645
|
-
return renderOutput$4(annotate(figures$11.tick, green), annotate(figures$11.ellipsis, blackBright), renderParts(state, true), options).pipe(cat(hardLine), optimize(Deep), render$1({
|
|
2645
|
+
return renderOutput$4(annotate(figures$11.tick, green$1), annotate(figures$11.ellipsis, blackBright), renderParts(state, true), options).pipe(cat(hardLine), optimize(Deep), render$1({
|
|
2646
2646
|
style: "pretty",
|
|
2647
2647
|
options: { lineWidth: columns }
|
|
2648
2648
|
}));
|
|
@@ -3106,7 +3106,7 @@ function handleClear$4(options) {
|
|
|
3106
3106
|
}
|
|
3107
3107
|
const NEWLINE_REGEX$2 = /\r?\n/;
|
|
3108
3108
|
function renderPrompt(confirm, message, leadingSymbol, trailingSymbol) {
|
|
3109
|
-
const annotateLine = (line) => annotate(text$7(line), bold);
|
|
3109
|
+
const annotateLine = (line) => annotate(text$7(line), bold$1);
|
|
3110
3110
|
const prefix = cat(leadingSymbol, space$1);
|
|
3111
3111
|
return Arr.match(message.split(NEWLINE_REGEX$2), {
|
|
3112
3112
|
onEmpty: () => hsep([
|
|
@@ -3159,7 +3159,7 @@ function renderNextFrame$4(state, options) {
|
|
|
3159
3159
|
options: { lineWidth: columns }
|
|
3160
3160
|
}));
|
|
3161
3161
|
}
|
|
3162
|
-
const leadingSymbol = annotate(figures$10.tick, green);
|
|
3162
|
+
const leadingSymbol = annotate(figures$10.tick, green$1);
|
|
3163
3163
|
const trailingSymbol = annotate(figures$10.ellipsis, blackBright);
|
|
3164
3164
|
const promptMsg = renderPrompt(empty$3, options.message, leadingSymbol, trailingSymbol);
|
|
3165
3165
|
const files = renderFiles(state, state.files, figures$10, options);
|
|
@@ -3173,7 +3173,7 @@ function renderSubmission$4(value, options) {
|
|
|
3173
3173
|
return Effect$1.gen(function* () {
|
|
3174
3174
|
const columns = yield* (yield* Terminal).columns;
|
|
3175
3175
|
const figures$9 = yield* figures;
|
|
3176
|
-
const leadingSymbol = annotate(figures$9.tick, green);
|
|
3176
|
+
const leadingSymbol = annotate(figures$9.tick, green$1);
|
|
3177
3177
|
const trailingSymbol = annotate(figures$9.ellipsis, blackBright);
|
|
3178
3178
|
return renderPrompt(empty$3, options.message, leadingSymbol, trailingSymbol).pipe(cat(space$1), cat(annotate(text$7(value), white$1)), cat(hardLine), render$1({
|
|
3179
3179
|
style: "pretty",
|
|
@@ -3316,7 +3316,7 @@ function handleClear$3(options) {
|
|
|
3316
3316
|
function renderInput$1(state, submitted) {
|
|
3317
3317
|
const annotation = Option$1.match(state.error, {
|
|
3318
3318
|
onNone: () => combine(underlined, cyanBright),
|
|
3319
|
-
onSome: () => red$
|
|
3319
|
+
onSome: () => red$2
|
|
3320
3320
|
});
|
|
3321
3321
|
const value = state.value === "" ? empty$3 : text$7(`${state.value}`);
|
|
3322
3322
|
return submitted ? value : annotate(value, annotation);
|
|
@@ -3328,8 +3328,8 @@ function renderError$1(state, pointer) {
|
|
|
3328
3328
|
onSome: (error) => Arr.match(error.split(NEWLINE_REGEX$1), {
|
|
3329
3329
|
onEmpty: () => empty$3,
|
|
3330
3330
|
onNonEmpty: (errorLines) => {
|
|
3331
|
-
const annotateLine = (line) => annotate(text$7(line), combine(italicized, red$
|
|
3332
|
-
const prefix = cat(annotate(pointer, red$
|
|
3331
|
+
const annotateLine = (line) => annotate(text$7(line), combine(italicized, red$2));
|
|
3332
|
+
const prefix = cat(annotate(pointer, red$2), space$1);
|
|
3333
3333
|
const lines = Arr.map(errorLines, (str) => annotateLine(str));
|
|
3334
3334
|
return cursorSavePosition.pipe(cat(hardLine), cat(prefix), cat(align(vsep(lines))), cat(cursorRestorePosition));
|
|
3335
3335
|
}
|
|
@@ -3337,7 +3337,7 @@ function renderError$1(state, pointer) {
|
|
|
3337
3337
|
});
|
|
3338
3338
|
}
|
|
3339
3339
|
function renderOutput$3(state, leadingSymbol, trailingSymbol, options, submitted = false) {
|
|
3340
|
-
const annotateLine = (line) => annotate(text$7(line), bold);
|
|
3340
|
+
const annotateLine = (line) => annotate(text$7(line), bold$1);
|
|
3341
3341
|
const prefix = cat(leadingSymbol, space$1);
|
|
3342
3342
|
return Arr.match(options.message.split(/\r?\n/), {
|
|
3343
3343
|
onEmpty: () => hsep([
|
|
@@ -3368,7 +3368,7 @@ function renderSubmission$3(nextState, options) {
|
|
|
3368
3368
|
return Effect$1.gen(function* () {
|
|
3369
3369
|
const columns = yield* (yield* Terminal).columns;
|
|
3370
3370
|
const figures$7 = yield* figures;
|
|
3371
|
-
return renderOutput$3(nextState, annotate(figures$7.tick, green), annotate(figures$7.ellipsis, blackBright), options, true).pipe(cat(hardLine), optimize(Deep), render$1({
|
|
3371
|
+
return renderOutput$3(nextState, annotate(figures$7.tick, green$1), annotate(figures$7.ellipsis, blackBright), options, true).pipe(cat(hardLine), optimize(Deep), render$1({
|
|
3372
3372
|
style: "pretty",
|
|
3373
3373
|
options: { lineWidth: columns }
|
|
3374
3374
|
}));
|
|
@@ -3551,7 +3551,7 @@ const float = (options) => {
|
|
|
3551
3551
|
const renderBeep$2 = /*#__PURE__*/ render$1(beep, { style: "pretty" });
|
|
3552
3552
|
const NEWLINE_REGEX = /\r?\n/;
|
|
3553
3553
|
function renderOutput$2(leadingSymbol, trailingSymbol, options) {
|
|
3554
|
-
const annotateLine = (line) => annotate(text$7(line), bold);
|
|
3554
|
+
const annotateLine = (line) => annotate(text$7(line), bold$1);
|
|
3555
3555
|
const prefix = cat(leadingSymbol, space$1);
|
|
3556
3556
|
return Arr.match(options.message.split(NEWLINE_REGEX), {
|
|
3557
3557
|
onEmpty: () => hsep([prefix, trailingSymbol]),
|
|
@@ -3566,7 +3566,7 @@ function renderChoicePrefix(state, choices, toDisplay, currentIndex, figures) {
|
|
|
3566
3566
|
if (currentIndex === toDisplay.startIndex && toDisplay.startIndex > 0) prefix = figures.arrowUp;
|
|
3567
3567
|
else if (currentIndex === toDisplay.endIndex - 1 && toDisplay.endIndex < choices.length) prefix = figures.arrowDown;
|
|
3568
3568
|
if (choices[currentIndex].disabled) {
|
|
3569
|
-
const annotation = combine(bold, blackBright);
|
|
3569
|
+
const annotation = combine(bold$1, blackBright);
|
|
3570
3570
|
return state === currentIndex ? figures.pointer.pipe(annotate(annotation), cat(prefix)) : prefix.pipe(cat(space$1));
|
|
3571
3571
|
}
|
|
3572
3572
|
return state === currentIndex ? figures.pointer.pipe(annotate(cyanBright), cat(prefix)) : prefix.pipe(cat(space$1));
|
|
@@ -3611,7 +3611,7 @@ function renderSubmission$2(state, options) {
|
|
|
3611
3611
|
const columns = yield* (yield* Terminal).columns;
|
|
3612
3612
|
const figures$6 = yield* figures;
|
|
3613
3613
|
const selected = text$7(options.choices[state].title);
|
|
3614
|
-
return renderOutput$2(annotate(figures$6.tick, green), annotate(figures$6.ellipsis, blackBright), options).pipe(cat(space$1), cat(annotate(selected, white$1)), cat(hardLine), render$1({
|
|
3614
|
+
return renderOutput$2(annotate(figures$6.tick, green$1), annotate(figures$6.ellipsis, blackBright), options).pipe(cat(space$1), cat(annotate(selected, white$1)), cat(hardLine), render$1({
|
|
3615
3615
|
style: "pretty",
|
|
3616
3616
|
options: { lineWidth: columns }
|
|
3617
3617
|
}));
|
|
@@ -3714,7 +3714,7 @@ function renderInput(nextState, options, submitted) {
|
|
|
3714
3714
|
if (nextState.value.length === 0) return blackBright;
|
|
3715
3715
|
return combine(underlined, cyanBright);
|
|
3716
3716
|
},
|
|
3717
|
-
onSome: () => red$
|
|
3717
|
+
onSome: () => red$2
|
|
3718
3718
|
});
|
|
3719
3719
|
switch (options.type) {
|
|
3720
3720
|
case "hidden": return empty$3;
|
|
@@ -3728,8 +3728,8 @@ function renderError(nextState, pointer) {
|
|
|
3728
3728
|
onSome: (error) => Arr.match(error.split(/\r?\n/), {
|
|
3729
3729
|
onEmpty: () => empty$3,
|
|
3730
3730
|
onNonEmpty: (errorLines) => {
|
|
3731
|
-
const annotateLine = (line) => text$7(line).pipe(annotate(combine(italicized, red$
|
|
3732
|
-
const prefix = cat(annotate(pointer, red$
|
|
3731
|
+
const annotateLine = (line) => text$7(line).pipe(annotate(combine(italicized, red$2)));
|
|
3732
|
+
const prefix = cat(annotate(pointer, red$2), space$1);
|
|
3733
3733
|
const lines = Arr.map(errorLines, (str) => annotateLine(str));
|
|
3734
3734
|
return cursorSavePosition.pipe(cat(hardLine), cat(prefix), cat(align(vsep(lines))), cat(cursorRestorePosition));
|
|
3735
3735
|
}
|
|
@@ -3737,7 +3737,7 @@ function renderError(nextState, pointer) {
|
|
|
3737
3737
|
});
|
|
3738
3738
|
}
|
|
3739
3739
|
function renderOutput$1(nextState, leadingSymbol, trailingSymbol, options, submitted = false) {
|
|
3740
|
-
const annotateLine = (line) => annotate(text$7(line), bold);
|
|
3740
|
+
const annotateLine = (line) => annotate(text$7(line), bold$1);
|
|
3741
3741
|
const promptLines = options.message.split(/\r?\n/);
|
|
3742
3742
|
const prefix = cat(leadingSymbol, space$1);
|
|
3743
3743
|
if (Arr.isNonEmptyReadonlyArray(promptLines)) {
|
|
@@ -3767,7 +3767,7 @@ function renderSubmission$1(state, options) {
|
|
|
3767
3767
|
return Effect$1.gen(function* () {
|
|
3768
3768
|
const columns = yield* (yield* Terminal).columns;
|
|
3769
3769
|
const figures$4 = yield* figures;
|
|
3770
|
-
return renderOutput$1(state, annotate(figures$4.tick, green), annotate(figures$4.ellipsis, blackBright), options, true).pipe(cat(hardLine), optimize(Deep), render$1({
|
|
3770
|
+
return renderOutput$1(state, annotate(figures$4.tick, green$1), annotate(figures$4.ellipsis, blackBright), options, true).pipe(cat(hardLine), optimize(Deep), render$1({
|
|
3771
3771
|
style: "pretty",
|
|
3772
3772
|
options: { lineWidth: columns }
|
|
3773
3773
|
}));
|
|
@@ -3909,7 +3909,7 @@ function renderToggle(value, options, submitted = false) {
|
|
|
3909
3909
|
]);
|
|
3910
3910
|
}
|
|
3911
3911
|
function renderOutput(toggle, leadingSymbol, trailingSymbol, options) {
|
|
3912
|
-
const annotateLine = (line) => annotate(text$7(line), bold);
|
|
3912
|
+
const annotateLine = (line) => annotate(text$7(line), bold$1);
|
|
3913
3913
|
const promptLines = options.message.split(/\r?\n/);
|
|
3914
3914
|
const prefix = cat(leadingSymbol, space$1);
|
|
3915
3915
|
if (Arr.isNonEmptyReadonlyArray(promptLines)) {
|
|
@@ -3941,7 +3941,7 @@ function renderSubmission(value, options) {
|
|
|
3941
3941
|
const terminal = yield* Terminal;
|
|
3942
3942
|
const figures$2 = yield* figures;
|
|
3943
3943
|
const columns = yield* terminal.columns;
|
|
3944
|
-
const leadingSymbol = annotate(figures$2.tick, green);
|
|
3944
|
+
const leadingSymbol = annotate(figures$2.tick, green$1);
|
|
3945
3945
|
const trailingSymbol = annotate(figures$2.ellipsis, blackBright);
|
|
3946
3946
|
return renderOutput(renderToggle(value, options, true), leadingSymbol, trailingSymbol, options).pipe(cat(hardLine), optimize(Deep), render$1({
|
|
3947
3947
|
style: "pretty",
|
|
@@ -5919,7 +5919,7 @@ const completionsOptions = /*#__PURE__*/ choiceWithValue("completions", [
|
|
|
5919
5919
|
["zsh", "zsh"]
|
|
5920
5920
|
]).pipe(optional$1, /*#__PURE__*/ withDescription("Generate a completion script for a specific shell."));
|
|
5921
5921
|
/** @internal */
|
|
5922
|
-
const logLevelOptions = /*#__PURE__*/ choiceWithValue("log-level", LogLevel.allLevels.map((level) => [level._tag.toLowerCase(), level])).pipe(optional$1, /*#__PURE__*/ withDescription("Sets the minimum log level for a command."));
|
|
5922
|
+
const logLevelOptions = /*#__PURE__*/ choiceWithValue("log-level", LogLevel$1.allLevels.map((level) => [level._tag.toLowerCase(), level])).pipe(optional$1, /*#__PURE__*/ withDescription("Sets the minimum log level for a command."));
|
|
5923
5923
|
/** @internal */
|
|
5924
5924
|
const helpOptions = /*#__PURE__*/ boolean$1("help").pipe(/*#__PURE__*/ withAlias("h"), /*#__PURE__*/ withDescription("Show the help documentation for a command."));
|
|
5925
5925
|
/** @internal */
|
|
@@ -6294,7 +6294,7 @@ const wizardInternal = (self, prefix, config) => {
|
|
|
6294
6294
|
case "GetUserInput":
|
|
6295
6295
|
case "Standard": return Effect$1.gen(function* () {
|
|
6296
6296
|
const logCurrentCommand = Ref.get(commandLineRef).pipe(Effect$1.flatMap((commandLine) => {
|
|
6297
|
-
const currentCommand = p(pipe(strong(highlight("COMMAND:", cyan)), concat$1(space), concat$1(highlight(Arr.join(commandLine, " "), magenta))));
|
|
6297
|
+
const currentCommand = p(pipe(strong(highlight("COMMAND:", cyan$1)), concat$1(space), concat$1(highlight(Arr.join(commandLine, " "), magenta))));
|
|
6298
6298
|
return Console.log(toAnsiText(currentCommand));
|
|
6299
6299
|
}));
|
|
6300
6300
|
if (isStandard(self)) {
|
|
@@ -6634,7 +6634,7 @@ const handleBuiltInOption = (self, executable, args, builtIn, execute, config, o
|
|
|
6634
6634
|
filteredArgs.push(args[i]);
|
|
6635
6635
|
}
|
|
6636
6636
|
const nextArgs = Arr.appendAll(baseArgs, filteredArgs);
|
|
6637
|
-
return run$2(self, nextArgs, execute).pipe(Logger.withMinimumLogLevel(builtIn.level));
|
|
6637
|
+
return run$2(self, nextArgs, execute).pipe(Logger$1.withMinimumLogLevel(builtIn.level));
|
|
6638
6638
|
}
|
|
6639
6639
|
case "ShowHelp": {
|
|
6640
6640
|
const banner = h1(code(self.name));
|
|
@@ -6740,7 +6740,7 @@ const renderWizardArgs = (args) => {
|
|
|
6740
6740
|
return blocks([
|
|
6741
6741
|
p(strong(code("Wizard Mode Complete!"))),
|
|
6742
6742
|
p(executeMsg),
|
|
6743
|
-
p(concat$1(text$6(" "), highlight(params, cyan)))
|
|
6743
|
+
p(concat$1(text$6(" "), highlight(params, cyan$1)))
|
|
6744
6744
|
]);
|
|
6745
6745
|
};
|
|
6746
6746
|
const isLogLevelArg = (arg) => {
|
|
@@ -8050,6 +8050,8 @@ const layer = /*#__PURE__*/ pipe(/*#__PURE__*/ Layer.mergeAll(layer$3, layer$5,
|
|
|
8050
8050
|
//#endregion
|
|
8051
8051
|
//#region ../cli/src/dashboard-reports.ts
|
|
8052
8052
|
const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
8053
|
+
/** Written into a run directory by `evalkit tidy`; deleting it restores the run. */
|
|
8054
|
+
const ARCHIVE_MARKER = "archived.json";
|
|
8053
8055
|
function isMissing(error) {
|
|
8054
8056
|
return error.code === "ENOENT";
|
|
8055
8057
|
}
|
|
@@ -8086,15 +8088,35 @@ function fileContentType(file) {
|
|
|
8086
8088
|
".js": "text/javascript"
|
|
8087
8089
|
}[extname(file)] ?? "application/octet-stream";
|
|
8088
8090
|
}
|
|
8089
|
-
/**
|
|
8091
|
+
/**
|
|
8092
|
+
* Read saved local reports. A missing summary means "running"; malformed reports are errors.
|
|
8093
|
+
* Runs written by an older EvalKit (manifest `schemaVersion` < 2) are skipped and counted.
|
|
8094
|
+
*/
|
|
8090
8095
|
function createDashboardReportReader(root, registry) {
|
|
8096
|
+
const legacyRuns = /* @__PURE__ */ new Set();
|
|
8091
8097
|
async function listRuns() {
|
|
8092
8098
|
const entries = await ifPresent(() => readdir(root, { withFileTypes: true }));
|
|
8093
8099
|
if (!entries) return [];
|
|
8094
8100
|
const ids = entries.filter((entry) => entry.isDirectory() && UUID_PATTERN.test(entry.name)).map((entry) => entry.name);
|
|
8095
8101
|
return (await Promise.all(ids.map(readRun))).filter((run) => run !== void 0);
|
|
8096
8102
|
}
|
|
8103
|
+
/** True for a run from a report format this version cannot read; never for corrupt JSON. */
|
|
8104
|
+
async function isLegacyRun(id) {
|
|
8105
|
+
const text = await ifPresent(() => readFile(resolve(root, id, "manifest.json"), "utf8"));
|
|
8106
|
+
if (text === void 0) return false;
|
|
8107
|
+
let version;
|
|
8108
|
+
try {
|
|
8109
|
+
version = JSON.parse(text).schemaVersion;
|
|
8110
|
+
} catch {
|
|
8111
|
+
return false;
|
|
8112
|
+
}
|
|
8113
|
+
return typeof version === "number" && version < 2;
|
|
8114
|
+
}
|
|
8097
8115
|
async function readRun(id) {
|
|
8116
|
+
if (await isLegacyRun(id)) {
|
|
8117
|
+
legacyRuns.add(id);
|
|
8118
|
+
return;
|
|
8119
|
+
}
|
|
8098
8120
|
const manifest = await ifPresent(() => readRunManifest(root, id));
|
|
8099
8121
|
if (!manifest) return void 0;
|
|
8100
8122
|
const summary = await ifPresent(() => readRunSummary(root, id));
|
|
@@ -8103,6 +8125,7 @@ function createDashboardReportReader(root, registry) {
|
|
|
8103
8125
|
const scores = trials.flatMap((trial) => trial?.scoring?.overall === void 0 ? [] : [trial.scoring.overall]);
|
|
8104
8126
|
return {
|
|
8105
8127
|
id,
|
|
8128
|
+
...await ifPresent(() => stat(resolve(root, id, "archived.json"))) ? { archived: true } : {},
|
|
8106
8129
|
evalId: manifest.evalId,
|
|
8107
8130
|
...manifest.suiteId ? { suiteId: manifest.suiteId } : {},
|
|
8108
8131
|
...manifest.matrix ? {
|
|
@@ -8171,6 +8194,8 @@ function createDashboardReportReader(root, registry) {
|
|
|
8171
8194
|
}
|
|
8172
8195
|
return {
|
|
8173
8196
|
listRuns,
|
|
8197
|
+
/** Run IDs skipped by the last `listRuns()` because an older EvalKit wrote them. */
|
|
8198
|
+
legacyRunIds: () => [...legacyRuns],
|
|
8174
8199
|
listTrials,
|
|
8175
8200
|
trialDetail,
|
|
8176
8201
|
listArtifacts,
|
|
@@ -8205,66 +8230,154 @@ async function walkArtifacts(root) {
|
|
|
8205
8230
|
//#region ../cli/src/dashboard-build.ts
|
|
8206
8231
|
/** Marks a directory as EvalKit-owned so a rebuild may safely replace it. */
|
|
8207
8232
|
const MARKER = ".evalkit-dashboard";
|
|
8208
|
-
|
|
8209
|
-
|
|
8210
|
-
* the catalog and local reports, and each trial's candidate artifact files.
|
|
8211
|
-
* Evaluator workspaces are never copied; reports may still contain sensitive
|
|
8212
|
-
* messages, scores, or evidence, so review the output before sharing it.
|
|
8213
|
-
*/
|
|
8214
|
-
async function buildDashboard(options) {
|
|
8233
|
+
const quietStep = (_label, work) => work(() => {});
|
|
8234
|
+
async function buildDashboard(options, step = quietStep) {
|
|
8215
8235
|
const { registry, reportRoot, dashboardRoot, outDir } = options;
|
|
8216
|
-
await
|
|
8217
|
-
await cp(dashboardRoot, outDir, { recursive: true });
|
|
8218
|
-
await writeFile(resolve(outDir, MARKER), "Generated by `evalkit build`.\n");
|
|
8219
|
-
await writeFile(resolve(outDir, "index.html"), staticIndex(await readFile(resolve(dashboardRoot, "index.html"), "utf8")));
|
|
8236
|
+
const template = await readFile(resolve(dashboardRoot, "index.html"), "utf8");
|
|
8220
8237
|
const data = resolve(outDir, "data");
|
|
8221
8238
|
const write = async (path, value) => {
|
|
8222
8239
|
const file = resolve(data, path);
|
|
8223
8240
|
await mkdir(dirname(file), { recursive: true });
|
|
8224
8241
|
await writeFile(file, `${JSON.stringify(value)}\n`);
|
|
8225
8242
|
};
|
|
8243
|
+
await step("Dashboard assets", async () => {
|
|
8244
|
+
await prepareOutDir(outDir);
|
|
8245
|
+
await cp(dashboardRoot, outDir, { recursive: true });
|
|
8246
|
+
await writeFile(resolve(outDir, MARKER), "Generated by `evalkit build`.\n");
|
|
8247
|
+
await writeFile(resolve(outDir, "index.html"), staticIndex(template));
|
|
8248
|
+
}, () => "read-only UI with relative assets");
|
|
8249
|
+
await step("Eval catalog", async () => {
|
|
8250
|
+
await write("catalog.json", { evals: registry.catalog() });
|
|
8251
|
+
await write("evals.json", { evals: registry.metadata() });
|
|
8252
|
+
await write("suites.json", { suites: registry.suiteMetadata() });
|
|
8253
|
+
return {
|
|
8254
|
+
evals: registry.evals.length,
|
|
8255
|
+
suites: registry.suites.length
|
|
8256
|
+
};
|
|
8257
|
+
}, ({ evals, suites }) => `${count(evals, "eval")} · ${count(suites, "suite")}`);
|
|
8226
8258
|
const reports = createDashboardReportReader(reportRoot, registry);
|
|
8227
|
-
const
|
|
8228
|
-
|
|
8229
|
-
|
|
8230
|
-
|
|
8231
|
-
|
|
8232
|
-
|
|
8233
|
-
|
|
8234
|
-
|
|
8235
|
-
|
|
8236
|
-
|
|
8237
|
-
|
|
8238
|
-
|
|
8239
|
-
|
|
8240
|
-
|
|
8241
|
-
|
|
8242
|
-
await write(`${base}/artifacts.json`, { artifacts: await reports.listArtifacts(run.id, trial.id) });
|
|
8243
|
-
const workspace = await reports.listCandidateWorkspace(run.id, trial.id);
|
|
8244
|
-
await write(`${base}/workspace.json`, workspace);
|
|
8245
|
-
for (const entry of workspace) {
|
|
8246
|
-
if (entry.kind !== "file") continue;
|
|
8247
|
-
const response = await reports.candidateFile(run.id, trial.id, entry.path);
|
|
8248
|
-
const target = resolve(data, base, "workspace", entry.path);
|
|
8249
|
-
const root = resolve(data, base, "workspace");
|
|
8250
|
-
if (!target.startsWith(`${root}${sep}`)) throw new Error(`Invalid workspace path: ${entry.path}`);
|
|
8251
|
-
await mkdir(dirname(target), { recursive: true });
|
|
8252
|
-
await writeFile(target, new Uint8Array(await response.arrayBuffer()));
|
|
8253
|
-
files += 1;
|
|
8259
|
+
const snapshot = await step("Run reports", async (progress) => {
|
|
8260
|
+
const all = await reports.listRuns();
|
|
8261
|
+
const runs = all.filter((run) => !run.archived);
|
|
8262
|
+
await write("runs.json", { runs });
|
|
8263
|
+
let trials = 0;
|
|
8264
|
+
let files = 0;
|
|
8265
|
+
const trialIds = [];
|
|
8266
|
+
for (const [index, run] of runs.entries()) {
|
|
8267
|
+
progress(`run ${index + 1}/${runs.length} · ${run.id.slice(0, 8)}`);
|
|
8268
|
+
const runTrials = await reports.listTrials(run.id);
|
|
8269
|
+
await write(`runs/${run.id}/trials.json`, { trials: runTrials });
|
|
8270
|
+
for (const trial of runTrials) {
|
|
8271
|
+
trials += 1;
|
|
8272
|
+
trialIds.push(trial.id);
|
|
8273
|
+
files += await snapshotTrial(reports, write, data, run.id, trial.id);
|
|
8254
8274
|
}
|
|
8255
8275
|
}
|
|
8256
|
-
|
|
8276
|
+
return {
|
|
8277
|
+
runs: runs.length,
|
|
8278
|
+
trials,
|
|
8279
|
+
files,
|
|
8280
|
+
skipped: reports.legacyRunIds().length,
|
|
8281
|
+
archived: all.length - runs.length,
|
|
8282
|
+
runIds: runs.map((run) => run.id),
|
|
8283
|
+
trialIds
|
|
8284
|
+
};
|
|
8285
|
+
}, ({ runs, trials, files, skipped, archived }) => `${count(runs, "run")} · ${count(trials, "trial")} · ${count(files, "candidate file")}` + (archived ? ` · ${archived} archived excluded` : "") + (skipped ? ` · ${skipped} older-format skipped` : ""));
|
|
8286
|
+
await step("Route pages", async () => {
|
|
8287
|
+
const routes = staticRoutes({
|
|
8288
|
+
suites: registry.suites.map(authoringId),
|
|
8289
|
+
evals: registry.catalog().map((entry) => entry.path),
|
|
8290
|
+
runs: snapshot.runIds,
|
|
8291
|
+
trials: snapshot.trialIds
|
|
8292
|
+
});
|
|
8293
|
+
for (const route of routes) {
|
|
8294
|
+
const file = resolve(outDir, `${route}.html`);
|
|
8295
|
+
await mkdir(dirname(file), { recursive: true });
|
|
8296
|
+
await writeFile(file, staticIndex(template, baseFor(route)));
|
|
8297
|
+
}
|
|
8298
|
+
await writeFile(resolve(outDir, "404.html"), staticIndex(template, "/"));
|
|
8299
|
+
return routes.length;
|
|
8300
|
+
}, (pages) => `${count(pages, "page")} with clean /path URLs`);
|
|
8301
|
+
const { runIds: _runs, trialIds: _trials, ...summary } = snapshot;
|
|
8257
8302
|
return {
|
|
8258
8303
|
outDir,
|
|
8259
|
-
|
|
8260
|
-
|
|
8304
|
+
...summary
|
|
8305
|
+
};
|
|
8306
|
+
}
|
|
8307
|
+
/**
|
|
8308
|
+
* Every dashboard URL known at build time, as file paths without `.html`.
|
|
8309
|
+
* Static hosts serve `/runs/abc` from `runs/abc.html`, so links and reloads
|
|
8310
|
+
* work without SPA rewrite rules. IDs that cannot be a safe file name are
|
|
8311
|
+
* skipped; those pages still open by navigating inside the app.
|
|
8312
|
+
*/
|
|
8313
|
+
function staticRoutes(ids) {
|
|
8314
|
+
const safe = (id) => id.length > 0 && !/[/\\\0]/.test(id) && id !== "." && id !== "..";
|
|
8315
|
+
const under = (prefix, values) => values.filter(safe).map((id) => `${prefix}/${id}`);
|
|
8316
|
+
return [
|
|
8317
|
+
"suites",
|
|
8318
|
+
"agents",
|
|
8319
|
+
"fixtures",
|
|
8320
|
+
"runs",
|
|
8321
|
+
...under("suites", ids.suites),
|
|
8322
|
+
...under("evals", ids.evals),
|
|
8323
|
+
...under("runs", ids.runs),
|
|
8324
|
+
...under("trials", ids.trials),
|
|
8325
|
+
...under("workspaces", ids.trials)
|
|
8326
|
+
];
|
|
8327
|
+
}
|
|
8328
|
+
/** `runs/abc.html` is served at `<mount>/runs/abc`; its base is one level up. */
|
|
8329
|
+
function baseFor(route) {
|
|
8330
|
+
const depth = route.split("/").length - 1;
|
|
8331
|
+
return depth ? "../".repeat(depth) : "./";
|
|
8332
|
+
}
|
|
8333
|
+
function count(value, noun) {
|
|
8334
|
+
return `${value} ${noun}${value === 1 ? "" : "s"}`;
|
|
8335
|
+
}
|
|
8336
|
+
/** Writes one trial's JSON views and candidate files; returns the file count. */
|
|
8337
|
+
async function snapshotTrial(reports, write, data, runId, trialId) {
|
|
8338
|
+
const base = `runs/${runId}/trials/${trialId}`;
|
|
8339
|
+
await write(`${base}/detail.json`, await reports.trialDetail(runId, trialId));
|
|
8340
|
+
await write(`${base}/events.json`, { events: await reports.trajectory(runId, trialId) });
|
|
8341
|
+
await write(`${base}/artifacts.json`, { artifacts: await reports.listArtifacts(runId, trialId) });
|
|
8342
|
+
const workspace = await reports.listCandidateWorkspace(runId, trialId);
|
|
8343
|
+
await write(`${base}/workspace.json`, workspace);
|
|
8344
|
+
const root = resolve(data, base, "workspace");
|
|
8345
|
+
let files = 0;
|
|
8346
|
+
for (const entry of workspace) {
|
|
8347
|
+
if (entry.kind !== "file") continue;
|
|
8348
|
+
const target = resolve(root, entry.path);
|
|
8349
|
+
if (!target.startsWith(`${root}${sep}`)) throw new Error(`Invalid workspace path: ${entry.path}`);
|
|
8350
|
+
const response = await reports.candidateFile(runId, trialId, entry.path);
|
|
8351
|
+
await mkdir(dirname(target), { recursive: true });
|
|
8352
|
+
await writeFile(target, new Uint8Array(await response.arrayBuffer()));
|
|
8353
|
+
files += 1;
|
|
8354
|
+
}
|
|
8355
|
+
return files;
|
|
8356
|
+
}
|
|
8357
|
+
/** Total size and file count of a directory, for the build summary. */
|
|
8358
|
+
async function directorySize(root) {
|
|
8359
|
+
let bytes = 0;
|
|
8360
|
+
let files = 0;
|
|
8361
|
+
for (const entry of await readdir(root, {
|
|
8362
|
+
withFileTypes: true,
|
|
8363
|
+
recursive: true
|
|
8364
|
+
})) {
|
|
8365
|
+
if (!entry.isFile()) continue;
|
|
8366
|
+
bytes += (await stat(resolve(entry.parentPath, entry.name))).size;
|
|
8367
|
+
files += 1;
|
|
8368
|
+
}
|
|
8369
|
+
return {
|
|
8370
|
+
bytes,
|
|
8261
8371
|
files
|
|
8262
8372
|
};
|
|
8263
8373
|
}
|
|
8264
|
-
/**
|
|
8265
|
-
|
|
8266
|
-
|
|
8267
|
-
|
|
8374
|
+
/**
|
|
8375
|
+
* Select the static API and make every URL relative to `<base href>`, which
|
|
8376
|
+
* points at the snapshot root. The app derives its router basename from it.
|
|
8377
|
+
*/
|
|
8378
|
+
function staticIndex(html, base = "./") {
|
|
8379
|
+
const result = html.replace("<html lang=\"en\">", "<html lang=\"en\" data-evalkit-snapshot=\"static\">").replace("<head>", `<head>\n <base href="${base}" />`).replaceAll("\"/assets/", "\"./assets/");
|
|
8380
|
+
if (!result.includes("data-evalkit-snapshot=\"static\"") || !result.includes("<base href=")) throw new Error("Bundled dashboard index.html has an unexpected shape");
|
|
8268
8381
|
return result;
|
|
8269
8382
|
}
|
|
8270
8383
|
async function prepareOutDir(outDir) {
|
|
@@ -8460,6 +8573,111 @@ async function loadProject(cwd, configPath) {
|
|
|
8460
8573
|
};
|
|
8461
8574
|
}
|
|
8462
8575
|
//#endregion
|
|
8576
|
+
//#region ../cli/src/terminal.ts
|
|
8577
|
+
/** Human CLI output. Colors and live progress only appear on an interactive terminal. */
|
|
8578
|
+
const interactive = Boolean(process.stdout.isTTY);
|
|
8579
|
+
const colored = interactive && !process.env.NO_COLOR && process.env.TERM !== "dumb";
|
|
8580
|
+
const paint = (code, reset) => (text) => colored ? `\x1b[${code}m${text}\x1b[${reset}m` : text;
|
|
8581
|
+
const bold = paint(1, 22);
|
|
8582
|
+
const dim = paint(2, 22);
|
|
8583
|
+
const red = paint(31, 39);
|
|
8584
|
+
const green = paint(32, 39);
|
|
8585
|
+
const yellow = paint(33, 39);
|
|
8586
|
+
const cyan = paint(36, 39);
|
|
8587
|
+
function banner(command, version) {
|
|
8588
|
+
console.log(`\n ${bold(cyan("EvalKit"))} ${bold(command)} ${dim(`v${version}`)}\n`);
|
|
8589
|
+
}
|
|
8590
|
+
function duration(ms) {
|
|
8591
|
+
return ms < 1e3 ? `${Math.round(ms)}ms` : `${(ms / 1e3).toFixed(2)}s`;
|
|
8592
|
+
}
|
|
8593
|
+
function bytes(size) {
|
|
8594
|
+
if (size < 1024) return `${size} B`;
|
|
8595
|
+
if (size < 1024 ** 2) return `${(size / 1024).toFixed(1)} kB`;
|
|
8596
|
+
return `${(size / 1024 ** 2).toFixed(1)} MB`;
|
|
8597
|
+
}
|
|
8598
|
+
function plural(count, noun) {
|
|
8599
|
+
return `${count} ${noun}${count === 1 ? "" : "s"}`;
|
|
8600
|
+
}
|
|
8601
|
+
/** Times one build step, showing a live line while it runs. */
|
|
8602
|
+
async function step(label, work, describe) {
|
|
8603
|
+
const started = performance.now();
|
|
8604
|
+
const progress = (detail) => {
|
|
8605
|
+
if (interactive) process.stdout.write(`\r\x1b[2K ${cyan("◇")} ${label.padEnd(20)} ${dim(detail)}`);
|
|
8606
|
+
};
|
|
8607
|
+
progress("…");
|
|
8608
|
+
try {
|
|
8609
|
+
const value = await work(progress);
|
|
8610
|
+
const elapsed = duration(performance.now() - started);
|
|
8611
|
+
if (interactive) process.stdout.write("\r\x1B[2K");
|
|
8612
|
+
console.log(` ${green("✓")} ${label.padEnd(20)} ${describe(value)} ${dim(elapsed)}`);
|
|
8613
|
+
return value;
|
|
8614
|
+
} catch (error) {
|
|
8615
|
+
if (interactive) process.stdout.write("\r\x1B[2K");
|
|
8616
|
+
console.log(` ${red("✗")} ${label}`);
|
|
8617
|
+
throw error;
|
|
8618
|
+
}
|
|
8619
|
+
}
|
|
8620
|
+
function clock() {
|
|
8621
|
+
return dim((/* @__PURE__ */ new Date()).toTimeString().slice(0, 8));
|
|
8622
|
+
}
|
|
8623
|
+
function cellLabel(name, parameters) {
|
|
8624
|
+
const shown = Object.entries(parameters).map(([key, value]) => `${key}=${typeof value === "string" ? value : JSON.stringify(value)}`).join(" ");
|
|
8625
|
+
return shown ? `${bold(name)} ${dim(shown)}` : bold(name);
|
|
8626
|
+
}
|
|
8627
|
+
function passed(result) {
|
|
8628
|
+
return result.status === "completed" && (result.aggregateScoring?.passRate === 1 || Boolean(result.scoring?.passed));
|
|
8629
|
+
}
|
|
8630
|
+
/** Live log for runs launched from the `evalkit dev` dashboard. */
|
|
8631
|
+
const runLog = {
|
|
8632
|
+
started(name, parameters = {}, suiteId) {
|
|
8633
|
+
const suite = suiteId ? dim(` (suite ${suiteId})`) : "";
|
|
8634
|
+
console.log(`${clock()} ${cyan("▶ run ")} ${cellLabel(name, parameters)}${suite}`);
|
|
8635
|
+
},
|
|
8636
|
+
finished(name, parameters, result, ms) {
|
|
8637
|
+
const ok = passed(result);
|
|
8638
|
+
const trials = result.trials?.length ?? 1;
|
|
8639
|
+
const passedTrials = result.aggregateScoring?.passed ?? (ok ? trials : 0);
|
|
8640
|
+
const status = result.status === "completed" ? ok ? green("✓ pass") : red("✗ fail") : red(`✗ ${result.status}`);
|
|
8641
|
+
console.log(`${clock()} ${status} ${cellLabel(name, parameters)} ${dim(`${passedTrials}/${trials} trials · ${duration(ms)}`)}`);
|
|
8642
|
+
console.log(` ${dim(`report ${result.reportLocation}`)}`);
|
|
8643
|
+
},
|
|
8644
|
+
cell(cell, result, ms) {
|
|
8645
|
+
runLog.finished(cell.eval.name ?? authoringId(cell.eval), cell.parameters, result, ms);
|
|
8646
|
+
},
|
|
8647
|
+
failed(name, error) {
|
|
8648
|
+
console.log(`${clock()} ${red("✗ error")} ${bold(name)} ${error instanceof Error ? error.message : String(error)}`);
|
|
8649
|
+
}
|
|
8650
|
+
};
|
|
8651
|
+
/**
|
|
8652
|
+
* The runner logs structured Effect lines; the CLI prints its own summaries, so
|
|
8653
|
+
* only warnings show by default. `EVALKIT_LOG_LEVEL=info` (or `debug`) restores them.
|
|
8654
|
+
*/
|
|
8655
|
+
function runnerLogs(effect) {
|
|
8656
|
+
const requested = process.env.EVALKIT_LOG_LEVEL?.toLowerCase();
|
|
8657
|
+
const level = LogLevel.allLevels.find((item) => item.label.toLowerCase() === requested) ?? LogLevel.Warning;
|
|
8658
|
+
return effect.pipe(Logger.withMinimumLogLevel(level));
|
|
8659
|
+
}
|
|
8660
|
+
/** True when a person can answer a prompt (not CI, pipes, or redirected input). */
|
|
8661
|
+
function canPrompt() {
|
|
8662
|
+
return Boolean(process.stdin.isTTY && process.stdout.isTTY);
|
|
8663
|
+
}
|
|
8664
|
+
async function confirm(question) {
|
|
8665
|
+
const { createInterface } = await import("node:readline/promises");
|
|
8666
|
+
const prompt = createInterface({
|
|
8667
|
+
input: process.stdin,
|
|
8668
|
+
output: process.stdout
|
|
8669
|
+
});
|
|
8670
|
+
try {
|
|
8671
|
+
const answer = await prompt.question(` ${cyan("?")} ${question} ${dim("(y/N)")} `);
|
|
8672
|
+
return /^y(es)?$/i.test(answer.trim());
|
|
8673
|
+
} catch {
|
|
8674
|
+
console.log("");
|
|
8675
|
+
return false;
|
|
8676
|
+
} finally {
|
|
8677
|
+
prompt.close();
|
|
8678
|
+
}
|
|
8679
|
+
}
|
|
8680
|
+
//#endregion
|
|
8463
8681
|
//#region ../cli/src/run-command.ts
|
|
8464
8682
|
function parseRunArgs(args) {
|
|
8465
8683
|
const { values, positionals } = parseArgs({
|
|
@@ -8607,7 +8825,7 @@ async function runProjectCommand(args, cwd = process.cwd(), parsed = parseRunArg
|
|
|
8607
8825
|
}
|
|
8608
8826
|
if (cells > (project.config.execution?.maxCells ?? 100) && !values.all) throw new Error(`Selected ${cells} cells. Narrow the selection, inspect --dry-run, or pass --all.`);
|
|
8609
8827
|
if (!values.json) console.log(`Running ${cells} cell(s) from ${plan.matrix}`);
|
|
8610
|
-
if ((await Effect.runPromise(runMatrix(matrix, {
|
|
8828
|
+
if ((await Effect.runPromise(runnerLogs(runMatrix(matrix, {
|
|
8611
8829
|
selection,
|
|
8612
8830
|
concurrency: parsed.concurrency ?? project.config.execution?.concurrency ?? 4,
|
|
8613
8831
|
trials,
|
|
@@ -8633,7 +8851,56 @@ async function runProjectCommand(args, cwd = process.cwd(), parsed = parseRunArg
|
|
|
8633
8851
|
}
|
|
8634
8852
|
}
|
|
8635
8853
|
}
|
|
8636
|
-
}))).failed) process.exitCode = 1;
|
|
8854
|
+
})))).failed) process.exitCode = 1;
|
|
8855
|
+
}
|
|
8856
|
+
//#endregion
|
|
8857
|
+
//#region ../cli/src/tidy.ts
|
|
8858
|
+
/**
|
|
8859
|
+
* Every eval the project can run: standalone, in suites, or only in matrices.
|
|
8860
|
+
* A run is archived only when its eval is in none of these.
|
|
8861
|
+
*/
|
|
8862
|
+
function projectEvalIds(registry) {
|
|
8863
|
+
return new Set([
|
|
8864
|
+
...registry.evals,
|
|
8865
|
+
...registry.suites.flatMap((suite) => suite.evals),
|
|
8866
|
+
...registry.matrices.flatMap((matrix) => matrix.evals)
|
|
8867
|
+
].map(authoringId));
|
|
8868
|
+
}
|
|
8869
|
+
/**
|
|
8870
|
+
* Mark runs of removed evals as archived, and restore runs whose eval returned.
|
|
8871
|
+
* Reports are never deleted: archiving writes a marker file beside the manifest.
|
|
8872
|
+
*/
|
|
8873
|
+
async function tidyReports(options) {
|
|
8874
|
+
const known = projectEvalIds(options.registry);
|
|
8875
|
+
const reports = createDashboardReportReader(options.reportRoot, options.registry);
|
|
8876
|
+
const result = {
|
|
8877
|
+
archived: [],
|
|
8878
|
+
restored: [],
|
|
8879
|
+
stillArchived: 0,
|
|
8880
|
+
current: 0
|
|
8881
|
+
};
|
|
8882
|
+
for (const run of await reports.listRuns()) {
|
|
8883
|
+
const marker = resolve(options.reportRoot, run.id, ARCHIVE_MARKER);
|
|
8884
|
+
const entry = {
|
|
8885
|
+
runId: run.id,
|
|
8886
|
+
evalId: run.evalId
|
|
8887
|
+
};
|
|
8888
|
+
if (known.has(run.evalId)) {
|
|
8889
|
+
result.current += 1;
|
|
8890
|
+
if (run.archived) {
|
|
8891
|
+
await rm(marker, { force: true });
|
|
8892
|
+
result.restored.push(entry);
|
|
8893
|
+
}
|
|
8894
|
+
} else if (run.archived) result.stillArchived += 1;
|
|
8895
|
+
else {
|
|
8896
|
+
await writeFile(marker, `${JSON.stringify({
|
|
8897
|
+
archivedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
8898
|
+
reason: `Eval "${run.evalId}" is no longer in the project`
|
|
8899
|
+
})}\n`);
|
|
8900
|
+
result.archived.push(entry);
|
|
8901
|
+
}
|
|
8902
|
+
}
|
|
8903
|
+
return result;
|
|
8637
8904
|
}
|
|
8638
8905
|
//#endregion
|
|
8639
8906
|
//#region ../cli/src/new-project-template.ts
|
|
@@ -8750,7 +9017,7 @@ scores, evidence, and candidate files (never evaluator workspaces): review it be
|
|
|
8750
9017
|
}
|
|
8751
9018
|
//#endregion
|
|
8752
9019
|
//#region ../cli/src/new-project.ts
|
|
8753
|
-
const npmDependency = "^0.
|
|
9020
|
+
const npmDependency = "^0.3.0";
|
|
8754
9021
|
const ignored = [
|
|
8755
9022
|
"node_modules/",
|
|
8756
9023
|
"_evalkit-results/",
|
|
@@ -8867,6 +9134,7 @@ async function writeStarterReadme(root, name, inPlace) {
|
|
|
8867
9134
|
}
|
|
8868
9135
|
//#endregion
|
|
8869
9136
|
//#region ../cli/src/index.ts
|
|
9137
|
+
const VERSION = "0.3.0";
|
|
8870
9138
|
let projectRoot = process.cwd();
|
|
8871
9139
|
let reportRoot = resolve(projectRoot, "_evalkit-results");
|
|
8872
9140
|
const sandboxRoot = resolve(projectRoot, "_evalkit-sandbox");
|
|
@@ -8886,13 +9154,17 @@ async function runEvals(options) {
|
|
|
8886
9154
|
const registry = await loadRegistry();
|
|
8887
9155
|
const evaluations = options.evalIds.map((id) => registry.get(id));
|
|
8888
9156
|
if (evaluations.some((e) => !e)) throw new Error("Unknown eval");
|
|
8889
|
-
await Effect.runPromise(Effect.all(evaluations.map((evaluation) =>
|
|
8890
|
-
|
|
8891
|
-
|
|
8892
|
-
|
|
8893
|
-
|
|
8894
|
-
|
|
8895
|
-
|
|
9157
|
+
await Effect.runPromise(runnerLogs(Effect.all(evaluations.map((evaluation) => {
|
|
9158
|
+
const name = evaluation.name ?? authoringId(evaluation);
|
|
9159
|
+
const started = performance.now();
|
|
9160
|
+
return Effect.sync(() => runLog.started(name, {}, options.suiteId)).pipe(Effect.andThen(runEval(evaluation, {
|
|
9161
|
+
report: localReportStore(reportRoot),
|
|
9162
|
+
workspaceRoot: sandboxRoot,
|
|
9163
|
+
suiteId: options.suiteId,
|
|
9164
|
+
runtime: runtimeFor(evaluation),
|
|
9165
|
+
concurrency: options.concurrency
|
|
9166
|
+
})), Effect.tap((result) => runLog.finished(name, {}, result, performance.now() - started)), Effect.tapError((error) => Effect.sync(() => runLog.failed(name, error))));
|
|
9167
|
+
}), { concurrency: options.concurrency })));
|
|
8896
9168
|
}
|
|
8897
9169
|
function registerMatrixRoutes(app, registry, dashboardMatrix, trials) {
|
|
8898
9170
|
let matrixRunning = false;
|
|
@@ -8935,14 +9207,18 @@ function registerMatrixRoutes(app, registry, dashboardMatrix, trials) {
|
|
|
8935
9207
|
}
|
|
8936
9208
|
if (matrixRunning) return context.text("A dashboard matrix cell is already running", 409);
|
|
8937
9209
|
matrixRunning = true;
|
|
8938
|
-
|
|
9210
|
+
const name = evaluation.name ?? authoringId(evaluation);
|
|
9211
|
+
const started = performance.now();
|
|
9212
|
+
runLog.started(name, body.parameters ?? {}, suiteId);
|
|
9213
|
+
Effect.runPromise(runnerLogs(runMatrix(dashboardMatrix, {
|
|
8939
9214
|
selection,
|
|
8940
9215
|
concurrency: 1,
|
|
8941
9216
|
trials,
|
|
8942
9217
|
...suiteId ? { suiteId } : {},
|
|
8943
9218
|
report: localReportStore(reportRoot),
|
|
8944
|
-
workspaceRoot: sandboxRoot
|
|
8945
|
-
|
|
9219
|
+
workspaceRoot: sandboxRoot,
|
|
9220
|
+
onResult: (cell, result) => runLog.cell(cell, result, performance.now() - started)
|
|
9221
|
+
}))).catch((error) => runLog.failed(name, error)).finally(() => {
|
|
8946
9222
|
matrixRunning = false;
|
|
8947
9223
|
});
|
|
8948
9224
|
} else {
|
|
@@ -8951,7 +9227,7 @@ function registerMatrixRoutes(app, registry, dashboardMatrix, trials) {
|
|
|
8951
9227
|
evalIds: [authoringId(evaluation)],
|
|
8952
9228
|
suiteId,
|
|
8953
9229
|
concurrency: 32
|
|
8954
|
-
}).catch((
|
|
9230
|
+
}).catch(() => {});
|
|
8955
9231
|
}
|
|
8956
9232
|
return context.json({ accepted: true }, 202);
|
|
8957
9233
|
});
|
|
@@ -8963,7 +9239,7 @@ function registerMatrixRoutes(app, registry, dashboardMatrix, trials) {
|
|
|
8963
9239
|
evalIds: suite.evals.map(authoringId),
|
|
8964
9240
|
suiteId: authoringId(suite),
|
|
8965
9241
|
concurrency: 32
|
|
8966
|
-
}).catch((
|
|
9242
|
+
}).catch(() => {});
|
|
8967
9243
|
return context.json({ accepted: true }, 202);
|
|
8968
9244
|
});
|
|
8969
9245
|
}
|
|
@@ -9047,14 +9323,32 @@ async function serveDashboard() {
|
|
|
9047
9323
|
const registry = await loadRegistry();
|
|
9048
9324
|
const dashboardRoot = dashboardAssets();
|
|
9049
9325
|
const app = new Hono();
|
|
9050
|
-
|
|
9326
|
+
const dashboardMatrix = project?.config.matrix ? registry.matrices.at(-1) : void 0;
|
|
9327
|
+
registerMatrixRoutes(app, registry, dashboardMatrix, project?.config.execution?.trials);
|
|
9051
9328
|
registerReportRoutes(app, registry, createDashboardReportReader(reportRoot, registry));
|
|
9052
9329
|
registerDashboardAssets(app, dashboardRoot);
|
|
9053
|
-
const
|
|
9054
|
-
|
|
9055
|
-
|
|
9056
|
-
|
|
9057
|
-
|
|
9330
|
+
const port = Number(process.env.PORT ?? 4317);
|
|
9331
|
+
let server;
|
|
9332
|
+
try {
|
|
9333
|
+
server = Bun.serve({
|
|
9334
|
+
port,
|
|
9335
|
+
fetch: app.fetch
|
|
9336
|
+
});
|
|
9337
|
+
} catch (error) {
|
|
9338
|
+
if (error.code === "EADDRINUSE") throw new Error(`Port ${port} is already in use. Stop the other server or choose another port: PORT=4318 evalkit dev`);
|
|
9339
|
+
throw error;
|
|
9340
|
+
}
|
|
9341
|
+
const url = `http://localhost:${server.port}`;
|
|
9342
|
+
const reader = createDashboardReportReader(reportRoot, registry);
|
|
9343
|
+
const runs = (await reader.listRuns()).length;
|
|
9344
|
+
const legacy = reader.legacyRunIds().length;
|
|
9345
|
+
const matrixCells = dashboardMatrix?.count({ parameters: {} });
|
|
9346
|
+
const row = (label, value) => console.log(` ${cyan("➜")} ${bold(label.padEnd(9))}${value}`);
|
|
9347
|
+
banner("dev", VERSION);
|
|
9348
|
+
row("Local:", cyan(url));
|
|
9349
|
+
row("Project:", `${plural(registry.evals.length, "eval")} · ${plural(registry.suites.length, "suite")}` + (dashboardMatrix ? ` · matrix ${authoringId(dashboardMatrix)} (${plural(matrixCells, "cell")})` : ""));
|
|
9350
|
+
row("Reports:", `${relative(process.cwd(), reportRoot) || "."} ${dim(`(${plural(runs, "run")}${legacy ? `, ${legacy} from an older EvalKit hidden` : ""})`)}`);
|
|
9351
|
+
console.log(`\n ${dim("Runs started from the dashboard are logged below. Press Ctrl+C to stop.")}\n`);
|
|
9058
9352
|
if (process.env.EVALKIT_NO_OPEN !== "1") {
|
|
9059
9353
|
const opener = process.platform === "darwin" ? "open" : process.platform === "win32" ? "start" : "xdg-open";
|
|
9060
9354
|
try {
|
|
@@ -9132,18 +9426,82 @@ const devCommand = make$1("dev", { config }, ({ config }) => Effect.tryPromise({
|
|
|
9132
9426
|
}));
|
|
9133
9427
|
const buildCommand = make$1("build", {
|
|
9134
9428
|
config,
|
|
9135
|
-
out: text("out").pipe(optional)
|
|
9136
|
-
|
|
9429
|
+
out: text("out").pipe(optional),
|
|
9430
|
+
run: boolean("run"),
|
|
9431
|
+
noRun: boolean("no-run")
|
|
9432
|
+
}, ({ config, out, run, noRun }) => Effect.tryPromise({
|
|
9137
9433
|
try: async () => {
|
|
9138
|
-
|
|
9434
|
+
if (run && noRun) throw new Error("Choose either --run or --no-run");
|
|
9435
|
+
const started = performance.now();
|
|
9436
|
+
const configPath = Option.getOrUndefined(config);
|
|
9437
|
+
banner("build", VERSION);
|
|
9438
|
+
await loadProjectStep(configPath);
|
|
9439
|
+
if (await shouldRunEvals({
|
|
9440
|
+
run,
|
|
9441
|
+
noRun
|
|
9442
|
+
})) await runAllEvals(configPath);
|
|
9139
9443
|
const result = await buildDashboard({
|
|
9140
9444
|
registry: project.registry,
|
|
9141
9445
|
reportRoot,
|
|
9142
9446
|
dashboardRoot: dashboardAssets(),
|
|
9143
9447
|
outDir: resolve(projectRoot, Option.getOrElse(out, () => "_evalkit-dashboard"))
|
|
9144
|
-
});
|
|
9145
|
-
|
|
9146
|
-
|
|
9448
|
+
}, step);
|
|
9449
|
+
const size = await directorySize(result.outDir);
|
|
9450
|
+
const shown = relative(process.cwd(), result.outDir) || ".";
|
|
9451
|
+
console.log(`\n ${bold(`${shown}/`)} ${dim(`${bytes(size.bytes)} · ${plural(size.files, "file")}`)}`);
|
|
9452
|
+
console.log(`\n ${bold("Built read-only dashboard")} in ${duration(performance.now() - started)}\n`);
|
|
9453
|
+
console.log(` ${cyan("➜")} ${bold("Preview:")} bunx serve ${shown}`);
|
|
9454
|
+
console.log(` ${yellow("⚠")} Contains transcripts, scores, evidence, and candidate files\n (never evaluator workspaces). Review before sharing.\n`);
|
|
9455
|
+
},
|
|
9456
|
+
catch: (error) => error
|
|
9457
|
+
}));
|
|
9458
|
+
async function loadProjectStep(configPath) {
|
|
9459
|
+
await step("Eval project", () => useProject(configPath), () => {
|
|
9460
|
+
const where = relative(process.cwd(), projectRoot);
|
|
9461
|
+
return `${plural(project.registry.evals.length, "eval")} loaded${where ? ` from ${where}` : ""}`;
|
|
9462
|
+
});
|
|
9463
|
+
}
|
|
9464
|
+
/**
|
|
9465
|
+
* A snapshot needs data: with no current results, run the evals. Otherwise ask,
|
|
9466
|
+
* or keep the existing results when no one can answer (CI, pipes).
|
|
9467
|
+
*/
|
|
9468
|
+
async function shouldRunEvals(flags) {
|
|
9469
|
+
if (flags.run) return true;
|
|
9470
|
+
if (flags.noRun) return false;
|
|
9471
|
+
const runs = (await createDashboardReportReader(reportRoot, project.registry).listRuns()).filter((entry) => !entry.archived).length;
|
|
9472
|
+
if (!runs) {
|
|
9473
|
+
console.log(` ${cyan("◇")} No results yet, so running the evals to have data to build.`);
|
|
9474
|
+
return true;
|
|
9475
|
+
}
|
|
9476
|
+
if (!canPrompt()) {
|
|
9477
|
+
console.log(` ${dim(`Using ${plural(runs, "existing run")}; pass --run to run the evals first.`)}`);
|
|
9478
|
+
return false;
|
|
9479
|
+
}
|
|
9480
|
+
return confirm(`Run the evals before building? ${dim(`(${plural(runs, "existing run")})`)}`);
|
|
9481
|
+
}
|
|
9482
|
+
async function runAllEvals(configPath) {
|
|
9483
|
+
const started = performance.now();
|
|
9484
|
+
console.log(`\n ${cyan("▶")} ${bold("Running evals")}\n`);
|
|
9485
|
+
await runProjectCommand([], process.cwd(), parseRunArgs(configPath ? ["--config", configPath] : []));
|
|
9486
|
+
const failed = process.exitCode === 1;
|
|
9487
|
+
console.log(`\n ${failed ? yellow("⚠") : cyan("✓")} ${bold("Evals finished")} in ${duration(performance.now() - started)}` + (failed ? dim(" (some failed; building with their results)") : "") + "\n");
|
|
9488
|
+
}
|
|
9489
|
+
const tidyCommand = make$1("tidy", { config }, ({ config }) => Effect.tryPromise({
|
|
9490
|
+
try: async () => {
|
|
9491
|
+
banner("tidy", VERSION);
|
|
9492
|
+
await loadProjectStep(Option.getOrUndefined(config));
|
|
9493
|
+
const result = await step("Run reports", () => tidyReports({
|
|
9494
|
+
registry: project.registry,
|
|
9495
|
+
reportRoot
|
|
9496
|
+
}), (value) => `${plural(value.current + value.archived.length + value.stillArchived, "run")} checked`);
|
|
9497
|
+
const byEval = (entries) => [...entries.reduce((groups, entry) => groups.set(entry.evalId, (groups.get(entry.evalId) ?? 0) + 1), /* @__PURE__ */ new Map()).entries()].map(([id, total]) => ` ${bold(id)} ${dim(`(${plural(total, "run")})`)}`).join("\n");
|
|
9498
|
+
console.log("");
|
|
9499
|
+
if (result.archived.length) console.log(` ${yellow("◆")} Archived ${plural(result.archived.length, "run")} of evals no longer in the project:\n${byEval(result.archived)}`);
|
|
9500
|
+
if (result.restored.length) console.log(` ${cyan("◆")} Restored ${plural(result.restored.length, "run")} whose eval is back:\n${byEval(result.restored)}`);
|
|
9501
|
+
if (!result.archived.length && !result.restored.length) console.log(` ${cyan("✓")} Nothing new to tidy.`);
|
|
9502
|
+
const totalArchived = result.archived.length + result.stillArchived;
|
|
9503
|
+
if (totalArchived) console.log(`\n ${dim(`Archived runs (${totalArchived}) are hidden in \`evalkit dev\` (filter "Show archived")`)}\n ${dim("and left out of `evalkit build`. Nothing was deleted.")}`);
|
|
9504
|
+
console.log("");
|
|
9147
9505
|
},
|
|
9148
9506
|
catch: (error) => error
|
|
9149
9507
|
}));
|
|
@@ -9151,10 +9509,11 @@ const cli = run(make$1("evalkit").pipe(withSubcommands([
|
|
|
9151
9509
|
newCommand,
|
|
9152
9510
|
runCommand,
|
|
9153
9511
|
devCommand,
|
|
9154
|
-
buildCommand
|
|
9512
|
+
buildCommand,
|
|
9513
|
+
tidyCommand
|
|
9155
9514
|
])), {
|
|
9156
9515
|
name: "EvalKit",
|
|
9157
|
-
version:
|
|
9516
|
+
version: VERSION
|
|
9158
9517
|
});
|
|
9159
9518
|
const argv = process.argv.slice();
|
|
9160
9519
|
if (argv[2] === "help") argv.splice(2, 1, "--help");
|