@leostera/evalkit 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +81 -0
- package/README.md +8 -7
- package/dist/cli.mjs +909 -149
- package/dist/cli.mjs.map +1 -1
- package/dist/dashboard/assets/index-D9NtA46z.css +2 -0
- package/dist/dashboard/assets/index-DrBwLKJD.js +30 -0
- package/dist/dashboard/index.html +2 -2
- package/dist/files-C-U0-PWW.mjs +26 -0
- package/dist/files-C-U0-PWW.mjs.map +1 -0
- package/dist/{index-M5eRlMfd.d.mts → index-U6TUTBf7.d.mts} +87 -9
- package/dist/index-U6TUTBf7.d.mts.map +1 -0
- package/dist/index-db-HdZXelOS.d.mts +142 -0
- package/dist/index-db-HdZXelOS.d.mts.map +1 -0
- package/dist/index-db-vbqEGFaU.mjs +548 -0
- package/dist/index-db-vbqEGFaU.mjs.map +1 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.mjs +2 -2
- package/dist/index.mjs.map +1 -1
- package/dist/runner.d.mts +21 -3
- package/dist/runner.d.mts.map +1 -1
- package/dist/runner.mjs +1 -1
- package/dist/share-worker.d.mts +51 -0
- package/dist/share-worker.d.mts.map +1 -0
- package/dist/share-worker.mjs +73 -0
- package/dist/share-worker.mjs.map +1 -0
- package/dist/share.d.mts +116 -0
- package/dist/share.d.mts.map +1 -0
- package/dist/share.mjs +4 -0
- package/dist/{src-DrwwgaPn.mjs → src-ButXPY8P.mjs} +20 -5
- package/dist/src-ButXPY8P.mjs.map +1 -0
- package/dist/{src-DluldGKn.mjs → src-DoPs9Yug.mjs} +11 -7
- package/dist/src-DoPs9Yug.mjs.map +1 -0
- package/dist/src-DzZueFn3.mjs +98 -0
- package/dist/src-DzZueFn3.mjs.map +1 -0
- package/package.json +10 -1
- package/dist/dashboard/assets/index-Z6raM-uW.js +0 -30
- package/dist/dashboard/assets/index-fUDsY6z-.css +0 -2
- package/dist/index-M5eRlMfd.d.mts.map +0 -1
- package/dist/src-DluldGKn.mjs.map +0 -1
- package/dist/src-DrwwgaPn.mjs.map +0 -1
package/dist/cli.mjs
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
|
-
import {
|
|
3
|
-
import { a as readTrialEvents, c as readTrialSummary, i as readRunSummary, l as localReportStore, n as runMatrix, o as readTrialManifest, r as readRunManifest, t as runEval } from "./src-
|
|
2
|
+
import { I as defineEvalMatrix, L as canonicalParameters, m as registerEvals, n as authoringId } from "./src-ButXPY8P.mjs";
|
|
3
|
+
import { a as readTrialEvents, c as readTrialSummary, i as readRunSummary, l as localReportStore, n as runMatrix, o as readTrialManifest, r as readRunManifest, t as runEval } from "./src-DoPs9Yug.mjs";
|
|
4
|
+
import { c as dashboardIndex, d as encodeBundle, o as repoKey } from "./index-db-vbqEGFaU.mjs";
|
|
5
|
+
import { t as createDirectoryShareApp } from "./src-DzZueFn3.mjs";
|
|
4
6
|
import * as Schema from "effect/Schema";
|
|
5
|
-
import { Effect, Option } from "effect";
|
|
6
|
-
import { cp, mkdir, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
|
|
7
|
+
import { Effect, LogLevel, Logger, Option } from "effect";
|
|
8
|
+
import { cp, mkdir, readFile, readdir, realpath, rm, stat, writeFile } from "node:fs/promises";
|
|
7
9
|
import * as Path from "node:path";
|
|
8
|
-
import { basename, dirname, extname, relative, resolve, sep } from "node:path";
|
|
10
|
+
import { basename, dirname, extname, join, relative, resolve, sep } from "node:path";
|
|
9
11
|
import * as OS from "node:os";
|
|
10
12
|
import * as Crypto from "node:crypto";
|
|
11
13
|
import * as NFS from "node:fs";
|
|
@@ -43,11 +45,11 @@ import * as EffectSecret from "effect/Secret";
|
|
|
43
45
|
import * as Effectable from "effect/Effectable";
|
|
44
46
|
import * as Match from "effect/Match";
|
|
45
47
|
import * as EffectNumber from "effect/Number";
|
|
46
|
-
import * as LogLevel from "effect/LogLevel";
|
|
48
|
+
import * as LogLevel$1 from "effect/LogLevel";
|
|
47
49
|
import * as Config from "effect/Config";
|
|
48
50
|
import * as HashMap from "effect/HashMap";
|
|
49
51
|
import * as Order from "effect/Order";
|
|
50
|
-
import * as Logger from "effect/Logger";
|
|
52
|
+
import * as Logger$1 from "effect/Logger";
|
|
51
53
|
import * as Unify from "effect/Unify";
|
|
52
54
|
import * as HashSet from "effect/HashSet";
|
|
53
55
|
import * as SynchronizedRef from "effect/SynchronizedRef";
|
|
@@ -65,8 +67,9 @@ import * as FiberSet from "effect/FiberSet";
|
|
|
65
67
|
import "effect/Fiber";
|
|
66
68
|
import "effect/Pool";
|
|
67
69
|
import * as MutableRef from "effect/MutableRef";
|
|
68
|
-
import { parseArgs } from "node:util";
|
|
70
|
+
import { parseArgs, promisify } from "node:util";
|
|
69
71
|
import * as ChildProcess from "node:child_process";
|
|
72
|
+
import { execFile } from "node:child_process";
|
|
70
73
|
import * as RcRef from "effect/RcRef";
|
|
71
74
|
import * as readline from "node:readline";
|
|
72
75
|
import { Hono } from "hono";
|
|
@@ -392,13 +395,13 @@ const struct = (fields) => {
|
|
|
392
395
|
/** @internal */
|
|
393
396
|
const black = { _tag: "Black" };
|
|
394
397
|
/** @internal */
|
|
395
|
-
const red$
|
|
398
|
+
const red$4 = { _tag: "Red" };
|
|
396
399
|
/** @internal */
|
|
397
|
-
const green$
|
|
400
|
+
const green$3 = { _tag: "Green" };
|
|
398
401
|
/** @internal */
|
|
399
402
|
const magenta$1 = { _tag: "Magenta" };
|
|
400
403
|
/** @internal */
|
|
401
|
-
const cyan$
|
|
404
|
+
const cyan$2 = { _tag: "Cyan" };
|
|
402
405
|
/** @internal */
|
|
403
406
|
const white$3 = { _tag: "White" };
|
|
404
407
|
/** @internal */
|
|
@@ -500,7 +503,7 @@ const ESC = "\x1B[";
|
|
|
500
503
|
const BEL = "\x07";
|
|
501
504
|
const SEP = ";";
|
|
502
505
|
/** @internal */
|
|
503
|
-
const bold$
|
|
506
|
+
const bold$2 = /*#__PURE__*/ make$5({ bold: /*#__PURE__*/ Option$1.some(/*#__PURE__*/ setBold(true)) });
|
|
504
507
|
/** @internal */
|
|
505
508
|
const italicized$1 = /*#__PURE__*/ make$5({ italicized: /*#__PURE__*/ Option$1.some(/*#__PURE__*/ setItalicized(true)) });
|
|
506
509
|
/** @internal */
|
|
@@ -512,15 +515,15 @@ const brightColor = (color) => make$5({ foreground: Option$1.some(setColor(color
|
|
|
512
515
|
/** @internal */
|
|
513
516
|
const color$1 = (color) => make$5({ foreground: Option$1.some(setColor(color, false, "foreground")) });
|
|
514
517
|
/** @internal */
|
|
515
|
-
const red$
|
|
518
|
+
const red$3 = /*#__PURE__*/ color$1(red$4);
|
|
516
519
|
/** @internal */
|
|
517
|
-
const green$
|
|
520
|
+
const green$2 = /*#__PURE__*/ color$1(green$3);
|
|
518
521
|
/** @internal */
|
|
519
522
|
const white$2 = /*#__PURE__*/ color$1(white$3);
|
|
520
523
|
/** @internal */
|
|
521
524
|
const blackBright$1 = /*#__PURE__*/ brightColor(black);
|
|
522
525
|
/** @internal */
|
|
523
|
-
const cyanBright$1 = /*#__PURE__*/ brightColor(cyan$
|
|
526
|
+
const cyanBright$1 = /*#__PURE__*/ brightColor(cyan$2);
|
|
524
527
|
/** @internal */
|
|
525
528
|
const beep$2 = /*#__PURE__*/ make$5({ commands: /*#__PURE__*/ Arr.of(BEL) });
|
|
526
529
|
/** @internal */
|
|
@@ -587,7 +590,7 @@ const stringifyInternal = (self) => {
|
|
|
587
590
|
* @since 1.0.0
|
|
588
591
|
* @category constructors
|
|
589
592
|
*/
|
|
590
|
-
const bold = bold$
|
|
593
|
+
const bold$1 = bold$2;
|
|
591
594
|
/**
|
|
592
595
|
* @since 1.0.0
|
|
593
596
|
* @category constructors
|
|
@@ -612,12 +615,12 @@ const color = color$1;
|
|
|
612
615
|
* @since 1.0.0
|
|
613
616
|
* @category colors
|
|
614
617
|
*/
|
|
615
|
-
const red$
|
|
618
|
+
const red$2 = red$3;
|
|
616
619
|
/**
|
|
617
620
|
* @since 1.0.0
|
|
618
621
|
* @category colors
|
|
619
622
|
*/
|
|
620
|
-
const green = green$
|
|
623
|
+
const green$1 = green$2;
|
|
621
624
|
/**
|
|
622
625
|
* @since 1.0.0
|
|
623
626
|
* @category colors
|
|
@@ -2218,7 +2221,7 @@ const optimize = optimize$1;
|
|
|
2218
2221
|
* @since 1.0.0
|
|
2219
2222
|
* @category constructors
|
|
2220
2223
|
*/
|
|
2221
|
-
const red = red$
|
|
2224
|
+
const red$1 = red$4;
|
|
2222
2225
|
/**
|
|
2223
2226
|
* @since 1.0.0
|
|
2224
2227
|
* @category constructors
|
|
@@ -2228,7 +2231,7 @@ const magenta = magenta$1;
|
|
|
2228
2231
|
* @since 1.0.0
|
|
2229
2232
|
* @category constructors
|
|
2230
2233
|
*/
|
|
2231
|
-
const cyan = cyan$
|
|
2234
|
+
const cyan$1 = cyan$2;
|
|
2232
2235
|
/**
|
|
2233
2236
|
* @since 1.0.0
|
|
2234
2237
|
* @category constructors
|
|
@@ -2248,7 +2251,7 @@ const space = /*#__PURE__*/ text$6(" ");
|
|
|
2248
2251
|
/** @internal */
|
|
2249
2252
|
const code = (value) => highlight(value, white);
|
|
2250
2253
|
/** @internal */
|
|
2251
|
-
const error = (value) => highlight(value, red);
|
|
2254
|
+
const error = (value) => highlight(value, red$1);
|
|
2252
2255
|
/** @internal */
|
|
2253
2256
|
const highlight = (value, color) => ({
|
|
2254
2257
|
_tag: "Highlight",
|
|
@@ -2307,7 +2310,7 @@ const toAnsiDoc$1 = (self) => {
|
|
|
2307
2310
|
switch (self._tag) {
|
|
2308
2311
|
case "Highlight": return annotate(toAnsiDoc$1(self.value), color(self.color));
|
|
2309
2312
|
case "Sequence": return cat(toAnsiDoc$1(self.left), toAnsiDoc$1(self.right));
|
|
2310
|
-
case "Strong": return annotate(toAnsiDoc$1(self.value), bold);
|
|
2313
|
+
case "Strong": return annotate(toAnsiDoc$1(self.value), bold$1);
|
|
2311
2314
|
case "Text": return text$7(self.value);
|
|
2312
2315
|
case "URI": return annotate(text$7(self.value), underlined);
|
|
2313
2316
|
case "Weak": return annotate(toAnsiDoc$1(self.value), blackBright);
|
|
@@ -2373,11 +2376,11 @@ const toAnsiText = (self) => render$1(toAnsiDoc(self), { style: "pretty" });
|
|
|
2373
2376
|
const toAnsiDocInternal = (self) => {
|
|
2374
2377
|
switch (self._tag) {
|
|
2375
2378
|
case "Empty": return empty$3;
|
|
2376
|
-
case "Header": return pipe(annotate(toAnsiDoc$1(self.value), bold), cat(hardLine));
|
|
2379
|
+
case "Header": return pipe(annotate(toAnsiDoc$1(self.value), bold$1), cat(hardLine));
|
|
2377
2380
|
case "Paragraph": return pipe(toAnsiDoc$1(self.value), cat(hardLine));
|
|
2378
2381
|
case "DescriptionList": {
|
|
2379
2382
|
const definitions = self.definitions.map(([span, doc]) => cats([
|
|
2380
|
-
annotate(toAnsiDoc$1(span), bold),
|
|
2383
|
+
annotate(toAnsiDoc$1(span), bold$1),
|
|
2381
2384
|
empty$3,
|
|
2382
2385
|
indent(toAnsiDocInternal(doc), 2)
|
|
2383
2386
|
]));
|
|
@@ -2592,8 +2595,8 @@ function renderError$2(state, pointer) {
|
|
|
2592
2595
|
onSome: (error) => {
|
|
2593
2596
|
const errorLines = error.split(NEWLINE_REGEX$3);
|
|
2594
2597
|
if (Arr.isNonEmptyReadonlyArray(errorLines)) {
|
|
2595
|
-
const annotateLine = (line) => annotate(text$7(line), combine(italicized, red$
|
|
2596
|
-
const prefix = cat(annotate(pointer, red$
|
|
2598
|
+
const annotateLine = (line) => annotate(text$7(line), combine(italicized, red$2));
|
|
2599
|
+
const prefix = cat(annotate(pointer, red$2), space$1);
|
|
2597
2600
|
const lines = Arr.map(errorLines, (str) => annotateLine(str));
|
|
2598
2601
|
return cursorSavePosition.pipe(cat(hardLine), cat(prefix), cat(align(vsep(lines))), cat(cursorRestorePosition));
|
|
2599
2602
|
}
|
|
@@ -2612,7 +2615,7 @@ function renderParts(state, submitted = false) {
|
|
|
2612
2615
|
});
|
|
2613
2616
|
}
|
|
2614
2617
|
function renderOutput$4(leadingSymbol, trailingSymbol, parts, options) {
|
|
2615
|
-
const annotateLine = (line) => annotate(text$7(line), bold);
|
|
2618
|
+
const annotateLine = (line) => annotate(text$7(line), bold$1);
|
|
2616
2619
|
const prefix = cat(leadingSymbol, space$1);
|
|
2617
2620
|
return Arr.match(options.message.split(/\r?\n/), {
|
|
2618
2621
|
onEmpty: () => hsep([
|
|
@@ -2642,7 +2645,7 @@ function renderSubmission$5(state, options) {
|
|
|
2642
2645
|
return Effect$1.gen(function* () {
|
|
2643
2646
|
const columns = yield* (yield* Terminal).columns;
|
|
2644
2647
|
const figures$11 = yield* figures;
|
|
2645
|
-
return renderOutput$4(annotate(figures$11.tick, green), annotate(figures$11.ellipsis, blackBright), renderParts(state, true), options).pipe(cat(hardLine), optimize(Deep), render$1({
|
|
2648
|
+
return renderOutput$4(annotate(figures$11.tick, green$1), annotate(figures$11.ellipsis, blackBright), renderParts(state, true), options).pipe(cat(hardLine), optimize(Deep), render$1({
|
|
2646
2649
|
style: "pretty",
|
|
2647
2650
|
options: { lineWidth: columns }
|
|
2648
2651
|
}));
|
|
@@ -3106,7 +3109,7 @@ function handleClear$4(options) {
|
|
|
3106
3109
|
}
|
|
3107
3110
|
const NEWLINE_REGEX$2 = /\r?\n/;
|
|
3108
3111
|
function renderPrompt(confirm, message, leadingSymbol, trailingSymbol) {
|
|
3109
|
-
const annotateLine = (line) => annotate(text$7(line), bold);
|
|
3112
|
+
const annotateLine = (line) => annotate(text$7(line), bold$1);
|
|
3110
3113
|
const prefix = cat(leadingSymbol, space$1);
|
|
3111
3114
|
return Arr.match(message.split(NEWLINE_REGEX$2), {
|
|
3112
3115
|
onEmpty: () => hsep([
|
|
@@ -3159,7 +3162,7 @@ function renderNextFrame$4(state, options) {
|
|
|
3159
3162
|
options: { lineWidth: columns }
|
|
3160
3163
|
}));
|
|
3161
3164
|
}
|
|
3162
|
-
const leadingSymbol = annotate(figures$10.tick, green);
|
|
3165
|
+
const leadingSymbol = annotate(figures$10.tick, green$1);
|
|
3163
3166
|
const trailingSymbol = annotate(figures$10.ellipsis, blackBright);
|
|
3164
3167
|
const promptMsg = renderPrompt(empty$3, options.message, leadingSymbol, trailingSymbol);
|
|
3165
3168
|
const files = renderFiles(state, state.files, figures$10, options);
|
|
@@ -3173,7 +3176,7 @@ function renderSubmission$4(value, options) {
|
|
|
3173
3176
|
return Effect$1.gen(function* () {
|
|
3174
3177
|
const columns = yield* (yield* Terminal).columns;
|
|
3175
3178
|
const figures$9 = yield* figures;
|
|
3176
|
-
const leadingSymbol = annotate(figures$9.tick, green);
|
|
3179
|
+
const leadingSymbol = annotate(figures$9.tick, green$1);
|
|
3177
3180
|
const trailingSymbol = annotate(figures$9.ellipsis, blackBright);
|
|
3178
3181
|
return renderPrompt(empty$3, options.message, leadingSymbol, trailingSymbol).pipe(cat(space$1), cat(annotate(text$7(value), white$1)), cat(hardLine), render$1({
|
|
3179
3182
|
style: "pretty",
|
|
@@ -3316,7 +3319,7 @@ function handleClear$3(options) {
|
|
|
3316
3319
|
function renderInput$1(state, submitted) {
|
|
3317
3320
|
const annotation = Option$1.match(state.error, {
|
|
3318
3321
|
onNone: () => combine(underlined, cyanBright),
|
|
3319
|
-
onSome: () => red$
|
|
3322
|
+
onSome: () => red$2
|
|
3320
3323
|
});
|
|
3321
3324
|
const value = state.value === "" ? empty$3 : text$7(`${state.value}`);
|
|
3322
3325
|
return submitted ? value : annotate(value, annotation);
|
|
@@ -3328,8 +3331,8 @@ function renderError$1(state, pointer) {
|
|
|
3328
3331
|
onSome: (error) => Arr.match(error.split(NEWLINE_REGEX$1), {
|
|
3329
3332
|
onEmpty: () => empty$3,
|
|
3330
3333
|
onNonEmpty: (errorLines) => {
|
|
3331
|
-
const annotateLine = (line) => annotate(text$7(line), combine(italicized, red$
|
|
3332
|
-
const prefix = cat(annotate(pointer, red$
|
|
3334
|
+
const annotateLine = (line) => annotate(text$7(line), combine(italicized, red$2));
|
|
3335
|
+
const prefix = cat(annotate(pointer, red$2), space$1);
|
|
3333
3336
|
const lines = Arr.map(errorLines, (str) => annotateLine(str));
|
|
3334
3337
|
return cursorSavePosition.pipe(cat(hardLine), cat(prefix), cat(align(vsep(lines))), cat(cursorRestorePosition));
|
|
3335
3338
|
}
|
|
@@ -3337,7 +3340,7 @@ function renderError$1(state, pointer) {
|
|
|
3337
3340
|
});
|
|
3338
3341
|
}
|
|
3339
3342
|
function renderOutput$3(state, leadingSymbol, trailingSymbol, options, submitted = false) {
|
|
3340
|
-
const annotateLine = (line) => annotate(text$7(line), bold);
|
|
3343
|
+
const annotateLine = (line) => annotate(text$7(line), bold$1);
|
|
3341
3344
|
const prefix = cat(leadingSymbol, space$1);
|
|
3342
3345
|
return Arr.match(options.message.split(/\r?\n/), {
|
|
3343
3346
|
onEmpty: () => hsep([
|
|
@@ -3368,7 +3371,7 @@ function renderSubmission$3(nextState, options) {
|
|
|
3368
3371
|
return Effect$1.gen(function* () {
|
|
3369
3372
|
const columns = yield* (yield* Terminal).columns;
|
|
3370
3373
|
const figures$7 = yield* figures;
|
|
3371
|
-
return renderOutput$3(nextState, annotate(figures$7.tick, green), annotate(figures$7.ellipsis, blackBright), options, true).pipe(cat(hardLine), optimize(Deep), render$1({
|
|
3374
|
+
return renderOutput$3(nextState, annotate(figures$7.tick, green$1), annotate(figures$7.ellipsis, blackBright), options, true).pipe(cat(hardLine), optimize(Deep), render$1({
|
|
3372
3375
|
style: "pretty",
|
|
3373
3376
|
options: { lineWidth: columns }
|
|
3374
3377
|
}));
|
|
@@ -3463,7 +3466,7 @@ function handleProcessInteger(options) {
|
|
|
3463
3466
|
};
|
|
3464
3467
|
}
|
|
3465
3468
|
/** @internal */
|
|
3466
|
-
const integer = (options) => {
|
|
3469
|
+
const integer$3 = (options) => {
|
|
3467
3470
|
const opts = {
|
|
3468
3471
|
min: Number.NEGATIVE_INFINITY,
|
|
3469
3472
|
max: Number.POSITIVE_INFINITY,
|
|
@@ -3551,7 +3554,7 @@ const float = (options) => {
|
|
|
3551
3554
|
const renderBeep$2 = /*#__PURE__*/ render$1(beep, { style: "pretty" });
|
|
3552
3555
|
const NEWLINE_REGEX = /\r?\n/;
|
|
3553
3556
|
function renderOutput$2(leadingSymbol, trailingSymbol, options) {
|
|
3554
|
-
const annotateLine = (line) => annotate(text$7(line), bold);
|
|
3557
|
+
const annotateLine = (line) => annotate(text$7(line), bold$1);
|
|
3555
3558
|
const prefix = cat(leadingSymbol, space$1);
|
|
3556
3559
|
return Arr.match(options.message.split(NEWLINE_REGEX), {
|
|
3557
3560
|
onEmpty: () => hsep([prefix, trailingSymbol]),
|
|
@@ -3566,7 +3569,7 @@ function renderChoicePrefix(state, choices, toDisplay, currentIndex, figures) {
|
|
|
3566
3569
|
if (currentIndex === toDisplay.startIndex && toDisplay.startIndex > 0) prefix = figures.arrowUp;
|
|
3567
3570
|
else if (currentIndex === toDisplay.endIndex - 1 && toDisplay.endIndex < choices.length) prefix = figures.arrowDown;
|
|
3568
3571
|
if (choices[currentIndex].disabled) {
|
|
3569
|
-
const annotation = combine(bold, blackBright);
|
|
3572
|
+
const annotation = combine(bold$1, blackBright);
|
|
3570
3573
|
return state === currentIndex ? figures.pointer.pipe(annotate(annotation), cat(prefix)) : prefix.pipe(cat(space$1));
|
|
3571
3574
|
}
|
|
3572
3575
|
return state === currentIndex ? figures.pointer.pipe(annotate(cyanBright), cat(prefix)) : prefix.pipe(cat(space$1));
|
|
@@ -3611,7 +3614,7 @@ function renderSubmission$2(state, options) {
|
|
|
3611
3614
|
const columns = yield* (yield* Terminal).columns;
|
|
3612
3615
|
const figures$6 = yield* figures;
|
|
3613
3616
|
const selected = text$7(options.choices[state].title);
|
|
3614
|
-
return renderOutput$2(annotate(figures$6.tick, green), annotate(figures$6.ellipsis, blackBright), options).pipe(cat(space$1), cat(annotate(selected, white$1)), cat(hardLine), render$1({
|
|
3617
|
+
return renderOutput$2(annotate(figures$6.tick, green$1), annotate(figures$6.ellipsis, blackBright), options).pipe(cat(space$1), cat(annotate(selected, white$1)), cat(hardLine), render$1({
|
|
3615
3618
|
style: "pretty",
|
|
3616
3619
|
options: { lineWidth: columns }
|
|
3617
3620
|
}));
|
|
@@ -3714,7 +3717,7 @@ function renderInput(nextState, options, submitted) {
|
|
|
3714
3717
|
if (nextState.value.length === 0) return blackBright;
|
|
3715
3718
|
return combine(underlined, cyanBright);
|
|
3716
3719
|
},
|
|
3717
|
-
onSome: () => red$
|
|
3720
|
+
onSome: () => red$2
|
|
3718
3721
|
});
|
|
3719
3722
|
switch (options.type) {
|
|
3720
3723
|
case "hidden": return empty$3;
|
|
@@ -3728,8 +3731,8 @@ function renderError(nextState, pointer) {
|
|
|
3728
3731
|
onSome: (error) => Arr.match(error.split(/\r?\n/), {
|
|
3729
3732
|
onEmpty: () => empty$3,
|
|
3730
3733
|
onNonEmpty: (errorLines) => {
|
|
3731
|
-
const annotateLine = (line) => text$7(line).pipe(annotate(combine(italicized, red$
|
|
3732
|
-
const prefix = cat(annotate(pointer, red$
|
|
3734
|
+
const annotateLine = (line) => text$7(line).pipe(annotate(combine(italicized, red$2)));
|
|
3735
|
+
const prefix = cat(annotate(pointer, red$2), space$1);
|
|
3733
3736
|
const lines = Arr.map(errorLines, (str) => annotateLine(str));
|
|
3734
3737
|
return cursorSavePosition.pipe(cat(hardLine), cat(prefix), cat(align(vsep(lines))), cat(cursorRestorePosition));
|
|
3735
3738
|
}
|
|
@@ -3737,7 +3740,7 @@ function renderError(nextState, pointer) {
|
|
|
3737
3740
|
});
|
|
3738
3741
|
}
|
|
3739
3742
|
function renderOutput$1(nextState, leadingSymbol, trailingSymbol, options, submitted = false) {
|
|
3740
|
-
const annotateLine = (line) => annotate(text$7(line), bold);
|
|
3743
|
+
const annotateLine = (line) => annotate(text$7(line), bold$1);
|
|
3741
3744
|
const promptLines = options.message.split(/\r?\n/);
|
|
3742
3745
|
const prefix = cat(leadingSymbol, space$1);
|
|
3743
3746
|
if (Arr.isNonEmptyReadonlyArray(promptLines)) {
|
|
@@ -3767,7 +3770,7 @@ function renderSubmission$1(state, options) {
|
|
|
3767
3770
|
return Effect$1.gen(function* () {
|
|
3768
3771
|
const columns = yield* (yield* Terminal).columns;
|
|
3769
3772
|
const figures$4 = yield* figures;
|
|
3770
|
-
return renderOutput$1(state, annotate(figures$4.tick, green), annotate(figures$4.ellipsis, blackBright), options, true).pipe(cat(hardLine), optimize(Deep), render$1({
|
|
3773
|
+
return renderOutput$1(state, annotate(figures$4.tick, green$1), annotate(figures$4.ellipsis, blackBright), options, true).pipe(cat(hardLine), optimize(Deep), render$1({
|
|
3771
3774
|
style: "pretty",
|
|
3772
3775
|
options: { lineWidth: columns }
|
|
3773
3776
|
}));
|
|
@@ -3909,7 +3912,7 @@ function renderToggle(value, options, submitted = false) {
|
|
|
3909
3912
|
]);
|
|
3910
3913
|
}
|
|
3911
3914
|
function renderOutput(toggle, leadingSymbol, trailingSymbol, options) {
|
|
3912
|
-
const annotateLine = (line) => annotate(text$7(line), bold);
|
|
3915
|
+
const annotateLine = (line) => annotate(text$7(line), bold$1);
|
|
3913
3916
|
const promptLines = options.message.split(/\r?\n/);
|
|
3914
3917
|
const prefix = cat(leadingSymbol, space$1);
|
|
3915
3918
|
if (Arr.isNonEmptyReadonlyArray(promptLines)) {
|
|
@@ -3941,7 +3944,7 @@ function renderSubmission(value, options) {
|
|
|
3941
3944
|
const terminal = yield* Terminal;
|
|
3942
3945
|
const figures$2 = yield* figures;
|
|
3943
3946
|
const columns = yield* terminal.columns;
|
|
3944
|
-
const leadingSymbol = annotate(figures$2.tick, green);
|
|
3947
|
+
const leadingSymbol = annotate(figures$2.tick, green$1);
|
|
3945
3948
|
const trailingSymbol = annotate(figures$2.ellipsis, blackBright);
|
|
3946
3949
|
return renderOutput(renderToggle(value, options, true), leadingSymbol, trailingSymbol, options).pipe(cat(hardLine), optimize(Deep), render$1({
|
|
3947
3950
|
style: "pretty",
|
|
@@ -4029,6 +4032,12 @@ const choice = (alternatives) => {
|
|
|
4029
4032
|
return op;
|
|
4030
4033
|
};
|
|
4031
4034
|
/** @internal */
|
|
4035
|
+
const integer$2 = /*#__PURE__*/ (() => {
|
|
4036
|
+
const op = /*#__PURE__*/ Object.create(proto$5);
|
|
4037
|
+
op._tag = "Integer";
|
|
4038
|
+
return op;
|
|
4039
|
+
})();
|
|
4040
|
+
/** @internal */
|
|
4032
4041
|
const text$4 = /*#__PURE__*/ (() => {
|
|
4033
4042
|
const op = /*#__PURE__*/ Object.create(proto$5);
|
|
4034
4043
|
op._tag = "Text";
|
|
@@ -4184,7 +4193,7 @@ const wizardInternal$3 = (self, help) => {
|
|
|
4184
4193
|
case "Integer": {
|
|
4185
4194
|
const primitiveHelp = p("Enter an integer");
|
|
4186
4195
|
const message = sequence(help, primitiveHelp);
|
|
4187
|
-
return integer({ message: toAnsiText(message).trimEnd() }).pipe(map$3((value) => `${value}`));
|
|
4196
|
+
return integer$3({ message: toAnsiText(message).trimEnd() }).pipe(map$3((value) => `${value}`));
|
|
4188
4197
|
}
|
|
4189
4198
|
case "Path": {
|
|
4190
4199
|
const primitiveHelp = p("Select a file system path");
|
|
@@ -4689,7 +4698,7 @@ const wizardInternal$2 = (self, config) => {
|
|
|
4689
4698
|
case "Variadic": {
|
|
4690
4699
|
const repeatHelp = p("How many times should this argument should be repeated?");
|
|
4691
4700
|
const message = pipe(getHelpInternal$2(self), sequence(repeatHelp));
|
|
4692
|
-
return integer({
|
|
4701
|
+
return integer$3({
|
|
4693
4702
|
message: toAnsiText(message).trimEnd(),
|
|
4694
4703
|
min: getMinSizeInternal$1(self),
|
|
4695
4704
|
max: getMaxSizeInternal$1(self)
|
|
@@ -4874,13 +4883,15 @@ const boolean$1 = (name, options) => {
|
|
|
4874
4883
|
const head = Arr.headNonEmpty(negationNames);
|
|
4875
4884
|
const tail = Arr.tailNonEmpty(negationNames);
|
|
4876
4885
|
const negationOption = makeSingle(head, tail, boolean$2(Option$1.some(!ifPresent)));
|
|
4877
|
-
return withDefault(orElse(option, negationOption), !ifPresent);
|
|
4886
|
+
return withDefault$1(orElse(option, negationOption), !ifPresent);
|
|
4878
4887
|
}
|
|
4879
|
-
return withDefault(option, !ifPresent);
|
|
4888
|
+
return withDefault$1(option, !ifPresent);
|
|
4880
4889
|
};
|
|
4881
4890
|
/** @internal */
|
|
4882
4891
|
const choiceWithValue = (name, choices) => makeSingle(name, Arr.empty(), choice(choices));
|
|
4883
4892
|
/** @internal */
|
|
4893
|
+
const integer$1 = (name) => makeSingle(name, Arr.empty(), integer$2);
|
|
4894
|
+
/** @internal */
|
|
4884
4895
|
const none = /*#__PURE__*/ (() => {
|
|
4885
4896
|
const op = /*#__PURE__*/ Object.create(proto$2);
|
|
4886
4897
|
op._tag = "Empty";
|
|
@@ -4895,7 +4906,7 @@ const getUsage$1 = (self) => getUsageInternal$1(self);
|
|
|
4895
4906
|
/** @internal */
|
|
4896
4907
|
const map$1 = /*#__PURE__*/ dual(2, (self, f) => makeMap(self, (a) => Either$1.right(f(a))));
|
|
4897
4908
|
/** @internal */
|
|
4898
|
-
const optional$1 = (self) => withDefault(map$1(self, Option$1.some), Option$1.none());
|
|
4909
|
+
const optional$1 = (self) => withDefault$1(map$1(self, Option$1.some), Option$1.none());
|
|
4899
4910
|
/** @internal */
|
|
4900
4911
|
const orElse = /*#__PURE__*/ dual(2, (self, that) => orElseEither(self, that).pipe(map$1(Either$1.merge)));
|
|
4901
4912
|
/** @internal */
|
|
@@ -4917,7 +4928,7 @@ const withAlias = /*#__PURE__*/ dual(2, (self, alias) => modifySingle(self, (sin
|
|
|
4917
4928
|
return makeSingle(single.name, aliases, single.primitiveType, single.description, single.pseudoName);
|
|
4918
4929
|
}));
|
|
4919
4930
|
/** @internal */
|
|
4920
|
-
const withDefault = /*#__PURE__*/ dual(2, (self, fallback) => makeWithDefault(self, fallback));
|
|
4931
|
+
const withDefault$1 = /*#__PURE__*/ dual(2, (self, fallback) => makeWithDefault(self, fallback));
|
|
4921
4932
|
/** @internal */
|
|
4922
4933
|
const withDescription = /*#__PURE__*/ dual(2, (self, desc) => modifySingle(self, (single) => {
|
|
4923
4934
|
const description = sequence(single.description, p(desc));
|
|
@@ -5305,7 +5316,7 @@ const wizardInternal$1 = (self, config) => {
|
|
|
5305
5316
|
case "Variadic": {
|
|
5306
5317
|
const repeatHelp = p("How many times should this argument be repeated?");
|
|
5307
5318
|
const message = pipe(getHelpInternal$1(self), sequence(repeatHelp));
|
|
5308
|
-
return integer({
|
|
5319
|
+
return integer$3({
|
|
5309
5320
|
message: toAnsiText(message).trimEnd(),
|
|
5310
5321
|
min: getMinSizeInternal(self),
|
|
5311
5322
|
max: getMaxSizeInternal(self)
|
|
@@ -5919,7 +5930,7 @@ const completionsOptions = /*#__PURE__*/ choiceWithValue("completions", [
|
|
|
5919
5930
|
["zsh", "zsh"]
|
|
5920
5931
|
]).pipe(optional$1, /*#__PURE__*/ withDescription("Generate a completion script for a specific shell."));
|
|
5921
5932
|
/** @internal */
|
|
5922
|
-
const logLevelOptions = /*#__PURE__*/ choiceWithValue("log-level", LogLevel.allLevels.map((level) => [level._tag.toLowerCase(), level])).pipe(optional$1, /*#__PURE__*/ withDescription("Sets the minimum log level for a command."));
|
|
5933
|
+
const logLevelOptions = /*#__PURE__*/ choiceWithValue("log-level", LogLevel$1.allLevels.map((level) => [level._tag.toLowerCase(), level])).pipe(optional$1, /*#__PURE__*/ withDescription("Sets the minimum log level for a command."));
|
|
5923
5934
|
/** @internal */
|
|
5924
5935
|
const helpOptions = /*#__PURE__*/ boolean$1("help").pipe(/*#__PURE__*/ withAlias("h"), /*#__PURE__*/ withDescription("Show the help documentation for a command."));
|
|
5925
5936
|
/** @internal */
|
|
@@ -5957,6 +5968,11 @@ const boolean = boolean$1;
|
|
|
5957
5968
|
* @since 1.0.0
|
|
5958
5969
|
* @category constructors
|
|
5959
5970
|
*/
|
|
5971
|
+
const integer = integer$1;
|
|
5972
|
+
/**
|
|
5973
|
+
* @since 1.0.0
|
|
5974
|
+
* @category constructors
|
|
5975
|
+
*/
|
|
5960
5976
|
const text = text$1;
|
|
5961
5977
|
/**
|
|
5962
5978
|
* @since 1.0.0
|
|
@@ -5974,6 +5990,11 @@ const optional = optional$1;
|
|
|
5974
5990
|
* @category combinators
|
|
5975
5991
|
*/
|
|
5976
5992
|
const repeated = repeated$1;
|
|
5993
|
+
/**
|
|
5994
|
+
* @since 1.0.0
|
|
5995
|
+
* @category combinators
|
|
5996
|
+
*/
|
|
5997
|
+
const withDefault = withDefault$1;
|
|
5977
5998
|
//#endregion
|
|
5978
5999
|
//#region ../../node_modules/.bun/@effect+cli@0.77.2+74e017304edf4bf0/node_modules/@effect/cli/dist/esm/internal/commandDirective.js
|
|
5979
6000
|
/** @internal */
|
|
@@ -6294,7 +6315,7 @@ const wizardInternal = (self, prefix, config) => {
|
|
|
6294
6315
|
case "GetUserInput":
|
|
6295
6316
|
case "Standard": return Effect$1.gen(function* () {
|
|
6296
6317
|
const logCurrentCommand = Ref.get(commandLineRef).pipe(Effect$1.flatMap((commandLine) => {
|
|
6297
|
-
const currentCommand = p(pipe(strong(highlight("COMMAND:", cyan)), concat$1(space), concat$1(highlight(Arr.join(commandLine, " "), magenta))));
|
|
6318
|
+
const currentCommand = p(pipe(strong(highlight("COMMAND:", cyan$1)), concat$1(space), concat$1(highlight(Arr.join(commandLine, " "), magenta))));
|
|
6298
6319
|
return Console.log(toAnsiText(currentCommand));
|
|
6299
6320
|
}));
|
|
6300
6321
|
if (isStandard(self)) {
|
|
@@ -6634,7 +6655,7 @@ const handleBuiltInOption = (self, executable, args, builtIn, execute, config, o
|
|
|
6634
6655
|
filteredArgs.push(args[i]);
|
|
6635
6656
|
}
|
|
6636
6657
|
const nextArgs = Arr.appendAll(baseArgs, filteredArgs);
|
|
6637
|
-
return run$2(self, nextArgs, execute).pipe(Logger.withMinimumLogLevel(builtIn.level));
|
|
6658
|
+
return run$2(self, nextArgs, execute).pipe(Logger$1.withMinimumLogLevel(builtIn.level));
|
|
6638
6659
|
}
|
|
6639
6660
|
case "ShowHelp": {
|
|
6640
6661
|
const banner = h1(code(self.name));
|
|
@@ -6740,7 +6761,7 @@ const renderWizardArgs = (args) => {
|
|
|
6740
6761
|
return blocks([
|
|
6741
6762
|
p(strong(code("Wizard Mode Complete!"))),
|
|
6742
6763
|
p(executeMsg),
|
|
6743
|
-
p(concat$1(text$6(" "), highlight(params, cyan)))
|
|
6764
|
+
p(concat$1(text$6(" "), highlight(params, cyan$1)))
|
|
6744
6765
|
]);
|
|
6745
6766
|
};
|
|
6746
6767
|
const isLogLevelArg = (arg) => {
|
|
@@ -8050,6 +8071,8 @@ const layer = /*#__PURE__*/ pipe(/*#__PURE__*/ Layer.mergeAll(layer$3, layer$5,
|
|
|
8050
8071
|
//#endregion
|
|
8051
8072
|
//#region ../cli/src/dashboard-reports.ts
|
|
8052
8073
|
const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
8074
|
+
/** Written into a run directory by `evalkit tidy`; deleting it restores the run. */
|
|
8075
|
+
const ARCHIVE_MARKER = "archived.json";
|
|
8053
8076
|
function isMissing(error) {
|
|
8054
8077
|
return error.code === "ENOENT";
|
|
8055
8078
|
}
|
|
@@ -8086,15 +8109,35 @@ function fileContentType(file) {
|
|
|
8086
8109
|
".js": "text/javascript"
|
|
8087
8110
|
}[extname(file)] ?? "application/octet-stream";
|
|
8088
8111
|
}
|
|
8089
|
-
/**
|
|
8112
|
+
/**
|
|
8113
|
+
* Read saved local reports. A missing summary means "running"; malformed reports are errors.
|
|
8114
|
+
* Runs written by an older EvalKit (manifest `schemaVersion` < 2) are skipped and counted.
|
|
8115
|
+
*/
|
|
8090
8116
|
function createDashboardReportReader(root, registry) {
|
|
8117
|
+
const legacyRuns = /* @__PURE__ */ new Set();
|
|
8091
8118
|
async function listRuns() {
|
|
8092
8119
|
const entries = await ifPresent(() => readdir(root, { withFileTypes: true }));
|
|
8093
8120
|
if (!entries) return [];
|
|
8094
8121
|
const ids = entries.filter((entry) => entry.isDirectory() && UUID_PATTERN.test(entry.name)).map((entry) => entry.name);
|
|
8095
8122
|
return (await Promise.all(ids.map(readRun))).filter((run) => run !== void 0);
|
|
8096
8123
|
}
|
|
8124
|
+
/** True for a run from a report format this version cannot read; never for corrupt JSON. */
|
|
8125
|
+
async function isLegacyRun(id) {
|
|
8126
|
+
const text = await ifPresent(() => readFile(resolve(root, id, "manifest.json"), "utf8"));
|
|
8127
|
+
if (text === void 0) return false;
|
|
8128
|
+
let version;
|
|
8129
|
+
try {
|
|
8130
|
+
version = JSON.parse(text).schemaVersion;
|
|
8131
|
+
} catch {
|
|
8132
|
+
return false;
|
|
8133
|
+
}
|
|
8134
|
+
return typeof version === "number" && version < 2;
|
|
8135
|
+
}
|
|
8097
8136
|
async function readRun(id) {
|
|
8137
|
+
if (await isLegacyRun(id)) {
|
|
8138
|
+
legacyRuns.add(id);
|
|
8139
|
+
return;
|
|
8140
|
+
}
|
|
8098
8141
|
const manifest = await ifPresent(() => readRunManifest(root, id));
|
|
8099
8142
|
if (!manifest) return void 0;
|
|
8100
8143
|
const summary = await ifPresent(() => readRunSummary(root, id));
|
|
@@ -8103,6 +8146,8 @@ function createDashboardReportReader(root, registry) {
|
|
|
8103
8146
|
const scores = trials.flatMap((trial) => trial?.scoring?.overall === void 0 ? [] : [trial.scoring.overall]);
|
|
8104
8147
|
return {
|
|
8105
8148
|
id,
|
|
8149
|
+
...await ifPresent(() => stat(resolve(root, id, "archived.json"))) ? { archived: true } : {},
|
|
8150
|
+
...manifest.source ? { source: manifest.source } : {},
|
|
8106
8151
|
evalId: manifest.evalId,
|
|
8107
8152
|
...manifest.suiteId ? { suiteId: manifest.suiteId } : {},
|
|
8108
8153
|
...manifest.matrix ? {
|
|
@@ -8171,6 +8216,8 @@ function createDashboardReportReader(root, registry) {
|
|
|
8171
8216
|
}
|
|
8172
8217
|
return {
|
|
8173
8218
|
listRuns,
|
|
8219
|
+
/** Run IDs skipped by the last `listRuns()` because an older EvalKit wrote them. */
|
|
8220
|
+
legacyRunIds: () => [...legacyRuns],
|
|
8174
8221
|
listTrials,
|
|
8175
8222
|
trialDetail,
|
|
8176
8223
|
listArtifacts,
|
|
@@ -8205,67 +8252,172 @@ async function walkArtifacts(root) {
|
|
|
8205
8252
|
//#region ../cli/src/dashboard-build.ts
|
|
8206
8253
|
/** Marks a directory as EvalKit-owned so a rebuild may safely replace it. */
|
|
8207
8254
|
const MARKER = ".evalkit-dashboard";
|
|
8208
|
-
|
|
8209
|
-
|
|
8210
|
-
* the catalog and local reports, and each trial's candidate artifact files.
|
|
8211
|
-
* Evaluator workspaces are never copied; reports may still contain sensitive
|
|
8212
|
-
* messages, scores, or evidence, so review the output before sharing it.
|
|
8213
|
-
*/
|
|
8214
|
-
async function buildDashboard(options) {
|
|
8255
|
+
const quietStep = (_label, work) => work(() => {});
|
|
8256
|
+
async function buildDashboard(options, step = quietStep) {
|
|
8215
8257
|
const { registry, reportRoot, dashboardRoot, outDir } = options;
|
|
8216
|
-
await
|
|
8217
|
-
await cp(dashboardRoot, outDir, { recursive: true });
|
|
8218
|
-
await writeFile(resolve(outDir, MARKER), "Generated by `evalkit build`.\n");
|
|
8219
|
-
await writeFile(resolve(outDir, "index.html"), staticIndex(await readFile(resolve(dashboardRoot, "index.html"), "utf8")));
|
|
8258
|
+
const template = await readFile(resolve(dashboardRoot, "index.html"), "utf8");
|
|
8220
8259
|
const data = resolve(outDir, "data");
|
|
8221
8260
|
const write = async (path, value) => {
|
|
8222
8261
|
const file = resolve(data, path);
|
|
8223
8262
|
await mkdir(dirname(file), { recursive: true });
|
|
8224
8263
|
await writeFile(file, `${JSON.stringify(value)}\n`);
|
|
8225
8264
|
};
|
|
8265
|
+
await step("Dashboard assets", async () => {
|
|
8266
|
+
await prepareOutDir(outDir);
|
|
8267
|
+
await cp(dashboardRoot, outDir, { recursive: true });
|
|
8268
|
+
await writeFile(resolve(outDir, MARKER), "Generated by `evalkit build`.\n");
|
|
8269
|
+
await writeFile(resolve(outDir, "index.html"), dashboardIndex(template, { base: "./" }));
|
|
8270
|
+
}, () => "read-only UI with relative assets");
|
|
8271
|
+
await step("Eval catalog", async () => {
|
|
8272
|
+
await write("catalog.json", { evals: registry.catalog() });
|
|
8273
|
+
await write("evals.json", { evals: registry.metadata() });
|
|
8274
|
+
await write("suites.json", { suites: registry.suiteMetadata() });
|
|
8275
|
+
return {
|
|
8276
|
+
evals: registry.evals.length,
|
|
8277
|
+
suites: registry.suites.length
|
|
8278
|
+
};
|
|
8279
|
+
}, ({ evals, suites }) => `${count(evals, "eval")} · ${count(suites, "suite")}`);
|
|
8226
8280
|
const reports = createDashboardReportReader(reportRoot, registry);
|
|
8227
|
-
const
|
|
8228
|
-
|
|
8229
|
-
|
|
8230
|
-
|
|
8231
|
-
|
|
8232
|
-
|
|
8281
|
+
const snapshot = await step("Run reports", async (progress) => {
|
|
8282
|
+
const all = await reports.listRuns();
|
|
8283
|
+
const runs = all.filter((run) => !run.archived);
|
|
8284
|
+
await write("runs.json", { runs });
|
|
8285
|
+
let trials = 0;
|
|
8286
|
+
let files = 0;
|
|
8287
|
+
const trialIds = [];
|
|
8288
|
+
for (const [index, run] of runs.entries()) {
|
|
8289
|
+
progress(`run ${index + 1}/${runs.length} · ${run.id.slice(0, 8)}`);
|
|
8290
|
+
const written = await snapshotRun(reports, run.id, directorySink(resolve(data, "runs", run.id)));
|
|
8291
|
+
trials += written.trialIds.length;
|
|
8292
|
+
files += written.files;
|
|
8293
|
+
trialIds.push(...written.trialIds);
|
|
8294
|
+
}
|
|
8295
|
+
return {
|
|
8296
|
+
runs: runs.length,
|
|
8297
|
+
trials,
|
|
8298
|
+
files,
|
|
8299
|
+
skipped: reports.legacyRunIds().length,
|
|
8300
|
+
archived: all.length - runs.length,
|
|
8301
|
+
runIds: runs.map((run) => run.id),
|
|
8302
|
+
trialIds
|
|
8303
|
+
};
|
|
8304
|
+
}, ({ runs, trials, files, skipped, archived }) => `${count(runs, "run")} · ${count(trials, "trial")} · ${count(files, "candidate file")}` + (archived ? ` · ${archived} archived excluded` : "") + (skipped ? ` · ${skipped} older-format skipped` : ""));
|
|
8305
|
+
await step("Route pages", async () => {
|
|
8306
|
+
const routes = staticRoutes({
|
|
8307
|
+
suites: registry.suites.map(authoringId),
|
|
8308
|
+
evals: registry.catalog().map((entry) => entry.path),
|
|
8309
|
+
runs: snapshot.runIds,
|
|
8310
|
+
trials: snapshot.trialIds
|
|
8311
|
+
});
|
|
8312
|
+
for (const route of routes) {
|
|
8313
|
+
const file = resolve(outDir, `${route}.html`);
|
|
8314
|
+
await mkdir(dirname(file), { recursive: true });
|
|
8315
|
+
await writeFile(file, dashboardIndex(template, { base: baseFor(route) }));
|
|
8316
|
+
}
|
|
8317
|
+
await writeFile(resolve(outDir, "404.html"), dashboardIndex(template, { base: "/" }));
|
|
8318
|
+
return routes.length;
|
|
8319
|
+
}, (pages) => `${count(pages, "page")} with clean /path URLs`);
|
|
8320
|
+
const { runIds: _runs, trialIds: _trials, ...summary } = snapshot;
|
|
8321
|
+
return {
|
|
8322
|
+
outDir,
|
|
8323
|
+
...summary
|
|
8324
|
+
};
|
|
8325
|
+
}
|
|
8326
|
+
/**
|
|
8327
|
+
* Every dashboard URL known at build time, as file paths without `.html`.
|
|
8328
|
+
* Static hosts serve `/runs/abc` from `runs/abc.html`, so links and reloads
|
|
8329
|
+
* work without SPA rewrite rules. IDs that cannot be a safe file name are
|
|
8330
|
+
* skipped; those pages still open by navigating inside the app.
|
|
8331
|
+
*/
|
|
8332
|
+
function staticRoutes(ids) {
|
|
8333
|
+
const safe = (id) => id.length > 0 && !/[/\\\0]/.test(id) && id !== "." && id !== "..";
|
|
8334
|
+
const under = (prefix, values) => values.filter(safe).map((id) => `${prefix}/${id}`);
|
|
8335
|
+
return [
|
|
8336
|
+
"suites",
|
|
8337
|
+
"agents",
|
|
8338
|
+
"fixtures",
|
|
8339
|
+
"runs",
|
|
8340
|
+
...under("suites", ids.suites),
|
|
8341
|
+
...under("evals", ids.evals),
|
|
8342
|
+
...under("runs", ids.runs),
|
|
8343
|
+
...under("trials", ids.trials),
|
|
8344
|
+
...under("workspaces", ids.trials)
|
|
8345
|
+
];
|
|
8346
|
+
}
|
|
8347
|
+
/** `runs/abc.html` is served at `<mount>/runs/abc`; its base is one level up. */
|
|
8348
|
+
function baseFor(route) {
|
|
8349
|
+
const depth = route.split("/").length - 1;
|
|
8350
|
+
return depth ? "../".repeat(depth) : "./";
|
|
8351
|
+
}
|
|
8352
|
+
function count(value, noun) {
|
|
8353
|
+
return `${value} ${noun}${value === 1 ? "" : "s"}`;
|
|
8354
|
+
}
|
|
8355
|
+
/** Writes a run's snapshot below `root` (static builds). */
|
|
8356
|
+
function directorySink(root) {
|
|
8357
|
+
const target = (path) => {
|
|
8358
|
+
const file = resolve(root, path);
|
|
8359
|
+
if (!file.startsWith(`${resolve(root)}${sep}`)) throw new Error(`Invalid snapshot path: ${path}`);
|
|
8360
|
+
return file;
|
|
8361
|
+
};
|
|
8362
|
+
return {
|
|
8363
|
+
async json(path, value) {
|
|
8364
|
+
const file = target(path);
|
|
8365
|
+
await mkdir(dirname(file), { recursive: true });
|
|
8366
|
+
await writeFile(file, `${JSON.stringify(value)}\n`);
|
|
8367
|
+
},
|
|
8368
|
+
async bytes(path, data) {
|
|
8369
|
+
const file = target(path);
|
|
8370
|
+
await mkdir(dirname(file), { recursive: true });
|
|
8371
|
+
await writeFile(file, data);
|
|
8372
|
+
}
|
|
8373
|
+
};
|
|
8374
|
+
}
|
|
8375
|
+
/**
|
|
8376
|
+
* One run's snapshot, shared by `evalkit build` (files on disk) and
|
|
8377
|
+
* `evalkit share` (an upload bundle): `trials.json`, then per trial
|
|
8378
|
+
* `detail.json`, `events.json`, `artifacts.json`, `workspace.json`, and the
|
|
8379
|
+
* candidate files under `workspace/`. Evaluator workspaces are never read.
|
|
8380
|
+
*/
|
|
8381
|
+
async function snapshotRun(reports, runId, sink, options = {}) {
|
|
8382
|
+
const trials = await reports.listTrials(runId);
|
|
8383
|
+
await sink.json("trials.json", { trials });
|
|
8233
8384
|
let files = 0;
|
|
8234
|
-
for (const
|
|
8235
|
-
const
|
|
8236
|
-
await
|
|
8237
|
-
|
|
8238
|
-
|
|
8239
|
-
|
|
8240
|
-
|
|
8241
|
-
|
|
8242
|
-
|
|
8243
|
-
|
|
8244
|
-
await
|
|
8245
|
-
|
|
8246
|
-
|
|
8247
|
-
const response = await reports.candidateFile(run.id, trial.id, entry.path);
|
|
8248
|
-
const target = resolve(data, base, "workspace", entry.path);
|
|
8249
|
-
const root = resolve(data, base, "workspace");
|
|
8250
|
-
if (!target.startsWith(`${root}${sep}`)) throw new Error(`Invalid workspace path: ${entry.path}`);
|
|
8251
|
-
await mkdir(dirname(target), { recursive: true });
|
|
8252
|
-
await writeFile(target, new Uint8Array(await response.arrayBuffer()));
|
|
8253
|
-
files += 1;
|
|
8254
|
-
}
|
|
8385
|
+
for (const trial of trials) {
|
|
8386
|
+
const base = `trials/${trial.id}`;
|
|
8387
|
+
await sink.json(`${base}/detail.json`, await reports.trialDetail(runId, trial.id));
|
|
8388
|
+
await sink.json(`${base}/events.json`, { events: await reports.trajectory(runId, trial.id) });
|
|
8389
|
+
await sink.json(`${base}/artifacts.json`, { artifacts: await reports.listArtifacts(runId, trial.id) });
|
|
8390
|
+
const workspace = await reports.listCandidateWorkspace(runId, trial.id);
|
|
8391
|
+
await sink.json(`${base}/workspace.json`, workspace);
|
|
8392
|
+
if (options.workspaceFiles === false) continue;
|
|
8393
|
+
for (const entry of workspace) {
|
|
8394
|
+
if (entry.kind !== "file") continue;
|
|
8395
|
+
const response = await reports.candidateFile(runId, trial.id, entry.path);
|
|
8396
|
+
await sink.bytes(`${base}/workspace/${entry.path}`, new Uint8Array(await response.arrayBuffer()));
|
|
8397
|
+
files += 1;
|
|
8255
8398
|
}
|
|
8256
8399
|
}
|
|
8257
8400
|
return {
|
|
8258
|
-
|
|
8259
|
-
runs: runs.length,
|
|
8260
|
-
trials,
|
|
8401
|
+
trialIds: trials.map((trial) => trial.id),
|
|
8261
8402
|
files
|
|
8262
8403
|
};
|
|
8263
8404
|
}
|
|
8264
|
-
/**
|
|
8265
|
-
function
|
|
8266
|
-
|
|
8267
|
-
|
|
8268
|
-
|
|
8405
|
+
/** Total size and file count of a directory, for the build summary. */
|
|
8406
|
+
async function directorySize(root) {
|
|
8407
|
+
let bytes = 0;
|
|
8408
|
+
let files = 0;
|
|
8409
|
+
for (const entry of await readdir(root, {
|
|
8410
|
+
withFileTypes: true,
|
|
8411
|
+
recursive: true
|
|
8412
|
+
})) {
|
|
8413
|
+
if (!entry.isFile()) continue;
|
|
8414
|
+
bytes += (await stat(resolve(entry.parentPath, entry.name))).size;
|
|
8415
|
+
files += 1;
|
|
8416
|
+
}
|
|
8417
|
+
return {
|
|
8418
|
+
bytes,
|
|
8419
|
+
files
|
|
8420
|
+
};
|
|
8269
8421
|
}
|
|
8270
8422
|
async function prepareOutDir(outDir) {
|
|
8271
8423
|
let entries;
|
|
@@ -8460,6 +8612,231 @@ async function loadProject(cwd, configPath) {
|
|
|
8460
8612
|
};
|
|
8461
8613
|
}
|
|
8462
8614
|
//#endregion
|
|
8615
|
+
//#region ../cli/src/terminal.ts
|
|
8616
|
+
/** Human CLI output. Colors and live progress only appear on an interactive terminal. */
|
|
8617
|
+
const interactive = Boolean(process.stdout.isTTY);
|
|
8618
|
+
const colored = interactive && !process.env.NO_COLOR && process.env.TERM !== "dumb";
|
|
8619
|
+
const paint = (code, reset) => (text) => colored ? `\x1b[${code}m${text}\x1b[${reset}m` : text;
|
|
8620
|
+
const bold = paint(1, 22);
|
|
8621
|
+
const dim = paint(2, 22);
|
|
8622
|
+
const red = paint(31, 39);
|
|
8623
|
+
const green = paint(32, 39);
|
|
8624
|
+
const yellow = paint(33, 39);
|
|
8625
|
+
const cyan = paint(36, 39);
|
|
8626
|
+
function banner(command, version) {
|
|
8627
|
+
console.log(`\n ${bold(cyan("EvalKit"))} ${bold(command)} ${dim(`v${version}`)}\n`);
|
|
8628
|
+
}
|
|
8629
|
+
function duration(ms) {
|
|
8630
|
+
return ms < 1e3 ? `${Math.round(ms)}ms` : `${(ms / 1e3).toFixed(2)}s`;
|
|
8631
|
+
}
|
|
8632
|
+
function bytes(size) {
|
|
8633
|
+
if (size < 1024) return `${size} B`;
|
|
8634
|
+
if (size < 1024 ** 2) return `${(size / 1024).toFixed(1)} kB`;
|
|
8635
|
+
return `${(size / 1024 ** 2).toFixed(1)} MB`;
|
|
8636
|
+
}
|
|
8637
|
+
function plural(count, noun) {
|
|
8638
|
+
return `${count} ${noun}${count === 1 ? "" : "s"}`;
|
|
8639
|
+
}
|
|
8640
|
+
/** Times one build step, showing a live line while it runs. */
|
|
8641
|
+
async function step(label, work, describe) {
|
|
8642
|
+
const started = performance.now();
|
|
8643
|
+
const progress = (detail) => {
|
|
8644
|
+
if (interactive) process.stdout.write(`\r\x1b[2K ${cyan("◇")} ${label.padEnd(20)} ${dim(detail)}`);
|
|
8645
|
+
};
|
|
8646
|
+
progress("…");
|
|
8647
|
+
try {
|
|
8648
|
+
const value = await work(progress);
|
|
8649
|
+
const elapsed = duration(performance.now() - started);
|
|
8650
|
+
if (interactive) process.stdout.write("\r\x1B[2K");
|
|
8651
|
+
console.log(` ${green("✓")} ${label.padEnd(20)} ${describe(value)} ${dim(elapsed)}`);
|
|
8652
|
+
return value;
|
|
8653
|
+
} catch (error) {
|
|
8654
|
+
if (interactive) process.stdout.write("\r\x1B[2K");
|
|
8655
|
+
console.log(` ${red("✗")} ${label}`);
|
|
8656
|
+
throw error;
|
|
8657
|
+
}
|
|
8658
|
+
}
|
|
8659
|
+
function clock() {
|
|
8660
|
+
return dim((/* @__PURE__ */ new Date()).toTimeString().slice(0, 8));
|
|
8661
|
+
}
|
|
8662
|
+
function cellLabel(name, parameters) {
|
|
8663
|
+
const shown = Object.entries(parameters).map(([key, value]) => `${key}=${typeof value === "string" ? value : JSON.stringify(value)}`).join(" ");
|
|
8664
|
+
return shown ? `${bold(name)} ${dim(shown)}` : bold(name);
|
|
8665
|
+
}
|
|
8666
|
+
function passed(result) {
|
|
8667
|
+
return result.status === "completed" && (result.aggregateScoring?.passRate === 1 || Boolean(result.scoring?.passed));
|
|
8668
|
+
}
|
|
8669
|
+
/** Live log for runs launched from the `evalkit dev` dashboard. */
|
|
8670
|
+
const runLog = {
|
|
8671
|
+
started(name, parameters = {}, suiteId) {
|
|
8672
|
+
const suite = suiteId ? dim(` (suite ${suiteId})`) : "";
|
|
8673
|
+
console.log(`${clock()} ${cyan("▶ run ")} ${cellLabel(name, parameters)}${suite}`);
|
|
8674
|
+
},
|
|
8675
|
+
finished(name, parameters, result, ms) {
|
|
8676
|
+
const ok = passed(result);
|
|
8677
|
+
const trials = result.trials?.length ?? 1;
|
|
8678
|
+
const passedTrials = result.aggregateScoring?.passed ?? (ok ? trials : 0);
|
|
8679
|
+
const status = result.status === "completed" ? ok ? green("✓ pass") : red("✗ fail") : red(`✗ ${result.status}`);
|
|
8680
|
+
console.log(`${clock()} ${status} ${cellLabel(name, parameters)} ${dim(`${passedTrials}/${trials} trials · ${duration(ms)}`)}`);
|
|
8681
|
+
console.log(` ${dim(`report ${result.reportLocation}`)}`);
|
|
8682
|
+
},
|
|
8683
|
+
cell(cell, result, ms) {
|
|
8684
|
+
runLog.finished(cell.eval.name ?? authoringId(cell.eval), cell.parameters, result, ms);
|
|
8685
|
+
},
|
|
8686
|
+
failed(name, error) {
|
|
8687
|
+
console.log(`${clock()} ${red("✗ error")} ${bold(name)} ${error instanceof Error ? error.message : String(error)}`);
|
|
8688
|
+
}
|
|
8689
|
+
};
|
|
8690
|
+
/**
|
|
8691
|
+
* The runner logs structured Effect lines; the CLI prints its own summaries, so
|
|
8692
|
+
* only warnings show by default. `EVALKIT_LOG_LEVEL=info` (or `debug`) restores them.
|
|
8693
|
+
*/
|
|
8694
|
+
function runnerLogs(effect) {
|
|
8695
|
+
const requested = process.env.EVALKIT_LOG_LEVEL?.toLowerCase();
|
|
8696
|
+
const level = LogLevel.allLevels.find((item) => item.label.toLowerCase() === requested) ?? LogLevel.Warning;
|
|
8697
|
+
return effect.pipe(Logger.withMinimumLogLevel(level));
|
|
8698
|
+
}
|
|
8699
|
+
/** True when a person can answer a prompt (not CI, pipes, or redirected input). */
|
|
8700
|
+
function canPrompt() {
|
|
8701
|
+
return Boolean(process.stdin.isTTY && process.stdout.isTTY);
|
|
8702
|
+
}
|
|
8703
|
+
async function confirm(question) {
|
|
8704
|
+
const { createInterface } = await import("node:readline/promises");
|
|
8705
|
+
const prompt = createInterface({
|
|
8706
|
+
input: process.stdin,
|
|
8707
|
+
output: process.stdout
|
|
8708
|
+
});
|
|
8709
|
+
try {
|
|
8710
|
+
const answer = await prompt.question(` ${cyan("?")} ${question} ${dim("(y/N)")} `);
|
|
8711
|
+
return /^y(es)?$/i.test(answer.trim());
|
|
8712
|
+
} catch {
|
|
8713
|
+
console.log("");
|
|
8714
|
+
return false;
|
|
8715
|
+
} finally {
|
|
8716
|
+
prompt.close();
|
|
8717
|
+
}
|
|
8718
|
+
}
|
|
8719
|
+
//#endregion
|
|
8720
|
+
//#region ../cli/src/source.ts
|
|
8721
|
+
const exec = promisify(execFile);
|
|
8722
|
+
/**
|
|
8723
|
+
* Normalize a git remote to a stable repository key: lowercase host, no
|
|
8724
|
+
* credentials, port, scheme, `.git`, or trailing slash.
|
|
8725
|
+
*
|
|
8726
|
+
* git@GitLab.example.com:group/sub/repo.git → gitlab.example.com/group/sub/repo
|
|
8727
|
+
* https://user:token@github.com/o/r/ → github.com/o/r
|
|
8728
|
+
* /srv/repos/thing.git (no host) → local/thing
|
|
8729
|
+
*/
|
|
8730
|
+
function normalizeRepoUrl(url) {
|
|
8731
|
+
const trimmed = url.trim().replace(/\/+$/, "").replace(/\.git$/, "");
|
|
8732
|
+
const scp = /^(?:[^@/]+@)?([^:/]+):(?!\/)(.+)$/.exec(trimmed);
|
|
8733
|
+
if (scp && !/^[a-z]+:\/\//i.test(trimmed)) return `${scp[1].toLowerCase()}/${scp[2].replace(/^\/+/, "")}`;
|
|
8734
|
+
try {
|
|
8735
|
+
const parsed = new URL(trimmed);
|
|
8736
|
+
if (parsed.protocol !== "file:" && parsed.hostname) return `${parsed.hostname.toLowerCase()}${parsed.pathname.replace(/\/+$/, "")}`;
|
|
8737
|
+
} catch {}
|
|
8738
|
+
return `local/${basename(trimmed)}`;
|
|
8739
|
+
}
|
|
8740
|
+
async function git(cwd, ...args) {
|
|
8741
|
+
try {
|
|
8742
|
+
const { stdout } = await exec("git", [
|
|
8743
|
+
"-C",
|
|
8744
|
+
cwd,
|
|
8745
|
+
...args
|
|
8746
|
+
], { timeout: 1e4 });
|
|
8747
|
+
return stdout.trim() || void 0;
|
|
8748
|
+
} catch {
|
|
8749
|
+
return;
|
|
8750
|
+
}
|
|
8751
|
+
}
|
|
8752
|
+
const posix = (path) => path.split(sep).join("/");
|
|
8753
|
+
const projectPath = (root, projectRoot) => root ? posix(relative(root, projectRoot)) : "";
|
|
8754
|
+
function ciSource(env, projectRoot) {
|
|
8755
|
+
if (env.EVALKIT_SOURCE_REPO && env.EVALKIT_SOURCE_COMMIT) return {
|
|
8756
|
+
repo: normalizeRepoUrl(env.EVALKIT_SOURCE_REPO),
|
|
8757
|
+
project: env.EVALKIT_SOURCE_PROJECT ?? "",
|
|
8758
|
+
commit: env.EVALKIT_SOURCE_COMMIT,
|
|
8759
|
+
...env.EVALKIT_SOURCE_REF ? { ref: env.EVALKIT_SOURCE_REF } : {},
|
|
8760
|
+
...env.EVALKIT_SOURCE_COMMITTED_AT ? { committedAt: env.EVALKIT_SOURCE_COMMITTED_AT } : {}
|
|
8761
|
+
};
|
|
8762
|
+
if (env.GITLAB_CI === "true" && env.CI_PROJECT_URL && env.CI_COMMIT_SHA) return {
|
|
8763
|
+
repo: normalizeRepoUrl(env.CI_PROJECT_URL),
|
|
8764
|
+
project: projectPath(env.CI_PROJECT_DIR, projectRoot),
|
|
8765
|
+
commit: env.CI_COMMIT_SHA,
|
|
8766
|
+
...env.CI_MERGE_REQUEST_SOURCE_BRANCH_NAME ?? env.CI_COMMIT_REF_NAME ? { ref: env.CI_MERGE_REQUEST_SOURCE_BRANCH_NAME ?? env.CI_COMMIT_REF_NAME } : {},
|
|
8767
|
+
...env.CI_COMMIT_TIMESTAMP ? { committedAt: env.CI_COMMIT_TIMESTAMP } : {},
|
|
8768
|
+
ci: {
|
|
8769
|
+
provider: "gitlab",
|
|
8770
|
+
...env.CI_PIPELINE_ID ? { pipeline: env.CI_PIPELINE_ID } : {},
|
|
8771
|
+
...env.CI_JOB_ID ? { job: env.CI_JOB_ID } : {},
|
|
8772
|
+
...env.CI_JOB_URL ? { url: env.CI_JOB_URL } : {}
|
|
8773
|
+
}
|
|
8774
|
+
};
|
|
8775
|
+
if (env.GITHUB_ACTIONS === "true" && env.GITHUB_REPOSITORY && env.GITHUB_SHA) {
|
|
8776
|
+
const server = env.GITHUB_SERVER_URL ?? "https://github.com";
|
|
8777
|
+
return {
|
|
8778
|
+
repo: normalizeRepoUrl(`${server}/${env.GITHUB_REPOSITORY}`),
|
|
8779
|
+
project: projectPath(env.GITHUB_WORKSPACE, projectRoot),
|
|
8780
|
+
commit: env.GITHUB_SHA,
|
|
8781
|
+
...env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME ? { ref: env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME } : {},
|
|
8782
|
+
ci: {
|
|
8783
|
+
provider: "github",
|
|
8784
|
+
...env.GITHUB_RUN_ID ? { pipeline: env.GITHUB_RUN_ID } : {},
|
|
8785
|
+
...env.GITHUB_JOB ? { job: env.GITHUB_JOB } : {},
|
|
8786
|
+
...env.GITHUB_RUN_ID ? { url: `${server}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}` } : {}
|
|
8787
|
+
}
|
|
8788
|
+
};
|
|
8789
|
+
}
|
|
8790
|
+
}
|
|
8791
|
+
/**
|
|
8792
|
+
* Determine which repository and commit this project's eval code comes from.
|
|
8793
|
+
* Order: EVALKIT_SOURCE_* variables, GitLab/GitHub CI variables, then git.
|
|
8794
|
+
* Returns undefined outside git with no CI variables ("untracked" results).
|
|
8795
|
+
*
|
|
8796
|
+
* `outputDirs` (results, sandbox, snapshot) never make the tree dirty, even
|
|
8797
|
+
* when a project forgot to ignore them.
|
|
8798
|
+
*/
|
|
8799
|
+
async function detectSource(givenRoot, options = {}) {
|
|
8800
|
+
const env = options.env ?? process.env;
|
|
8801
|
+
const projectRoot = await realpath(givenRoot).catch(() => givenRoot);
|
|
8802
|
+
const top = await git(projectRoot, "rev-parse", "--show-toplevel");
|
|
8803
|
+
const excludes = top ? (options.outputDirs ?? []).map((dir) => join(projectRoot, relative(givenRoot, dir))).map((dir) => posix(relative(top, dir))).filter((dir) => dir && !dir.startsWith("..")).map((dir) => `:(exclude)${dir}`) : [];
|
|
8804
|
+
const dirty = top ? Boolean(await git(top, "status", "--porcelain", "--", ".", ...excludes)) : false;
|
|
8805
|
+
const fromCi = ciSource(env, projectRoot);
|
|
8806
|
+
if (fromCi) return {
|
|
8807
|
+
...fromCi,
|
|
8808
|
+
...top && !env.EVALKIT_SOURCE_PROJECT ? { project: projectPath(top, projectRoot) } : {},
|
|
8809
|
+
dirty
|
|
8810
|
+
};
|
|
8811
|
+
if (!top) return void 0;
|
|
8812
|
+
const commit = await git(top, "rev-parse", "HEAD");
|
|
8813
|
+
if (!commit) return void 0;
|
|
8814
|
+
const remote = await git(top, "config", "--get", "remote.origin.url") ?? (await git(top, "remote"))?.split("\n").map((name) => name.trim()).find(Boolean);
|
|
8815
|
+
const remoteUrl = remote && !remote.includes("/") && !remote.includes(":") ? await git(top, "config", "--get", `remote.${remote}.url`) : remote;
|
|
8816
|
+
const ref = await git(top, "symbolic-ref", "--short", "-q", "HEAD");
|
|
8817
|
+
const committedAt = await git(top, "show", "-s", "--format=%cI", "HEAD");
|
|
8818
|
+
return {
|
|
8819
|
+
repo: normalizeRepoUrl(remoteUrl ?? top),
|
|
8820
|
+
project: projectPath(top, projectRoot),
|
|
8821
|
+
commit,
|
|
8822
|
+
...ref ? { ref } : {},
|
|
8823
|
+
...committedAt ? { committedAt } : {},
|
|
8824
|
+
dirty
|
|
8825
|
+
};
|
|
8826
|
+
}
|
|
8827
|
+
/** The project's evals and suites, saved with each run (`catalog.json`). */
|
|
8828
|
+
function runCatalog(registry) {
|
|
8829
|
+
return {
|
|
8830
|
+
catalog: registry.catalog(),
|
|
8831
|
+
evals: registry.metadata(),
|
|
8832
|
+
suites: registry.suiteMetadata()
|
|
8833
|
+
};
|
|
8834
|
+
}
|
|
8835
|
+
/** `abc1234` or `abc1234+dirty`. */
|
|
8836
|
+
function shortSource(source) {
|
|
8837
|
+
return `${source.commit.slice(0, 7)}${source.dirty ? "+dirty" : ""}`;
|
|
8838
|
+
}
|
|
8839
|
+
//#endregion
|
|
8463
8840
|
//#region ../cli/src/run-command.ts
|
|
8464
8841
|
function parseRunArgs(args) {
|
|
8465
8842
|
const { values, positionals } = parseArgs({
|
|
@@ -8606,15 +8983,24 @@ async function runProjectCommand(args, cwd = process.cwd(), parsed = parseRunArg
|
|
|
8606
8983
|
return;
|
|
8607
8984
|
}
|
|
8608
8985
|
if (cells > (project.config.execution?.maxCells ?? 100) && !values.all) throw new Error(`Selected ${cells} cells. Narrow the selection, inspect --dry-run, or pass --all.`);
|
|
8609
|
-
|
|
8610
|
-
|
|
8986
|
+
const reportDir = resolve(project.root, project.config.reportDir ?? "_evalkit-results");
|
|
8987
|
+
const sandboxDir = resolve(project.root, project.config.sandboxDir ?? "_evalkit-sandbox");
|
|
8988
|
+
const source = await detectSource(project.root, { outputDirs: [
|
|
8989
|
+
reportDir,
|
|
8990
|
+
sandboxDir,
|
|
8991
|
+
resolve(project.root, "_evalkit-dashboard")
|
|
8992
|
+
] });
|
|
8993
|
+
if (!values.json) console.log(`Running ${cells} cell(s) from ${plan.matrix}` + (source ? ` at ${shortSource(source)}${source.ref ? ` (${source.ref})` : ""}` : " (untracked: not in a git repository)"));
|
|
8994
|
+
if ((await Effect.runPromise(runnerLogs(runMatrix(matrix, {
|
|
8611
8995
|
selection,
|
|
8612
8996
|
concurrency: parsed.concurrency ?? project.config.execution?.concurrency ?? 4,
|
|
8613
8997
|
trials,
|
|
8614
8998
|
...suiteId ? { suiteId } : {},
|
|
8615
8999
|
...values.local ? { runtime: "local" } : {},
|
|
8616
|
-
|
|
8617
|
-
|
|
9000
|
+
...source ? { source } : {},
|
|
9001
|
+
catalog: runCatalog(registry),
|
|
9002
|
+
report: localReportStore(reportDir),
|
|
9003
|
+
workspaceRoot: sandboxDir,
|
|
8618
9004
|
onResult: (cell, result) => {
|
|
8619
9005
|
if (values.json) console.log(JSON.stringify({
|
|
8620
9006
|
cell: {
|
|
@@ -8633,7 +9019,211 @@ async function runProjectCommand(args, cwd = process.cwd(), parsed = parseRunArg
|
|
|
8633
9019
|
}
|
|
8634
9020
|
}
|
|
8635
9021
|
}
|
|
8636
|
-
}))).failed) process.exitCode = 1;
|
|
9022
|
+
})))).failed) process.exitCode = 1;
|
|
9023
|
+
}
|
|
9024
|
+
//#endregion
|
|
9025
|
+
//#region ../cli/src/tidy.ts
|
|
9026
|
+
/**
|
|
9027
|
+
* Every eval the project can run: standalone, in suites, or only in matrices.
|
|
9028
|
+
* A run is archived only when its eval is in none of these.
|
|
9029
|
+
*/
|
|
9030
|
+
function projectEvalIds(registry) {
|
|
9031
|
+
return new Set([
|
|
9032
|
+
...registry.evals,
|
|
9033
|
+
...registry.suites.flatMap((suite) => suite.evals),
|
|
9034
|
+
...registry.matrices.flatMap((matrix) => matrix.evals)
|
|
9035
|
+
].map(authoringId));
|
|
9036
|
+
}
|
|
9037
|
+
/**
|
|
9038
|
+
* Mark runs of removed evals as archived, and restore runs whose eval returned.
|
|
9039
|
+
* Reports are never deleted: archiving writes a marker file beside the manifest.
|
|
9040
|
+
*/
|
|
9041
|
+
async function tidyReports(options) {
|
|
9042
|
+
const known = projectEvalIds(options.registry);
|
|
9043
|
+
const reports = createDashboardReportReader(options.reportRoot, options.registry);
|
|
9044
|
+
const result = {
|
|
9045
|
+
archived: [],
|
|
9046
|
+
restored: [],
|
|
9047
|
+
stillArchived: 0,
|
|
9048
|
+
current: 0
|
|
9049
|
+
};
|
|
9050
|
+
for (const run of await reports.listRuns()) {
|
|
9051
|
+
const marker = resolve(options.reportRoot, run.id, ARCHIVE_MARKER);
|
|
9052
|
+
const entry = {
|
|
9053
|
+
runId: run.id,
|
|
9054
|
+
evalId: run.evalId
|
|
9055
|
+
};
|
|
9056
|
+
if (known.has(run.evalId)) {
|
|
9057
|
+
result.current += 1;
|
|
9058
|
+
if (run.archived) {
|
|
9059
|
+
await rm(marker, { force: true });
|
|
9060
|
+
result.restored.push(entry);
|
|
9061
|
+
}
|
|
9062
|
+
} else if (run.archived) result.stillArchived += 1;
|
|
9063
|
+
else {
|
|
9064
|
+
await writeFile(marker, `${JSON.stringify({
|
|
9065
|
+
archivedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
9066
|
+
reason: `Eval "${run.evalId}" is no longer in the project`
|
|
9067
|
+
})}\n`);
|
|
9068
|
+
result.archived.push(entry);
|
|
9069
|
+
}
|
|
9070
|
+
}
|
|
9071
|
+
return result;
|
|
9072
|
+
}
|
|
9073
|
+
//#endregion
|
|
9074
|
+
//#region ../cli/src/share-command.ts
|
|
9075
|
+
/**
|
|
9076
|
+
* Which local runs `evalkit share` uploads: finished runs with a source, at
|
|
9077
|
+
* HEAD (or any commit with `all`), not dirty (unless `allowDirty`), not archived.
|
|
9078
|
+
*/
|
|
9079
|
+
function selectRuns(runs, head, options) {
|
|
9080
|
+
const result = {
|
|
9081
|
+
selected: [],
|
|
9082
|
+
untracked: 0,
|
|
9083
|
+
otherCommits: 0,
|
|
9084
|
+
dirty: 0,
|
|
9085
|
+
archived: 0,
|
|
9086
|
+
running: 0
|
|
9087
|
+
};
|
|
9088
|
+
for (const run of runs) {
|
|
9089
|
+
const { source } = run;
|
|
9090
|
+
if (!source) result.untracked += 1;
|
|
9091
|
+
else if (!options.all && (source.commit !== head.commit || repoKey(source) !== repoKey(head))) result.otherCommits += 1;
|
|
9092
|
+
else if (source.dirty && !options.allowDirty) result.dirty += 1;
|
|
9093
|
+
else if (run.archived) result.archived += 1;
|
|
9094
|
+
else if (run.status === "running") result.running += 1;
|
|
9095
|
+
else result.selected.push({
|
|
9096
|
+
...run,
|
|
9097
|
+
source
|
|
9098
|
+
});
|
|
9099
|
+
}
|
|
9100
|
+
return result;
|
|
9101
|
+
}
|
|
9102
|
+
/** One line for the Runs step: what will be shared and what was skipped. */
|
|
9103
|
+
function selectionSummary(selection, options) {
|
|
9104
|
+
return [
|
|
9105
|
+
`${plural(selection.selected.length, "run")} to share`,
|
|
9106
|
+
selection.otherCommits && !options.all ? `${selection.otherCommits} from other commits` : "",
|
|
9107
|
+
selection.running ? `${selection.running} still running skipped` : "",
|
|
9108
|
+
selection.dirty ? `${selection.dirty} dirty skipped` : "",
|
|
9109
|
+
selection.untracked ? `${selection.untracked} untracked skipped` : "",
|
|
9110
|
+
selection.archived ? `${selection.archived} archived skipped` : ""
|
|
9111
|
+
].filter(Boolean).join(" · ");
|
|
9112
|
+
}
|
|
9113
|
+
/** Why nothing was selected, and what to do about it. */
|
|
9114
|
+
function noRunsMessage(selection, head, options) {
|
|
9115
|
+
const at = shortSource(head);
|
|
9116
|
+
if (selection.dirty) return `No committed runs for ${at}: ${plural(selection.dirty, "run")} came from uncommitted changes. Commit and rerun \`evalkit run\`, or pass --allow-dirty.`;
|
|
9117
|
+
if (selection.running) return `No finished runs for ${at}: ${plural(selection.running, "run")} still running. Share again when they finish.`;
|
|
9118
|
+
return `No runs for ${at}. Run \`evalkit run\` first${options.all ? "" : ", or pass --all to share runs from other commits"}.`;
|
|
9119
|
+
}
|
|
9120
|
+
/** Where to look after sharing: the commit, or the repository if several commits were shared. */
|
|
9121
|
+
function shareTarget(serverUrl, head, runs) {
|
|
9122
|
+
const commits = new Set(runs.map((run) => run.source.commit));
|
|
9123
|
+
const commit = commits.size === 1 ? `/-/${[...commits][0]}` : "";
|
|
9124
|
+
return new URL(`/${repoKey(head)}${commit}`, serverUrl).href;
|
|
9125
|
+
}
|
|
9126
|
+
/** Build one run's upload bundle: run.json, catalog.json, and its snapshot. */
|
|
9127
|
+
async function runBundle(options) {
|
|
9128
|
+
const files = [];
|
|
9129
|
+
const encode = (value) => new TextEncoder().encode(`${JSON.stringify(value)}\n`);
|
|
9130
|
+
files.push({
|
|
9131
|
+
path: "run.json",
|
|
9132
|
+
data: encode(options.run)
|
|
9133
|
+
});
|
|
9134
|
+
const saved = await readFile(resolve(options.reportRoot, options.run.id, "catalog.json")).catch(() => void 0);
|
|
9135
|
+
files.push({
|
|
9136
|
+
path: "catalog.json",
|
|
9137
|
+
data: saved ? new Uint8Array(saved) : encode(runCatalog(options.registry))
|
|
9138
|
+
});
|
|
9139
|
+
await snapshotRun(options.reports, options.run.id, {
|
|
9140
|
+
async json(path, value) {
|
|
9141
|
+
files.push({
|
|
9142
|
+
path,
|
|
9143
|
+
data: encode(value)
|
|
9144
|
+
});
|
|
9145
|
+
},
|
|
9146
|
+
async bytes(path, data) {
|
|
9147
|
+
files.push({
|
|
9148
|
+
path,
|
|
9149
|
+
data
|
|
9150
|
+
});
|
|
9151
|
+
}
|
|
9152
|
+
}, { workspaceFiles: options.workspaceFiles });
|
|
9153
|
+
return encodeBundle(files);
|
|
9154
|
+
}
|
|
9155
|
+
/** The server refused the bundle (4xx): retrying the same bundle cannot help. */
|
|
9156
|
+
var UploadRejected = class extends Error {};
|
|
9157
|
+
/** POST one bundle, retrying network errors and 5xx responses. */
|
|
9158
|
+
async function uploadBundle(serverUrl, bundle, options = {}) {
|
|
9159
|
+
const attempts = options.attempts ?? 3;
|
|
9160
|
+
let lastError;
|
|
9161
|
+
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
9162
|
+
try {
|
|
9163
|
+
return await postBundle(serverUrl, bundle);
|
|
9164
|
+
} catch (error) {
|
|
9165
|
+
if (error instanceof UploadRejected) throw error;
|
|
9166
|
+
lastError = error;
|
|
9167
|
+
}
|
|
9168
|
+
if (attempt < attempts) await Bun.sleep((options.delayMs ?? 500) * 2 ** (attempt - 1));
|
|
9169
|
+
}
|
|
9170
|
+
throw lastError instanceof Error ? lastError : /* @__PURE__ */ new Error(`Upload failed: ${String(lastError)}`);
|
|
9171
|
+
}
|
|
9172
|
+
async function postBundle(serverUrl, bundle) {
|
|
9173
|
+
const response = await fetch(new URL("/api/v1/uploads", serverUrl), {
|
|
9174
|
+
method: "POST",
|
|
9175
|
+
headers: { "content-type": "application/octet-stream" },
|
|
9176
|
+
body: bundle
|
|
9177
|
+
});
|
|
9178
|
+
if (response.ok) return parseUploadResult(await response.json());
|
|
9179
|
+
const reason = await response.text();
|
|
9180
|
+
if (response.status < 500) throw new UploadRejected(`Share server rejected the upload: ${reason}`);
|
|
9181
|
+
throw new Error(`Share server error ${response.status}: ${reason}`);
|
|
9182
|
+
}
|
|
9183
|
+
function parseUploadResult(value) {
|
|
9184
|
+
const result = value;
|
|
9185
|
+
if ((result?.status === "created" || result?.status === "exists") && typeof result.url === "string") return {
|
|
9186
|
+
status: result.status,
|
|
9187
|
+
url: result.url
|
|
9188
|
+
};
|
|
9189
|
+
throw new UploadRejected(`Unexpected share server response: ${JSON.stringify(value)}`);
|
|
9190
|
+
}
|
|
9191
|
+
async function shareRuns(options) {
|
|
9192
|
+
const reports = createDashboardReportReader(options.reportRoot, options.registry);
|
|
9193
|
+
return options.step("Uploading", async (progress) => {
|
|
9194
|
+
const result = {
|
|
9195
|
+
created: 0,
|
|
9196
|
+
exists: 0,
|
|
9197
|
+
failed: [],
|
|
9198
|
+
bytes: 0
|
|
9199
|
+
};
|
|
9200
|
+
for (const [index, run] of options.runs.entries()) {
|
|
9201
|
+
progress(`run ${index + 1}/${options.runs.length} · ${formatBytes(result.bytes)}`);
|
|
9202
|
+
try {
|
|
9203
|
+
const bundle = await runBundle({
|
|
9204
|
+
reports,
|
|
9205
|
+
reportRoot: options.reportRoot,
|
|
9206
|
+
registry: options.registry,
|
|
9207
|
+
run,
|
|
9208
|
+
workspaceFiles: options.workspaceFiles
|
|
9209
|
+
});
|
|
9210
|
+
const uploaded = await uploadBundle(options.serverUrl, bundle);
|
|
9211
|
+
result.bytes += bundle.byteLength;
|
|
9212
|
+
result[uploaded.status] += 1;
|
|
9213
|
+
} catch (error) {
|
|
9214
|
+
result.failed.push({
|
|
9215
|
+
runId: run.id,
|
|
9216
|
+
error: error instanceof Error ? error.message : String(error)
|
|
9217
|
+
});
|
|
9218
|
+
}
|
|
9219
|
+
}
|
|
9220
|
+
return result;
|
|
9221
|
+
}, (result) => `${result.created} new · ${result.exists} already shared` + (result.failed.length ? ` · ${result.failed.length} failed` : "") + ` · ${formatBytes(result.bytes)}`);
|
|
9222
|
+
}
|
|
9223
|
+
function formatBytes(size) {
|
|
9224
|
+
if (size < 1024) return `${size} B`;
|
|
9225
|
+
if (size < 1024 ** 2) return `${(size / 1024).toFixed(1)} kB`;
|
|
9226
|
+
return `${(size / 1024 ** 2).toFixed(1)} MB`;
|
|
8637
9227
|
}
|
|
8638
9228
|
//#endregion
|
|
8639
9229
|
//#region ../cli/src/new-project-template.ts
|
|
@@ -8735,12 +9325,11 @@ Replace these examples with your own agent, tasks, and checks.
|
|
|
8735
9325
|
|
|
8736
9326
|
\`\`\`sh
|
|
8737
9327
|
bun install
|
|
8738
|
-
bun run check
|
|
8739
|
-
bun run
|
|
8740
|
-
bun run matrix
|
|
8741
|
-
bun run
|
|
8742
|
-
bun run
|
|
8743
|
-
bun run dashboard:build # write a read-only, shareable snapshot
|
|
9328
|
+
bun run check # type-check your eval, agent, scorer, and config
|
|
9329
|
+
bun run evalkit run # run the example across both styles
|
|
9330
|
+
bun run evalkit run --matrix styles --dry-run # inspect the matrix without running it
|
|
9331
|
+
bun run evalkit dev # inspect local reports and launch runs
|
|
9332
|
+
bun run evalkit build # write a read-only, shareable snapshot
|
|
8744
9333
|
\`\`\`
|
|
8745
9334
|
|
|
8746
9335
|
EvalKit installs from npm. Reports, trial workspaces, and dashboard snapshots stay local
|
|
@@ -8750,12 +9339,13 @@ scores, evidence, and candidate files (never evaluator workspaces): review it be
|
|
|
8750
9339
|
}
|
|
8751
9340
|
//#endregion
|
|
8752
9341
|
//#region ../cli/src/new-project.ts
|
|
8753
|
-
const npmDependency = "^0.
|
|
9342
|
+
const npmDependency = "^0.4.0";
|
|
8754
9343
|
const ignored = [
|
|
8755
9344
|
"node_modules/",
|
|
8756
9345
|
"_evalkit-results/",
|
|
8757
9346
|
"_evalkit-sandbox/",
|
|
8758
|
-
"_evalkit-dashboard/"
|
|
9347
|
+
"_evalkit-dashboard/",
|
|
9348
|
+
"_evalkit-share/"
|
|
8759
9349
|
];
|
|
8760
9350
|
const readIfExists = async (path) => {
|
|
8761
9351
|
try {
|
|
@@ -8834,11 +9424,6 @@ async function preparePackage(packagePath, name) {
|
|
|
8834
9424
|
const devDependencies = pkg.devDependencies ?? {};
|
|
8835
9425
|
if (!scripts || typeof scripts !== "object" || Array.isArray(scripts) || !dependencies || typeof dependencies !== "object" || Array.isArray(dependencies) || !devDependencies || typeof devDependencies !== "object" || Array.isArray(devDependencies)) throw new Error(`Invalid package.json scripts/dependencies: ${packagePath}`);
|
|
8836
9426
|
scripts.check ??= "tsc --noEmit";
|
|
8837
|
-
scripts.evals ??= "evalkit run greeting --local";
|
|
8838
|
-
scripts.matrix ??= "evalkit run --matrix styles --local";
|
|
8839
|
-
scripts["matrix:plan"] ??= "evalkit run --matrix styles --dry-run";
|
|
8840
|
-
scripts.dashboard ??= "evalkit dev";
|
|
8841
|
-
scripts["dashboard:build"] ??= "evalkit build";
|
|
8842
9427
|
if (!dependencies["@leostera/evalkit"] && !devDependencies["@leostera/evalkit"]) dependencies["@leostera/evalkit"] = npmDependency;
|
|
8843
9428
|
if (!dependencies.typescript && !devDependencies.typescript) devDependencies.typescript = "^6.0.0";
|
|
8844
9429
|
if (!dependencies["@types/bun"] && !devDependencies["@types/bun"]) devDependencies["@types/bun"] = "^1.4.0";
|
|
@@ -8867,6 +9452,7 @@ async function writeStarterReadme(root, name, inPlace) {
|
|
|
8867
9452
|
}
|
|
8868
9453
|
//#endregion
|
|
8869
9454
|
//#region ../cli/src/index.ts
|
|
9455
|
+
const VERSION = "0.4.0";
|
|
8870
9456
|
let projectRoot = process.cwd();
|
|
8871
9457
|
let reportRoot = resolve(projectRoot, "_evalkit-results");
|
|
8872
9458
|
const sandboxRoot = resolve(projectRoot, "_evalkit-sandbox");
|
|
@@ -8886,13 +9472,31 @@ async function runEvals(options) {
|
|
|
8886
9472
|
const registry = await loadRegistry();
|
|
8887
9473
|
const evaluations = options.evalIds.map((id) => registry.get(id));
|
|
8888
9474
|
if (evaluations.some((e) => !e)) throw new Error("Unknown eval");
|
|
8889
|
-
await
|
|
8890
|
-
|
|
8891
|
-
|
|
8892
|
-
|
|
8893
|
-
|
|
8894
|
-
|
|
8895
|
-
|
|
9475
|
+
const provenance = await runProvenance(registry);
|
|
9476
|
+
await Effect.runPromise(runnerLogs(Effect.all(evaluations.map((evaluation) => {
|
|
9477
|
+
const name = evaluation.name ?? authoringId(evaluation);
|
|
9478
|
+
const started = performance.now();
|
|
9479
|
+
return Effect.sync(() => runLog.started(name, {}, options.suiteId)).pipe(Effect.andThen(runEval(evaluation, {
|
|
9480
|
+
report: localReportStore(reportRoot),
|
|
9481
|
+
workspaceRoot: sandboxRoot,
|
|
9482
|
+
suiteId: options.suiteId,
|
|
9483
|
+
runtime: runtimeFor(evaluation),
|
|
9484
|
+
concurrency: options.concurrency,
|
|
9485
|
+
...provenance
|
|
9486
|
+
})), Effect.tap((result) => runLog.finished(name, {}, result, performance.now() - started)), Effect.tapError((error) => Effect.sync(() => runLog.failed(name, error))));
|
|
9487
|
+
}), { concurrency: options.concurrency })));
|
|
9488
|
+
}
|
|
9489
|
+
/** Detected per launch: `evalkit dev` keeps running while commits change. */
|
|
9490
|
+
async function runProvenance(registry) {
|
|
9491
|
+
const source = await detectSource(projectRoot, { outputDirs: [
|
|
9492
|
+
reportRoot,
|
|
9493
|
+
sandboxRoot,
|
|
9494
|
+
resolve(projectRoot, "_evalkit-dashboard")
|
|
9495
|
+
] });
|
|
9496
|
+
return {
|
|
9497
|
+
...source ? { source } : {},
|
|
9498
|
+
catalog: runCatalog(registry)
|
|
9499
|
+
};
|
|
8896
9500
|
}
|
|
8897
9501
|
function registerMatrixRoutes(app, registry, dashboardMatrix, trials) {
|
|
8898
9502
|
let matrixRunning = false;
|
|
@@ -8935,14 +9539,19 @@ function registerMatrixRoutes(app, registry, dashboardMatrix, trials) {
|
|
|
8935
9539
|
}
|
|
8936
9540
|
if (matrixRunning) return context.text("A dashboard matrix cell is already running", 409);
|
|
8937
9541
|
matrixRunning = true;
|
|
8938
|
-
|
|
9542
|
+
const name = evaluation.name ?? authoringId(evaluation);
|
|
9543
|
+
const started = performance.now();
|
|
9544
|
+
runLog.started(name, body.parameters ?? {}, suiteId);
|
|
9545
|
+
runProvenance(registry).then((provenance) => Effect.runPromise(runnerLogs(runMatrix(dashboardMatrix, {
|
|
8939
9546
|
selection,
|
|
8940
9547
|
concurrency: 1,
|
|
8941
9548
|
trials,
|
|
8942
9549
|
...suiteId ? { suiteId } : {},
|
|
8943
9550
|
report: localReportStore(reportRoot),
|
|
8944
|
-
workspaceRoot: sandboxRoot
|
|
8945
|
-
|
|
9551
|
+
workspaceRoot: sandboxRoot,
|
|
9552
|
+
...provenance,
|
|
9553
|
+
onResult: (cell, result) => runLog.cell(cell, result, performance.now() - started)
|
|
9554
|
+
})))).catch((error) => runLog.failed(name, error)).finally(() => {
|
|
8946
9555
|
matrixRunning = false;
|
|
8947
9556
|
});
|
|
8948
9557
|
} else {
|
|
@@ -8951,7 +9560,7 @@ function registerMatrixRoutes(app, registry, dashboardMatrix, trials) {
|
|
|
8951
9560
|
evalIds: [authoringId(evaluation)],
|
|
8952
9561
|
suiteId,
|
|
8953
9562
|
concurrency: 32
|
|
8954
|
-
}).catch((
|
|
9563
|
+
}).catch(() => {});
|
|
8955
9564
|
}
|
|
8956
9565
|
return context.json({ accepted: true }, 202);
|
|
8957
9566
|
});
|
|
@@ -8963,7 +9572,7 @@ function registerMatrixRoutes(app, registry, dashboardMatrix, trials) {
|
|
|
8963
9572
|
evalIds: suite.evals.map(authoringId),
|
|
8964
9573
|
suiteId: authoringId(suite),
|
|
8965
9574
|
concurrency: 32
|
|
8966
|
-
}).catch((
|
|
9575
|
+
}).catch(() => {});
|
|
8967
9576
|
return context.json({ accepted: true }, 202);
|
|
8968
9577
|
});
|
|
8969
9578
|
}
|
|
@@ -9047,14 +9656,32 @@ async function serveDashboard() {
|
|
|
9047
9656
|
const registry = await loadRegistry();
|
|
9048
9657
|
const dashboardRoot = dashboardAssets();
|
|
9049
9658
|
const app = new Hono();
|
|
9050
|
-
|
|
9659
|
+
const dashboardMatrix = project?.config.matrix ? registry.matrices.at(-1) : void 0;
|
|
9660
|
+
registerMatrixRoutes(app, registry, dashboardMatrix, project?.config.execution?.trials);
|
|
9051
9661
|
registerReportRoutes(app, registry, createDashboardReportReader(reportRoot, registry));
|
|
9052
9662
|
registerDashboardAssets(app, dashboardRoot);
|
|
9053
|
-
const
|
|
9054
|
-
|
|
9055
|
-
|
|
9056
|
-
|
|
9057
|
-
|
|
9663
|
+
const port = Number(process.env.PORT ?? 4317);
|
|
9664
|
+
let server;
|
|
9665
|
+
try {
|
|
9666
|
+
server = Bun.serve({
|
|
9667
|
+
port,
|
|
9668
|
+
fetch: app.fetch
|
|
9669
|
+
});
|
|
9670
|
+
} catch (error) {
|
|
9671
|
+
if (error.code === "EADDRINUSE") throw new Error(`Port ${port} is already in use. Stop the other server or choose another port: PORT=4318 evalkit dev`);
|
|
9672
|
+
throw error;
|
|
9673
|
+
}
|
|
9674
|
+
const url = `http://localhost:${server.port}`;
|
|
9675
|
+
const reader = createDashboardReportReader(reportRoot, registry);
|
|
9676
|
+
const runs = (await reader.listRuns()).length;
|
|
9677
|
+
const legacy = reader.legacyRunIds().length;
|
|
9678
|
+
const matrixCells = dashboardMatrix?.count({ parameters: {} });
|
|
9679
|
+
const row = (label, value) => console.log(` ${cyan("➜")} ${bold(label.padEnd(9))}${value}`);
|
|
9680
|
+
banner("dev", VERSION);
|
|
9681
|
+
row("Local:", cyan(url));
|
|
9682
|
+
row("Project:", `${plural(registry.evals.length, "eval")} · ${plural(registry.suites.length, "suite")}` + (dashboardMatrix ? ` · matrix ${authoringId(dashboardMatrix)} (${plural(matrixCells, "cell")})` : ""));
|
|
9683
|
+
row("Reports:", `${relative(process.cwd(), reportRoot) || "."} ${dim(`(${plural(runs, "run")}${legacy ? `, ${legacy} from an older EvalKit hidden` : ""})`)}`);
|
|
9684
|
+
console.log(`\n ${dim("Runs started from the dashboard are logged below. Press Ctrl+C to stop.")}\n`);
|
|
9058
9685
|
if (process.env.EVALKIT_NO_OPEN !== "1") {
|
|
9059
9686
|
const opener = process.platform === "darwin" ? "open" : process.platform === "win32" ? "start" : "xdg-open";
|
|
9060
9687
|
try {
|
|
@@ -9119,7 +9746,7 @@ const runCommand = make$1("run", {
|
|
|
9119
9746
|
const newCommand = make$1("new", { directory: text$2({ name: "directory" }) }, ({ directory }) => Effect.tryPromise({
|
|
9120
9747
|
try: async () => {
|
|
9121
9748
|
const root = await newProject([directory]);
|
|
9122
|
-
console.log(`Created EvalKit project at ${root}\nRun cd ${root} && bun install && bun run
|
|
9749
|
+
console.log(`Created EvalKit project at ${root}\nRun cd ${root} && bun install && bun run evalkit run.`);
|
|
9123
9750
|
},
|
|
9124
9751
|
catch: (error) => error
|
|
9125
9752
|
}));
|
|
@@ -9132,29 +9759,162 @@ const devCommand = make$1("dev", { config }, ({ config }) => Effect.tryPromise({
|
|
|
9132
9759
|
}));
|
|
9133
9760
|
const buildCommand = make$1("build", {
|
|
9134
9761
|
config,
|
|
9135
|
-
out: text("out").pipe(optional)
|
|
9136
|
-
|
|
9762
|
+
out: text("out").pipe(optional),
|
|
9763
|
+
run: boolean("run"),
|
|
9764
|
+
noRun: boolean("no-run")
|
|
9765
|
+
}, ({ config, out, run, noRun }) => Effect.tryPromise({
|
|
9137
9766
|
try: async () => {
|
|
9138
|
-
|
|
9767
|
+
if (run && noRun) throw new Error("Choose either --run or --no-run");
|
|
9768
|
+
const started = performance.now();
|
|
9769
|
+
const configPath = Option.getOrUndefined(config);
|
|
9770
|
+
banner("build", VERSION);
|
|
9771
|
+
await loadProjectStep(configPath);
|
|
9772
|
+
if (await shouldRunEvals({
|
|
9773
|
+
run,
|
|
9774
|
+
noRun
|
|
9775
|
+
})) await runAllEvals(configPath);
|
|
9139
9776
|
const result = await buildDashboard({
|
|
9140
9777
|
registry: project.registry,
|
|
9141
9778
|
reportRoot,
|
|
9142
9779
|
dashboardRoot: dashboardAssets(),
|
|
9143
9780
|
outDir: resolve(projectRoot, Option.getOrElse(out, () => "_evalkit-dashboard"))
|
|
9781
|
+
}, step);
|
|
9782
|
+
const size = await directorySize(result.outDir);
|
|
9783
|
+
const shown = relative(process.cwd(), result.outDir) || ".";
|
|
9784
|
+
console.log(`\n ${bold(`${shown}/`)} ${dim(`${bytes(size.bytes)} · ${plural(size.files, "file")}`)}`);
|
|
9785
|
+
console.log(`\n ${bold("Built read-only dashboard")} in ${duration(performance.now() - started)}\n`);
|
|
9786
|
+
console.log(` ${cyan("➜")} ${bold("Preview:")} bunx serve ${shown}`);
|
|
9787
|
+
console.log(` ${yellow("⚠")} Contains transcripts, scores, evidence, and candidate files\n (never evaluator workspaces). Review before sharing.\n`);
|
|
9788
|
+
},
|
|
9789
|
+
catch: (error) => error
|
|
9790
|
+
}));
|
|
9791
|
+
async function loadProjectStep(configPath) {
|
|
9792
|
+
await step("Eval project", () => useProject(configPath), () => {
|
|
9793
|
+
const where = relative(process.cwd(), projectRoot);
|
|
9794
|
+
return `${plural(project.registry.evals.length, "eval")} loaded${where ? ` from ${where}` : ""}`;
|
|
9795
|
+
});
|
|
9796
|
+
}
|
|
9797
|
+
/**
|
|
9798
|
+
* A snapshot needs data: with no current results, run the evals. Otherwise ask,
|
|
9799
|
+
* or keep the existing results when no one can answer (CI, pipes).
|
|
9800
|
+
*/
|
|
9801
|
+
async function shouldRunEvals(flags) {
|
|
9802
|
+
if (flags.run) return true;
|
|
9803
|
+
if (flags.noRun) return false;
|
|
9804
|
+
const runs = (await createDashboardReportReader(reportRoot, project.registry).listRuns()).filter((entry) => !entry.archived).length;
|
|
9805
|
+
if (!runs) {
|
|
9806
|
+
console.log(` ${cyan("◇")} No results yet, so running the evals to have data to build.`);
|
|
9807
|
+
return true;
|
|
9808
|
+
}
|
|
9809
|
+
if (!canPrompt()) {
|
|
9810
|
+
console.log(` ${dim(`Using ${plural(runs, "existing run")}; pass --run to run the evals first.`)}`);
|
|
9811
|
+
return false;
|
|
9812
|
+
}
|
|
9813
|
+
return confirm(`Run the evals before building? ${dim(`(${plural(runs, "existing run")})`)}`);
|
|
9814
|
+
}
|
|
9815
|
+
async function runAllEvals(configPath) {
|
|
9816
|
+
const started = performance.now();
|
|
9817
|
+
console.log(`\n ${cyan("▶")} ${bold("Running evals")}\n`);
|
|
9818
|
+
await runProjectCommand([], process.cwd(), parseRunArgs(configPath ? ["--config", configPath] : []));
|
|
9819
|
+
const failed = process.exitCode === 1;
|
|
9820
|
+
console.log(`\n ${failed ? yellow("⚠") : cyan("✓")} ${bold("Evals finished")} in ${duration(performance.now() - started)}` + (failed ? dim(" (some failed; building with their results)") : "") + "\n");
|
|
9821
|
+
}
|
|
9822
|
+
const tidyCommand = make$1("tidy", { config }, ({ config }) => Effect.tryPromise({
|
|
9823
|
+
try: async () => {
|
|
9824
|
+
banner("tidy", VERSION);
|
|
9825
|
+
await loadProjectStep(Option.getOrUndefined(config));
|
|
9826
|
+
const result = await step("Run reports", () => tidyReports({
|
|
9827
|
+
registry: project.registry,
|
|
9828
|
+
reportRoot
|
|
9829
|
+
}), (value) => `${plural(value.current + value.archived.length + value.stillArchived, "run")} checked`);
|
|
9830
|
+
const byEval = (entries) => [...entries.reduce((groups, entry) => groups.set(entry.evalId, (groups.get(entry.evalId) ?? 0) + 1), /* @__PURE__ */ new Map()).entries()].map(([id, total]) => ` ${bold(id)} ${dim(`(${plural(total, "run")})`)}`).join("\n");
|
|
9831
|
+
console.log("");
|
|
9832
|
+
if (result.archived.length) console.log(` ${yellow("◆")} Archived ${plural(result.archived.length, "run")} of evals no longer in the project:\n${byEval(result.archived)}`);
|
|
9833
|
+
if (result.restored.length) console.log(` ${cyan("◆")} Restored ${plural(result.restored.length, "run")} whose eval is back:\n${byEval(result.restored)}`);
|
|
9834
|
+
if (!result.archived.length && !result.restored.length) console.log(` ${cyan("✓")} Nothing new to tidy.`);
|
|
9835
|
+
const totalArchived = result.archived.length + result.stillArchived;
|
|
9836
|
+
if (totalArchived) console.log(`\n ${dim(`Archived runs (${totalArchived}) are hidden in \`evalkit dev\` (filter "Show archived")`)}\n ${dim("and left out of `evalkit build`. Nothing was deleted.")}`);
|
|
9837
|
+
console.log("");
|
|
9838
|
+
},
|
|
9839
|
+
catch: (error) => error
|
|
9840
|
+
}));
|
|
9841
|
+
const shareServeCommand = make$1("serve", {
|
|
9842
|
+
dir: text("dir").pipe(withDefault("_evalkit-share")),
|
|
9843
|
+
port: integer("port").pipe(withDefault(4318))
|
|
9844
|
+
}, ({ dir, port }) => Effect.tryPromise({
|
|
9845
|
+
try: async () => {
|
|
9846
|
+
const root = resolve(process.cwd(), dir);
|
|
9847
|
+
const app = await createDirectoryShareApp({
|
|
9848
|
+
dir: root,
|
|
9849
|
+
dashboardRoot: dashboardAssets()
|
|
9144
9850
|
});
|
|
9145
|
-
|
|
9146
|
-
|
|
9851
|
+
let server;
|
|
9852
|
+
try {
|
|
9853
|
+
server = Bun.serve({
|
|
9854
|
+
port,
|
|
9855
|
+
fetch: app.fetch
|
|
9856
|
+
});
|
|
9857
|
+
} catch (error) {
|
|
9858
|
+
if (error.code === "EADDRINUSE") throw new Error(`Port ${port} is already in use. Choose another: evalkit share serve --port 4319`);
|
|
9859
|
+
throw error;
|
|
9860
|
+
}
|
|
9861
|
+
banner("share serve", VERSION);
|
|
9862
|
+
console.log(` ${cyan("➜")} ${bold("Local:".padEnd(9))}${cyan(`http://localhost:${server.port}`)}`);
|
|
9863
|
+
console.log(` ${cyan("➜")} ${bold("Storage:".padEnd(9))}${relative(process.cwd(), root) || "."}`);
|
|
9864
|
+
console.log(`\n ${dim(`Upload with share.url = "http://localhost:${server.port}". No authentication: keep it private.`)}\n`);
|
|
9147
9865
|
},
|
|
9148
9866
|
catch: (error) => error
|
|
9149
9867
|
}));
|
|
9868
|
+
const shareCommand = make$1("share", {
|
|
9869
|
+
config,
|
|
9870
|
+
all: boolean("all"),
|
|
9871
|
+
allowDirty: boolean("allow-dirty"),
|
|
9872
|
+
noWorkspace: boolean("no-workspace")
|
|
9873
|
+
}, ({ config, all, allowDirty, noWorkspace }) => Effect.tryPromise({
|
|
9874
|
+
try: async () => {
|
|
9875
|
+
const started = performance.now();
|
|
9876
|
+
banner("share", VERSION);
|
|
9877
|
+
await loadProjectStep(Option.getOrUndefined(config));
|
|
9878
|
+
const serverUrl = process.env.EVALKIT_SHARE_URL ?? project.config.share?.url;
|
|
9879
|
+
if (!serverUrl) throw new Error("No share server configured. Set share: { url } in evalkit.config.ts or EVALKIT_SHARE_URL.");
|
|
9880
|
+
const head = await step("Source", () => detectSource(projectRoot, { outputDirs: [
|
|
9881
|
+
reportRoot,
|
|
9882
|
+
sandboxRoot,
|
|
9883
|
+
resolve(projectRoot, "_evalkit-dashboard")
|
|
9884
|
+
] }), (source) => source ? `${repoKey(source)} @ ${shortSource(source)}${source.ref ? ` (${source.ref})` : ""}` : "not a git repository");
|
|
9885
|
+
if (!head) throw new Error("evalkit share needs to know the commit: run it inside a git repository or set EVALKIT_SOURCE_REPO and EVALKIT_SOURCE_COMMIT. `evalkit build` works without one.");
|
|
9886
|
+
const registry = project.registry;
|
|
9887
|
+
const selection = await step("Runs", async () => selectRuns(await createDashboardReportReader(reportRoot, registry).listRuns(), head, {
|
|
9888
|
+
all,
|
|
9889
|
+
allowDirty
|
|
9890
|
+
}), (value) => selectionSummary(value, { all }));
|
|
9891
|
+
if (!selection.selected.length) throw new Error(noRunsMessage(selection, head, { all }));
|
|
9892
|
+
const result = await shareRuns({
|
|
9893
|
+
serverUrl,
|
|
9894
|
+
reportRoot,
|
|
9895
|
+
registry,
|
|
9896
|
+
runs: selection.selected,
|
|
9897
|
+
workspaceFiles: !noWorkspace,
|
|
9898
|
+
step
|
|
9899
|
+
});
|
|
9900
|
+
for (const failure of result.failed) console.log(` ${yellow("✗")} ${failure.runId.slice(0, 8)} ${dim(failure.error)}`);
|
|
9901
|
+
const target = shareTarget(serverUrl, head, selection.selected);
|
|
9902
|
+
console.log(`\n ${bold(result.failed.length ? "Shared with errors" : "Shared")} in ${duration(performance.now() - started)}\n`);
|
|
9903
|
+
console.log(` ${cyan("➜")} ${cyan(target)}\n`);
|
|
9904
|
+
if (result.failed.length) process.exitCode = 1;
|
|
9905
|
+
},
|
|
9906
|
+
catch: (error) => error
|
|
9907
|
+
})).pipe(withSubcommands([shareServeCommand]));
|
|
9150
9908
|
const cli = run(make$1("evalkit").pipe(withSubcommands([
|
|
9151
9909
|
newCommand,
|
|
9152
9910
|
runCommand,
|
|
9153
9911
|
devCommand,
|
|
9154
|
-
buildCommand
|
|
9912
|
+
buildCommand,
|
|
9913
|
+
tidyCommand,
|
|
9914
|
+
shareCommand
|
|
9155
9915
|
])), {
|
|
9156
9916
|
name: "EvalKit",
|
|
9157
|
-
version:
|
|
9917
|
+
version: VERSION
|
|
9158
9918
|
});
|
|
9159
9919
|
const argv = process.argv.slice();
|
|
9160
9920
|
if (argv[2] === "help") argv.splice(2, 1, "--help");
|