@xsft/panels 0.8.1 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +58 -13
- package/bin/panels.js +8 -4
- package/lib/args.js +18 -4
- package/lib/auth.js +74 -0
- package/lib/judge.js +14 -4
- package/lib/panel.js +10 -9
- package/lib/report.js +12 -8
- package/lib/table.js +304 -386
- package/package.json +1 -2
- package/prompts/judge-system-prompt.txt +11 -13
- package/schema/report.json +7 -2
package/lib/table.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
// The terminal mode of `panels run`: the overview
|
|
2
|
-
//
|
|
1
|
+
// The terminal mode of `panels run`: one screen, the overview. The criteria at the left, and at the right the
|
|
2
|
+
// detail of the cursor criterion: its verdict, its judge's live terminal, or its text. Each judge runs its
|
|
3
|
+
// interactive harness in a pseudo-terminal, and a headless terminal keeps its screen; the Agent pane draws
|
|
4
|
+
// that screen from the buffer, so no mode of a judge reaches the real terminal.
|
|
3
5
|
import { existsSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs';
|
|
4
6
|
import { join } from 'node:path';
|
|
5
7
|
import pty from '@homebridge/node-pty-prebuilt-multiarch';
|
|
6
|
-
import { SerializeAddon } from '@xterm/addon-serialize';
|
|
7
8
|
import xterm from '@xterm/headless';
|
|
8
9
|
import { check, MINUTES_PER_CRITERION, parseAnswer, writeJudge } from './judge.js';
|
|
9
10
|
import { buildReport, VERSION } from './report.js';
|
|
@@ -17,29 +18,33 @@ const ORANGE = '\x1b[38;2;255;106;61m';
|
|
|
17
18
|
const GREY = '\x1b[90m';
|
|
18
19
|
const BOLD = '\x1b[1m';
|
|
19
20
|
const RESET = '\x1b[0m';
|
|
20
|
-
// The colour of a score, from the 16-colour palette: 4 green, 3
|
|
21
|
-
const SCORE_COLOUR = { 4: '\x1b[
|
|
21
|
+
// The colour of a score, from the 16-colour palette: 4 bright green, 3 green, 2 yellow, 1 red.
|
|
22
|
+
const SCORE_COLOUR = { 4: '\x1b[92m', 3: '\x1b[32m', 2: '\x1b[33m', 1: '\x1b[31m' };
|
|
23
|
+
// Inline code in the markdown of a verdict or a body: ink on a dark grey ground.
|
|
24
|
+
const CODE = '\x1b[48;5;236m';
|
|
25
|
+
const CODE_END = '\x1b[49m';
|
|
26
|
+
const BOLD_END = '\x1b[22m';
|
|
22
27
|
const LEVELS = { 1: 'Fails', 2: 'Falls short', 3: 'Almost met', 4: 'Met' };
|
|
23
|
-
// ctrl-q
|
|
28
|
+
// ctrl-q brings the focus back from the Agent pane. A terminal in the kitty keyboard mode sends it as CSI 113;5u.
|
|
24
29
|
const TOGGLE = ['\x11', '\x1b[113;5u'];
|
|
25
30
|
const RIGHT = ['\x1b[C', '\x1bOC'];
|
|
26
31
|
const LEFT = ['\x1b[D', '\x1bOD'];
|
|
27
32
|
const UP = ['\x1b[A', '\x1bOA'];
|
|
28
33
|
const DOWN = ['\x1b[B', '\x1bOB'];
|
|
29
|
-
// Enter, Esc, or ctrl-q give the focus back to the judge of the active tab.
|
|
30
|
-
const BACK = ['\r', '\x1b', ...TOGGLE];
|
|
31
|
-
// Turns off each mode that the screen of a judge can turn on, so the overview gets the keys and the screen as before.
|
|
32
|
-
const TABLE_MODES = '\x1b[?1049h\x1b[r\x1b[0m\x1b[4l\x1b[?6l\x1b[?7h\x1b[?1l\x1b>\x1b[?1000l\x1b[?1002l\x1b[?1003l\x1b[?1006l\x1b[?1004l\x1b[?2004l\x1b[<u\x1b[?25l\x1b[2J';
|
|
33
34
|
// The glimmer on the text of the running judges: grey text, and one soft wave three columns wide that moves one
|
|
34
35
|
// column per frame from the left edge of the screen to the right edge and starts over. A running label lights
|
|
35
36
|
// up only where the wave passes over its own columns, so every running label lights at the same columns.
|
|
36
37
|
const GLIMMER = { base: '\x1b[38;5;244m', mid: '\x1b[38;5;247m', high: '\x1b[38;5;250m' };
|
|
37
38
|
const GLIMMER_ORANGE = { base: ORANGE, mid: '\x1b[38;2;255;140;100m', high: '\x1b[38;2;255;175;145m' };
|
|
38
39
|
const WAVE = 3;
|
|
39
|
-
// The tab strip takes the last three lines of the terminal: a hairline, the tabs, then the status and the keys.
|
|
40
|
-
const STRIP_ROWS = 3;
|
|
41
40
|
// The overview shows two columns from this width up, and the list above the detail below it.
|
|
42
41
|
const TWO_COLUMNS = 100;
|
|
42
|
+
// The sub tabs of the detail pane, in the order the arrows walk them.
|
|
43
|
+
export const SUBS = ['verdict', 'agent', 'criterion'];
|
|
44
|
+
// The header line and the hairline under it; the panes take the rest of the screen.
|
|
45
|
+
const TOP_ROWS = 2;
|
|
46
|
+
// The detail pane holds the sub tab bar and its hairline, then the content.
|
|
47
|
+
const PANE_FRAME = 2;
|
|
43
48
|
|
|
44
49
|
/**
|
|
45
50
|
* text with the glimmer at frame, where the text starts at the screen column start (0-based) on a screen width
|
|
@@ -77,7 +82,6 @@ export function tokensOnScreen(text) {
|
|
|
77
82
|
|
|
78
83
|
const plainLength = (s) => s.replace(/\x1b\[[0-9;]*m/g, '').length;
|
|
79
84
|
const padTo = (s, width) => s + ' '.repeat(Math.max(0, width - plainLength(s)));
|
|
80
|
-
const centre = (s, width) => ' '.repeat(Math.max(0, Math.floor((width - plainLength(s)) / 2))) + s;
|
|
81
85
|
|
|
82
86
|
/** The first width visible characters of a styled line, with every escape sequence kept, so the line never wraps. */
|
|
83
87
|
export function clip(line, width) {
|
|
@@ -86,103 +90,40 @@ export function clip(line, width) {
|
|
|
86
90
|
for (const part of line.split(/(\x1b\[[0-9;]*m)/)) {
|
|
87
91
|
if (part.startsWith('\x1b')) { out += part; continue; }
|
|
88
92
|
const room = Math.max(0, width - seen);
|
|
89
|
-
|
|
90
|
-
|
|
93
|
+
const characters = [...part];
|
|
94
|
+
out += characters.slice(0, room).join('');
|
|
95
|
+
seen += Math.min(characters.length, room);
|
|
91
96
|
}
|
|
92
97
|
return out;
|
|
93
98
|
}
|
|
94
99
|
|
|
95
|
-
/** The words of text in lines of at most width characters. */
|
|
100
|
+
/** The words of text in lines of at most width visible characters. A word can carry escape sequences. */
|
|
96
101
|
export function wrap(text, width) {
|
|
97
102
|
const lines = [];
|
|
98
103
|
let line = '';
|
|
99
104
|
for (const word of text.split(/\s+/).filter(Boolean)) {
|
|
100
|
-
if (line && line
|
|
105
|
+
if (line && plainLength(line) + 1 + plainLength(word) > width) { lines.push(line); line = word; }
|
|
101
106
|
else line = line ? `${line} ${word}` : word;
|
|
102
107
|
}
|
|
103
108
|
if (line) lines.push(line);
|
|
104
109
|
return lines;
|
|
105
110
|
}
|
|
106
111
|
|
|
107
|
-
/** The scores added as quarters: 1/4 + 1/4 + 2/4 is "1". No trailing zeros. */
|
|
108
|
-
const quarters = (sum) => String(Math.round((sum / 4) * 100) / 100);
|
|
109
|
-
|
|
110
|
-
// The art of a running judge: a halftone piece inside a square frame, generated from a seed. Two sizes, rows by columns.
|
|
111
|
-
const MARK_SIZES = [[11, 22], [7, 14]];
|
|
112
|
-
const HALFTONE = [' ', '░', '▒', '▓'];
|
|
113
|
-
const SHAPES = ['half-left', 'half-right', 'half-top', 'half-bottom', 'disc', 'ring', 'quarter-tl', 'quarter-tr', 'quarter-bl', 'quarter-br', 'band', 'twin'];
|
|
114
|
-
|
|
115
|
-
/** A small seeded random generator (mulberry32): the same seed gives the same sequence. */
|
|
116
|
-
export function mulberry32(seed) {
|
|
117
|
-
let a = seed | 0;
|
|
118
|
-
return () => {
|
|
119
|
-
a = (a + 0x6D2B79F5) | 0;
|
|
120
|
-
let t = Math.imul(a ^ (a >>> 15), 1 | a);
|
|
121
|
-
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
|
|
122
|
-
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
|
123
|
-
};
|
|
124
|
-
}
|
|
125
|
-
|
|
126
112
|
/**
|
|
127
|
-
*
|
|
128
|
-
*
|
|
129
|
-
*
|
|
130
|
-
*
|
|
131
|
-
* cells by screen column, where start is the screen column of the frame. size is [rows, columns].
|
|
113
|
+
* Markdown as styled lines for a pane: each paragraph wraps on its own, one blank line stays where the text has
|
|
114
|
+
* one or more, a heading becomes a grey upper-case label, a list item keeps its dash with a hanging indent, a
|
|
115
|
+
* fenced block is indented two spaces as it is, a table line stays as it is, inline code is ink on a dark grey
|
|
116
|
+
* ground, bold is bold, and link marks go.
|
|
132
117
|
*/
|
|
133
|
-
export function
|
|
134
|
-
const
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
const dx = (rng() - 0.5) * 0.3;
|
|
138
|
-
const dy = (rng() - 0.5) * 0.3;
|
|
139
|
-
const phase = rng() * Math.PI * 2;
|
|
140
|
-
const period = 14 + rng() * 12;
|
|
141
|
-
const grain = 0.4 + rng() * 0.6;
|
|
142
|
-
const [rows, cols] = size;
|
|
143
|
-
const inner = { rows: rows - 2, cols: cols - 2 };
|
|
144
|
-
const w = inner.cols / 2;
|
|
145
|
-
const h = inner.rows;
|
|
146
|
-
const r = (scale * h) / 2;
|
|
147
|
-
const centres = {
|
|
148
|
-
'half-left': [0, h / 2], 'half-right': [w, h / 2], 'half-top': [w / 2, 0], 'half-bottom': [w / 2, h],
|
|
149
|
-
'quarter-tl': [0, 0], 'quarter-tr': [w, 0], 'quarter-bl': [0, h], 'quarter-br': [w, h],
|
|
150
|
-
};
|
|
151
|
-
const [cx, cy] = centres[shape] ?? [w / 2 + dx * w, h / 2 + dy * h];
|
|
152
|
-
// The distance of a cell from the shape, 0 at the heart and 1 at the edge; past 1 the cell is empty.
|
|
153
|
-
const distance = (u, v) => {
|
|
154
|
-
const d = Math.hypot(u - cx, v - cy);
|
|
155
|
-
if (shape === 'ring') { const t = Math.abs(d - r * 0.75) / (r * 0.3); return t; }
|
|
156
|
-
if (shape === 'band') return Math.abs((u - cx) - (v - cy)) / (r * 0.6);
|
|
157
|
-
if (shape === 'twin') return Math.min(Math.hypot(u - (cx - r * 0.5), v - cy), Math.hypot(u - (cx + r * 0.5), v - cy)) / (r * 0.75);
|
|
158
|
-
if (shape.startsWith('quarter')) return d / (r * 1.5);
|
|
159
|
-
return d / r;
|
|
160
|
-
};
|
|
161
|
-
const pulse = Math.sin((2 * Math.PI * frame) / period + phase);
|
|
162
|
-
const lines = [`${GREY}┌${'─'.repeat(inner.cols)}┐${RESET}`];
|
|
163
|
-
for (let y = 0; y < inner.rows; y++) {
|
|
164
|
-
let cells = '';
|
|
165
|
-
for (let x = 0; x < inner.cols; x++) {
|
|
166
|
-
const d = distance((x + 0.5) / 2, y + 0.5);
|
|
167
|
-
const noise = mulberry32(seed * 7919 + y * 1021 + x * 31)() - 0.5;
|
|
168
|
-
cells += d > 1 ? ' ' : HALFTONE[Math.max(1, Math.min(3, Math.round(1.5 + (1 - d) * 1.5 + pulse + noise * grain)))];
|
|
118
|
+
export function renderMarkdown(markdown, width) {
|
|
119
|
+
const inline = (text, heading = false) => text.split(/(`[^`]+`)/).map((part) => {
|
|
120
|
+
if (part.startsWith('`') && part.endsWith('`')) {
|
|
121
|
+
return part.slice(1, -1).split(/(\s+)/).map((word) => /^\s*$/.test(word) ? word : `${CODE}${word}${CODE_END}`).join('');
|
|
169
122
|
}
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
}
|
|
175
|
-
|
|
176
|
-
/** The size of the art that fits a pane of columnWidth by lines, or null when only the one-line form fits. */
|
|
177
|
-
export const markSize = (columnWidth, lines) => MARK_SIZES.find(([r, c]) => r + 2 <= lines && c <= columnWidth) ?? null;
|
|
178
|
-
|
|
179
|
-
/**
|
|
180
|
-
* The body of a criterion as plain lines for the text pane: a heading becomes a grey upper-case label, a list
|
|
181
|
-
* item keeps its dash, a fenced block is indented four spaces as it is, a table line stays as it is, and the
|
|
182
|
-
* other text is wrapped to width with the emphasis, code, and link marks removed.
|
|
183
|
-
*/
|
|
184
|
-
export function plainMarkdown(markdown, width) {
|
|
185
|
-
const inline = (t) => t.replace(/\*\*(.+?)\*\*/g, '$1').replace(/__(.+?)__/g, '$1').replace(/`([^`]+)`/g, '$1').replace(/\*(\S[^*]*?)\*/g, '$1').replace(/\[([^\]]+)\]\([^)]*\)/g, '$1');
|
|
123
|
+
const words = heading ? part.toUpperCase() : part;
|
|
124
|
+
return words.replace(/\*\*(.+?)\*\*/g, `${BOLD}$1${BOLD_END}`).replace(/__(.+?)__/g, `${BOLD}$1${BOLD_END}`)
|
|
125
|
+
.replace(/(^|\s)\*(\S[^*]*?)\*(?=\s|$|[.,;:])/g, '$1$2').replace(/\[([^\]]+)\]\(([^)]*)\)/g, '$1 ($2)');
|
|
126
|
+
}).join('');
|
|
186
127
|
const lines = [];
|
|
187
128
|
let paragraph = [];
|
|
188
129
|
let fence = false;
|
|
@@ -190,15 +131,21 @@ export function plainMarkdown(markdown, width) {
|
|
|
190
131
|
if (paragraph.length) lines.push(...wrap(inline(paragraph.join(' ')), width));
|
|
191
132
|
paragraph = [];
|
|
192
133
|
};
|
|
193
|
-
for (const raw of markdown.split('\n')) {
|
|
134
|
+
for (const raw of (markdown ?? '').split('\n')) {
|
|
194
135
|
const line = raw.trimEnd();
|
|
195
136
|
if (/^\s*```/.test(line)) { flush(); fence = !fence; continue; }
|
|
196
|
-
if (fence) { lines.push(`
|
|
197
|
-
if (!line.trim()) { flush(); if (lines.at(-1) !== '') lines.push(''); continue; }
|
|
137
|
+
if (fence) { lines.push(` ${line}`); continue; }
|
|
138
|
+
if (!line.trim()) { flush(); if (lines.length && lines.at(-1) !== '') lines.push(''); continue; }
|
|
198
139
|
const heading = /^#{1,6}\s+(.*)$/.exec(line);
|
|
199
|
-
if (heading) { flush(); if (lines.length && lines.at(-1) !== '') lines.push(''); lines.push(`${GREY}${inline(heading[1])
|
|
200
|
-
const item = /^\s*(
|
|
201
|
-
if (item) {
|
|
140
|
+
if (heading) { flush(); if (lines.length && lines.at(-1) !== '') lines.push(''); lines.push(`${GREY}${inline(heading[1], true)}${RESET}`); continue; }
|
|
141
|
+
const item = /^\s*([-*+]|\d+\.)\s+(.*)$/.exec(line);
|
|
142
|
+
if (item) {
|
|
143
|
+
flush();
|
|
144
|
+
const dash = item[1].length === 1 ? '-' : item[1];
|
|
145
|
+
const [first, ...rest] = wrap(inline(item[2]), width - dash.length - 1);
|
|
146
|
+
lines.push(`${dash} ${first ?? ''}`, ...rest.map((l) => `${' '.repeat(dash.length + 1)}${l}`));
|
|
147
|
+
continue;
|
|
148
|
+
}
|
|
202
149
|
if (/^\s*\|/.test(line)) { flush(); lines.push(inline(line)); continue; }
|
|
203
150
|
paragraph.push(line.trim());
|
|
204
151
|
}
|
|
@@ -208,96 +155,111 @@ export function plainMarkdown(markdown, width) {
|
|
|
208
155
|
return lines;
|
|
209
156
|
}
|
|
210
157
|
|
|
211
|
-
/** The
|
|
212
|
-
|
|
158
|
+
/** The scores added as quarters: 1/4 + 1/4 + 2/4 is "1". No trailing zeros. */
|
|
159
|
+
const quarters = (sum) => String(Math.round((sum / 4) * 100) / 100);
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* The lines of the screen of a judge, from the active buffer of its headless terminal: each row's cells with
|
|
163
|
+
* their attributes (bold, dim, italic, underline, inverse, strikethrough, and the colours in the default,
|
|
164
|
+
* palette, or rgb modes), runs with equal attributes merged, padded or clipped to width. A wide character
|
|
165
|
+
* takes its two cells.
|
|
166
|
+
*/
|
|
167
|
+
export function screenLines(term, width, rows) {
|
|
168
|
+
const buffer = term.buffer.active;
|
|
169
|
+
const sgr = (c) => {
|
|
170
|
+
const codes = [];
|
|
171
|
+
if (c.isBold()) codes.push('1');
|
|
172
|
+
if (c.isDim()) codes.push('2');
|
|
173
|
+
if (c.isItalic()) codes.push('3');
|
|
174
|
+
if (c.isUnderline()) codes.push('4');
|
|
175
|
+
if (c.isInverse()) codes.push('7');
|
|
176
|
+
if (c.isStrikethrough()) codes.push('9');
|
|
177
|
+
if (c.isFgPalette()) codes.push(`38;5;${c.getFgColor()}`);
|
|
178
|
+
else if (c.isFgRGB()) { const v = c.getFgColor(); codes.push(`38;2;${(v >> 16) & 255};${(v >> 8) & 255};${v & 255}`); }
|
|
179
|
+
if (c.isBgPalette()) codes.push(`48;5;${c.getBgColor()}`);
|
|
180
|
+
else if (c.isBgRGB()) { const v = c.getBgColor(); codes.push(`48;2;${(v >> 16) & 255};${(v >> 8) & 255};${v & 255}`); }
|
|
181
|
+
return codes.join(';');
|
|
182
|
+
};
|
|
213
183
|
const lines = [];
|
|
214
|
-
|
|
215
|
-
|
|
184
|
+
for (let y = 0; y < rows; y++) {
|
|
185
|
+
const line = buffer.getLine(buffer.viewportY + y);
|
|
186
|
+
let out = '';
|
|
187
|
+
let run = null;
|
|
188
|
+
let seen = 0;
|
|
189
|
+
for (let x = 0; line && x < line.length && seen < width; x++) {
|
|
190
|
+
const cell = line.getCell(x);
|
|
191
|
+
const chars = cell.getChars();
|
|
192
|
+
const w = cell.getWidth();
|
|
193
|
+
if (w === 0) continue;
|
|
194
|
+
if (seen + w > width) break;
|
|
195
|
+
const attrs = sgr(cell);
|
|
196
|
+
if (attrs !== run) { out += `${RESET}${attrs ? `\x1b[${attrs}m` : ''}`; run = attrs; }
|
|
197
|
+
out += chars || ' ';
|
|
198
|
+
seen += w;
|
|
199
|
+
}
|
|
200
|
+
lines.push(`${out}${RESET}${' '.repeat(Math.max(0, width - seen))}`);
|
|
201
|
+
}
|
|
216
202
|
return lines;
|
|
217
203
|
}
|
|
218
204
|
|
|
219
|
-
const
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
205
|
+
const completed = (state) => state === 'done' || state === 'error';
|
|
206
|
+
|
|
207
|
+
/** A manual tab choice lasts until the judge completes or restarts. subs maps ranks to { sub, state }. */
|
|
208
|
+
export function subOf(subs, row) {
|
|
209
|
+
const choice = subs[row.rank];
|
|
210
|
+
if (choice && completed(choice.state) === completed(row.state)) return choice.sub;
|
|
211
|
+
return completed(row.state) ? 'verdict' : 'agent';
|
|
212
|
+
}
|
|
224
213
|
|
|
225
|
-
const
|
|
226
|
-
|
|
227
|
-
const
|
|
228
|
-
|
|
214
|
+
const layout = (rows, width) => {
|
|
215
|
+
// The name column is as wide as the longest name, and at least as wide as the words of the total row.
|
|
216
|
+
const desired = Math.max(12, ...rows.map((r) => r.name.length)) + 27;
|
|
217
|
+
const leftWidth = Math.min(desired, width >= TWO_COLUMNS ? Math.floor(width * 0.48) : width);
|
|
218
|
+
return width >= TWO_COLUMNS ? { leftWidth, columnWidth: width - leftWidth - 3 } : { leftWidth, columnWidth: width - 2 };
|
|
229
219
|
};
|
|
230
220
|
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
const
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
function detailContent(row, sub, columnWidth, start, width, frame, window) {
|
|
238
|
-
if (sub === 'criterion') return criterionText(row, columnWidth);
|
|
239
|
-
if (row.state === 'running' && !row.trust) {
|
|
240
|
-
const since = row.restarted !== null ? `restarted with ${row.harness} · ` : '';
|
|
241
|
-
const line = `${GREY}${since}${clock(row.time ?? 0)} · ${row.rank} to watch${RESET}`;
|
|
242
|
-
const size = markSize(columnWidth, window ?? MARK_SIZES[0][0] + 2);
|
|
243
|
-
const art = size ? renderArt({ seed: row.seed ?? 0, frame, start: start + Math.floor((columnWidth - size[1]) / 2), width, size }) : [glimmer('judging', frame, start + Math.floor((columnWidth - 7) / 2), width)];
|
|
244
|
-
const block = [...art, '', line].map((l) => centre(l, columnWidth));
|
|
245
|
-
const above = window === undefined ? 0 : Math.max(0, Math.floor((window - block.length) / 2));
|
|
246
|
-
return [...Array(above).fill(''), ...block];
|
|
247
|
-
}
|
|
248
|
-
const text = row.state === 'done' ? row.result.verdict : row.state === 'error' ? row.error : row.state === 'queued' ? 'The judge has not started.' : `Press ${row.rank} and answer the trust prompt.`;
|
|
249
|
-
return wrap(text, columnWidth);
|
|
221
|
+
/** The size of the panes: the width of the detail pane and the rows of its content, for a screen width by height. */
|
|
222
|
+
export function paneSize(rows, width, height) {
|
|
223
|
+
const { columnWidth } = layout(rows, width);
|
|
224
|
+
const body = height - TOP_ROWS;
|
|
225
|
+
const paneRows = width >= TWO_COLUMNS ? body : body - Math.floor(body / 2) - 1;
|
|
226
|
+
return { cols: Math.max(1, columnWidth), rows: Math.max(1, paneRows - PANE_FRAME) };
|
|
250
227
|
}
|
|
251
228
|
|
|
252
|
-
/** The
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
229
|
+
/** The lines of the Verdict or Criterion content of a row, for a pane of columnWidth. */
|
|
230
|
+
function paneText(row, sub, columnWidth) {
|
|
231
|
+
if (sub === 'criterion') {
|
|
232
|
+
const lines = [];
|
|
233
|
+
if (row.description) lines.push(...wrap(row.description, columnWidth));
|
|
234
|
+
if (row.body) lines.push('', ...renderMarkdown(row.body, columnWidth));
|
|
235
|
+
return lines;
|
|
236
|
+
}
|
|
237
|
+
if (row.state === 'done') return renderMarkdown(row.result.verdict, columnWidth);
|
|
238
|
+
if (row.state === 'error') return renderMarkdown(row.error, columnWidth);
|
|
239
|
+
return [`${GREY}waiting for the verdict${RESET}`];
|
|
257
240
|
}
|
|
258
241
|
|
|
259
|
-
/**
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
const { leftWidth } = layout(left.rows, width);
|
|
266
|
-
const fill = (top, bottom, lines) => {
|
|
267
|
-
const n = lines === undefined || lines < top.length + bottom.length ? 0 : lines - top.length - bottom.length;
|
|
268
|
-
return [...top, ...Array(n).fill(''), ...bottom];
|
|
269
|
-
};
|
|
270
|
-
const lines = [header, `${GREY}${'─'.repeat(width)}${RESET}`];
|
|
271
|
-
const body = height === undefined ? undefined : height - 2;
|
|
272
|
-
if (width >= TWO_COLUMNS) {
|
|
273
|
-
const leftLines = fill(left.lines, leftBottom, body);
|
|
274
|
-
const rightLines = pane(body);
|
|
275
|
-
for (let i = 0; i < Math.max(leftLines.length, rightLines.length); i++) lines.push(`${padTo(leftLines[i] ?? '', leftWidth)} ${edge} ${rightLines[i] ?? ''}`);
|
|
276
|
-
} else {
|
|
277
|
-
const half = body === undefined ? undefined : Math.floor(body / 2);
|
|
278
|
-
const list = fill(left.lines, leftBottom, half);
|
|
279
|
-
lines.push(...list, '', ...pane(body === undefined ? undefined : body - list.length - 1).map((l) => `${edge === `${GREY}│${RESET}` ? ' ' : edge} ${l}`));
|
|
280
|
-
}
|
|
281
|
-
return lines.map((l) => clip(l, width));
|
|
242
|
+
/** The number of lines of the Verdict or Criterion content that the scroll can hide. */
|
|
243
|
+
export function scrollMax({ rows, cursor, sub, width, height }) {
|
|
244
|
+
if (sub === 'agent') return 0;
|
|
245
|
+
const { columnWidth } = layout(rows, width);
|
|
246
|
+
const window = paneSize(rows, width, height).rows;
|
|
247
|
+
return Math.max(0, paneText(rows[cursor], sub, columnWidth).length - window);
|
|
282
248
|
}
|
|
283
249
|
|
|
284
250
|
/**
|
|
285
|
-
*
|
|
286
|
-
*
|
|
287
|
-
*
|
|
288
|
-
* seed the seed of the art of the criterion.
|
|
289
|
-
* cursor is the index of the row whose detail shows at the right. The header line holds the run: panel and target
|
|
290
|
-
* at the left, the version, the harness, and the elapsed time at the right edge. Two columns from TWO_COLUMNS wide,
|
|
291
|
-
* the list above the detail below that. height is the number of lines the overview may fill: the column divider
|
|
292
|
-
* runs the whole height, the rows and the detail header stay at the top, and the total row and the key
|
|
293
|
-
* line sits at the bottom. Under the detail header a sub tab bar picks the content: sub 'verdict' shows the
|
|
294
|
-
* verdict, or the Cross mark centred in the pane with the time under it while the judge runs; sub 'criterion'
|
|
295
|
-
* shows the description and the body of the criterion as plain text. focus 'text' marks the pane with an orange
|
|
296
|
-
* edge and an orange header, and scroll hides that many lines of the content. Without a height, or when the
|
|
297
|
-
* content is taller than the height, the lines follow each other and nothing scrolls.
|
|
251
|
+
* Draws the criterion list and selected detail pane. cursor selects a row; scroll skips lines of its text.
|
|
252
|
+
* The selected row stays visible above the totals. At TWO_COLUMNS or more, the panes sit side by side.
|
|
253
|
+
* At smaller widths, the list sits above the detail. height can be omitted to include all content.
|
|
298
254
|
*/
|
|
299
|
-
export function
|
|
300
|
-
const
|
|
255
|
+
export function renderScreen({ title, target, harness, elapsed, rows, cursor, sub, agent = null, focus = 'list', scroll = 0, keys = '', width, height, frame = 0, version = VERSION }) {
|
|
256
|
+
const run = ` ${BOLD}panels run${RESET} ${title}${GREY} ${target} ${version} · ${harness} · ${clock(elapsed)}${RESET}`;
|
|
257
|
+
const parts = keys.split(' · ');
|
|
258
|
+
while (parts.length > 1 && parts.join(' · ').length > width - 24) parts.pop();
|
|
259
|
+
const keysText = parts.join(' · ');
|
|
260
|
+
const runWidth = Math.max(0, width - keysText.length - 3);
|
|
261
|
+
const header = `${padTo(clip(run, runWidth), runWidth)}${RESET} ${GREY}${keysText}${RESET}`;
|
|
262
|
+
const { leftWidth, columnWidth } = layout(rows, width);
|
|
301
263
|
const pad = leftWidth - 27;
|
|
302
264
|
const cells = (score, time, tokens) => ` ${score.padStart(3)} ${time.padStart(5)} ${tokens.padStart(6)}`;
|
|
303
265
|
const left = rows.map((r, i) => {
|
|
@@ -307,51 +269,75 @@ export function renderOverview({ title, target, harness, elapsed, rows, cursor,
|
|
|
307
269
|
const time = r.time === null ? '' : clock(r.time);
|
|
308
270
|
const tokens = tokensText(r.tokens);
|
|
309
271
|
if (r.state === 'running' && !r.trust) {
|
|
310
|
-
const name = glimmer(r.name.padEnd(pad), frame, 6, width, current ? GLIMMER_ORANGE : GLIMMER);
|
|
272
|
+
const name = glimmer(clip(r.name, pad).padEnd(pad), frame, 6, width, current ? GLIMMER_ORANGE : GLIMMER);
|
|
311
273
|
return `${mark}${current ? ORANGE : ''}${rank}${RESET}${name}${current ? ORANGE : GREY}${cells('·/4', time, tokens)}${RESET}`;
|
|
312
274
|
}
|
|
313
275
|
// A row without a score yet holds the place of the score with ·/4.
|
|
314
276
|
const score = r.state === 'done' ? `${r.result.score}/4` : r.state === 'error' ? '✗' : '·/4';
|
|
315
|
-
const colour =
|
|
316
|
-
const name = current ? `${ORANGE}${r.name.padEnd(pad)}${RESET}` : r.state === 'queued' ? `${GREY}${r.name.padEnd(pad)}${RESET}` : r.name.padEnd(pad);
|
|
317
|
-
const rest = r.state === 'running' ? `${ORANGE}
|
|
277
|
+
const colour = r.state === 'done' ? SCORE_COLOUR[r.result.score] : r.state === 'error' ? SCORE_COLOUR[1] : current ? ORANGE : GREY;
|
|
278
|
+
const name = current ? `${ORANGE}${clip(r.name, pad).padEnd(pad)}${RESET}` : r.state === 'queued' ? `${GREY}${clip(r.name, pad).padEnd(pad)}${RESET}` : clip(r.name, pad).padEnd(pad);
|
|
279
|
+
const rest = r.state === 'running' ? `${ORANGE}enter Agent to answer trust prompt${RESET}` : `${colour}${cells(score, time, tokens)}${RESET}`;
|
|
318
280
|
return `${mark}${current ? ORANGE : ''}${rank}${RESET}${name}${rest}`;
|
|
319
281
|
});
|
|
320
282
|
const done = rows.filter((r) => r.state === 'done');
|
|
283
|
+
const finished = rows.filter((r) => completed(r.state)).length;
|
|
321
284
|
const sum = done.length ? `${quarters(done.reduce((n, r) => n + r.result.score, 0))}/${rows.length}` : `·/${rows.length}`;
|
|
322
285
|
const total = cells(sum, clock(Math.max(0, ...rows.map((r) => r.time ?? 0))), tokensText(rows.some((r) => r.tokens !== null) ? rows.reduce((n, r) => n + (r.tokens ?? 0), 0) : null));
|
|
323
|
-
const totalLine = `${padTo(
|
|
286
|
+
const totalLine = `${padTo(`${GREY} ${finished} of ${rows.length} done${RESET}`, leftWidth - total.length)}${done.length ? total : total.replace(sum, `${GREY}${sum}${RESET}`)}`;
|
|
287
|
+
const leftBottom = [`${GREY}${'─'.repeat(leftWidth)}${RESET}`, totalLine];
|
|
324
288
|
|
|
325
289
|
const row = rows[cursor];
|
|
326
290
|
const level = row.state === 'done' ? `${row.result.score}/4 ${LEVELS[row.result.score]}` : row.state === 'error' ? 'error' : row.state;
|
|
327
291
|
const name = `${row.rank} ${row.name.toUpperCase()}`;
|
|
328
|
-
const
|
|
329
|
-
const
|
|
330
|
-
const subBar = [`${subTab('Verdict', 'verdict')}${GREY} │ ${RESET}${subTab('Criterion', 'criterion')}`, `${GREY}${'─'.repeat(columnWidth)}${RESET}`];
|
|
331
|
-
const bottom = [`${ORANGE}r${RESET} restart ${ORANGE}h${RESET} harness ${row.harness} ${ORANGE}${row.rank}${RESET} watch`];
|
|
332
|
-
// The pane: the header, the sub tab bar and its hairline, a window of the content that the scroll moves, then
|
|
333
|
-
// the key line on the last row.
|
|
292
|
+
const subTab = (label, key) => `${sub === key ? (focus === 'pane' ? `${ORANGE}${BOLD}` : BOLD) : GREY}${label}${RESET}`;
|
|
293
|
+
const levelColour = row.state === 'done' ? SCORE_COLOUR[row.result.score] : row.state === 'error' ? SCORE_COLOUR[1] : GREY;
|
|
334
294
|
const pane = (lines) => {
|
|
335
|
-
const window = lines === undefined ? undefined : lines - PANE_FRAME;
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
295
|
+
const window = lines === undefined ? undefined : Math.max(0, lines - PANE_FRAME);
|
|
296
|
+
let content;
|
|
297
|
+
if (sub === 'agent') content = agent ?? [`${GREY}${row.state === 'queued' ? 'the judge has not started' : 'no screen yet'}${RESET}`];
|
|
298
|
+
else {
|
|
299
|
+
const text = paneText(row, sub, columnWidth);
|
|
300
|
+
const from = window === undefined ? 0 : Math.min(scroll, Math.max(0, text.length - window));
|
|
301
|
+
content = window === undefined ? text : text.slice(from, from + window);
|
|
302
|
+
}
|
|
303
|
+
// The sub tab bar at the left, the name and the level of the row at the right, then a hairline.
|
|
304
|
+
const bar = `${subTab('Verdict', 'verdict')}${GREY} │ ${RESET}${subTab('Agent', 'agent')}${GREY} │ ${RESET}${subTab('Criterion', 'criterion')}`;
|
|
305
|
+
const tail = `${BOLD}${name}${RESET} ${levelColour}${level}${RESET}`;
|
|
306
|
+
const space = columnWidth - plainLength(bar) - 1;
|
|
307
|
+
const detail = space >= plainLength(tail) ? tail : `${levelColour}${level}${RESET}`;
|
|
308
|
+
const top = [`${bar}${' '.repeat(Math.max(1, columnWidth - plainLength(bar) - plainLength(detail)))}${space >= plainLength(detail) ? detail : ''}`, `${GREY}${'─'.repeat(columnWidth)}${RESET}`];
|
|
309
|
+
if (window === undefined) return [...top, ...content];
|
|
310
|
+
return [...top, ...content.slice(0, window), ...Array(Math.max(0, window - content.length)).fill('')];
|
|
341
311
|
};
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
312
|
+
const edge = focus === 'pane' ? `${ORANGE}▌${RESET}` : `${GREY}│${RESET}`;
|
|
313
|
+
const lines = [header, `${GREY}${'─'.repeat(width)}${RESET}`];
|
|
314
|
+
const body = height === undefined ? undefined : height - TOP_ROWS;
|
|
315
|
+
const fill = (top, bottom, n) => {
|
|
316
|
+
if (n === undefined) return [...top, ...bottom];
|
|
317
|
+
const visible = Math.max(0, n - bottom.length);
|
|
318
|
+
const from = Math.max(0, Math.min(cursor - visible + 1, top.length - visible));
|
|
319
|
+
const entries = top.slice(from, from + visible);
|
|
320
|
+
return [...entries, ...Array(Math.max(0, visible - entries.length)).fill(''), ...bottom].slice(0, n);
|
|
321
|
+
};
|
|
322
|
+
if (width >= TWO_COLUMNS) {
|
|
323
|
+
const leftLines = fill(left, leftBottom, body);
|
|
324
|
+
const rightLines = pane(body);
|
|
325
|
+
for (let i = 0; i < Math.max(leftLines.length, rightLines.length); i++) lines.push(`${padTo(clip(leftLines[i] ?? '', leftWidth), leftWidth)} ${edge} ${rightLines[i] ?? ''}`);
|
|
326
|
+
} else {
|
|
327
|
+
const half = body === undefined ? undefined : Math.floor(body / 2);
|
|
328
|
+
const list = fill(left, leftBottom, half);
|
|
329
|
+
lines.push(...list, '', ...pane(body === undefined ? undefined : body - list.length - 1).map((l) => `${focus === 'pane' ? `${ORANGE}▌${RESET}` : ' '} ${l}`));
|
|
330
|
+
}
|
|
331
|
+
return lines.map((l) => clip(l, width));
|
|
351
332
|
}
|
|
352
333
|
|
|
353
|
-
|
|
354
|
-
export
|
|
334
|
+
/** Where the Agent pane's content starts on the screen: { row, col }, 0-based, for the cursor of the judge. */
|
|
335
|
+
export function paneOrigin(rows, width, height) {
|
|
336
|
+
const { leftWidth } = layout(rows, width);
|
|
337
|
+
if (width >= TWO_COLUMNS) return { row: TOP_ROWS + PANE_FRAME, col: leftWidth + 3 };
|
|
338
|
+
const body = height - TOP_ROWS;
|
|
339
|
+
return { row: TOP_ROWS + Math.floor(body / 2) + 1 + PANE_FRAME, col: 2 };
|
|
340
|
+
}
|
|
355
341
|
|
|
356
342
|
const HARNESSES = ['claude', 'codex'];
|
|
357
343
|
|
|
@@ -381,131 +367,81 @@ export function renderModal({ modal, current, width, height }) {
|
|
|
381
367
|
return { top: Math.max(0, Math.floor((height - lines.length) / 2)), left: Math.max(0, Math.floor((width - inner - 2) / 2)), lines };
|
|
382
368
|
}
|
|
383
369
|
|
|
384
|
-
/**
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
* confirm replaces the progress with the quit question; focus 'modal' shows the keys of the open modal. The keys
|
|
391
|
-
* of the overview drop their last words when they do not fit beside the progress.
|
|
392
|
-
*/
|
|
393
|
-
export function renderStrip({ rows, view, frame = 0, width, focus = 'strip', elapsed = 0, confirm = false }) {
|
|
394
|
-
const finished = rows.filter((r) => r.state === 'done' || r.state === 'error').length;
|
|
395
|
-
const progress = confirm ? 'judges still run. q again quits, another key stays' : `${finished} of ${rows.length} done · ${clock(elapsed)}`;
|
|
396
|
-
const full = focus === 'judge' ? 'ctrl-q tabs'
|
|
397
|
-
: focus === 'modal' ? '↑ ↓ · enter pick · esc close'
|
|
398
|
-
: focus === 'text' ? '↑ ↓ scroll · enter list · ← → tabs · q q quit'
|
|
399
|
-
: view === 0 ? '↑ ↓ criterion · tab verdict/criterion · enter scroll · r restart · h harness · ← → tabs · q q quit' : '← → tabs · enter back · q quit';
|
|
400
|
-
const parts = full.split(' · ');
|
|
401
|
-
while (parts.length > 1 && progress.length + 2 + parts.join(' · ').length + 2 > width) parts.pop();
|
|
402
|
-
const keys = parts.join(' · ');
|
|
403
|
-
const names = (level) => ['Overview', ...rows.map((r) => (level === 0 ? r.name : r.name.split(' ')[0]))];
|
|
404
|
-
const used = (list) => 1 + list.reduce((n, t) => n + t.length, 0) + 3 * (list.length - 1);
|
|
405
|
-
const list = [0, 1].map(names).find((l) => used(l) <= width) ?? names(1);
|
|
406
|
-
const activeIndex = view;
|
|
407
|
-
let line = ' ';
|
|
408
|
-
let col = 1;
|
|
409
|
-
list.forEach((text, i) => {
|
|
410
|
-
if (i) { line += `${GREY} │ ${RESET}`; col += 3; }
|
|
411
|
-
const active = i === activeIndex;
|
|
412
|
-
const row = rows[i - 1];
|
|
413
|
-
const start = col;
|
|
414
|
-
col += text.length;
|
|
415
|
-
if (row?.state === 'running' && !row.trust) {
|
|
416
|
-
line += glimmer(text, frame, start, width, active && focus === 'strip' ? GLIMMER_ORANGE : GLIMMER, active ? BOLD : '');
|
|
417
|
-
return;
|
|
418
|
-
}
|
|
419
|
-
line += `${active ? (focus === 'judge' ? BOLD : `${ORANGE}${BOLD}`) : GREY}${text}${RESET}`;
|
|
420
|
-
});
|
|
421
|
-
// A narrow terminal drops the keys before the progress, and then the end of the progress.
|
|
422
|
-
const gap = width - progress.length - keys.length - 2;
|
|
423
|
-
const status = gap >= 2 ? ` ${progress}${' '.repeat(gap)}${keys}` : ` ${progress}`;
|
|
424
|
-
return [`${GREY}${'─'.repeat(width)}${RESET}`, clip(line, width), `${GREY}${clip(status, width)}${RESET}`];
|
|
370
|
+
/** The key hint for the state: the list, the pane on a text sub tab, the Agent pane, a modal, or the quit question. */
|
|
371
|
+
export function hint({ focus, sub, modal, confirm }) {
|
|
372
|
+
if (confirm) return 'judges still run · q again quits · another key stays';
|
|
373
|
+
if (modal) return '↑ ↓ · enter pick · esc close';
|
|
374
|
+
if (focus === 'pane') return sub === 'agent' ? 'ctrl-q list · keys go to judge' : '← → tabs · ↑ ↓ scroll · esc list · q quit';
|
|
375
|
+
return '↑ ↓ rows · ← → tabs · enter pane · r restart · h harness · q quit';
|
|
425
376
|
}
|
|
426
377
|
|
|
427
378
|
/**
|
|
428
|
-
* The next {
|
|
429
|
-
* judge, quit: true, restart: N
|
|
430
|
-
*
|
|
431
|
-
*
|
|
432
|
-
*
|
|
433
|
-
*
|
|
434
|
-
*
|
|
435
|
-
*
|
|
436
|
-
*
|
|
437
|
-
* the pane: there up and down scroll one line, PgUp and PgDn a page, and Esc or Enter return the focus. A modal
|
|
438
|
-
* (state.modal) takes every key until Enter or Esc. On a judge tab, Enter, Esc, or ctrl-q give the focus to the
|
|
439
|
-
* judge. The result carries harness: { rank, value } when the picker picked one.
|
|
379
|
+
* The next state after one key: { cursor, focus, subs, scroll, modal, confirm }, plus send: true when the key goes
|
|
380
|
+
* to the judge of the cursor row, quit: true, restart: N, or harness: { rank, value }. focus is 'list' or 'pane'.
|
|
381
|
+
* subs maps a rank to the explicit { sub, state } choice of the engineer. ctx holds open (the ranks of the judges
|
|
382
|
+
* that have started), running, count, states (by index), harnesses, maxScroll, and page.
|
|
383
|
+
* The list: up and down move the cursor, left and right the sub tab of the cursor row, Enter focuses the pane,
|
|
384
|
+
* r opens the restart modal, h the harness modal, q quits (a second q while a judge runs). The pane on Verdict or
|
|
385
|
+
* Criterion: up and down scroll, PgUp and PgDn a page, left and right the sub tab, Esc or Enter back, q quits.
|
|
386
|
+
* The pane on Agent: every key goes to the judge, except ctrl-q, which returns to the list. A modal takes every
|
|
387
|
+
* key until Enter or Esc.
|
|
440
388
|
*/
|
|
441
|
-
export function press(
|
|
442
|
-
const
|
|
389
|
+
export function press(state, key, { open, running, count, states, harnesses = [], maxScroll: limit = 0, page: step = 10 }) {
|
|
390
|
+
const { cursor, focus, subs, scroll, modal, confirm } = state;
|
|
391
|
+
const row = { rank: cursor + 1, state: states[cursor] };
|
|
392
|
+
const sub = subOf(subs, row);
|
|
393
|
+
const list = { cursor, focus: 'list', subs, scroll, modal: null, confirm: false };
|
|
394
|
+
const turn = (dir) => ({ ...subs, [row.rank]: { sub: SUBS[(SUBS.indexOf(sub) + dir + SUBS.length) % SUBS.length], state: row.state } });
|
|
443
395
|
if (modal) {
|
|
444
396
|
// The modal takes every key: up and down move its cursor, Enter picks the row, Esc closes it, the rest is ignored.
|
|
445
|
-
|
|
446
|
-
if (
|
|
447
|
-
if (
|
|
448
|
-
if (key === '\x1b') return strip(0);
|
|
397
|
+
if (UP.includes(key)) return { ...state, modal: { ...modal, cursor: Math.max(0, modal.cursor - 1) } };
|
|
398
|
+
if (DOWN.includes(key)) return { ...state, modal: { ...modal, cursor: Math.min(modal.options.length - 1, modal.cursor + 1) } };
|
|
399
|
+
if (key === '\x1b') return list;
|
|
449
400
|
if (key === '\r') {
|
|
450
401
|
const pick = modal.options[modal.cursor];
|
|
451
|
-
if (modal.kind === 'harness') return { ...
|
|
452
|
-
return pick === 'restart' ? { ...
|
|
402
|
+
if (modal.kind === 'harness') return { ...list, harness: { rank: modal.rank, value: pick } };
|
|
403
|
+
return pick === 'restart' ? { ...list, restart: modal.rank } : list;
|
|
453
404
|
}
|
|
454
|
-
return
|
|
455
|
-
}
|
|
456
|
-
if (focus === 'judge') return TOGGLE.includes(key) ? strip(view) : { view, confirm, focus, cursor, sub, scroll, modal: null, send: true };
|
|
457
|
-
if (confirm) return { ...strip(0), quit: key === 'q' };
|
|
458
|
-
if (key === 'q') return running ? { view: 0, confirm: true, focus: 'strip', cursor, sub, scroll, modal: null } : { ...strip(0), quit: true };
|
|
459
|
-
if (RIGHT.includes(key) || LEFT.includes(key)) {
|
|
460
|
-
const cycle = [0, ...open];
|
|
461
|
-
const dir = RIGHT.includes(key) ? 1 : -1;
|
|
462
|
-
return strip(cycle[(cycle.indexOf(view) + dir + cycle.length) % cycle.length]);
|
|
405
|
+
return state;
|
|
463
406
|
}
|
|
464
|
-
if (
|
|
465
|
-
if (
|
|
466
|
-
if (
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
if (
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
if (key === '\r' || key === '\x1b') return strip(0);
|
|
478
|
-
return keep;
|
|
479
|
-
}
|
|
480
|
-
if (UP.includes(key)) return { ...strip(0), cursor: Math.max(0, cursor - 1), scroll: 0 };
|
|
481
|
-
if (DOWN.includes(key)) return { ...strip(0), cursor: Math.min(count - 1, cursor + 1), scroll: 0 };
|
|
482
|
-
if (key === '\r') return { ...strip(0), focus: 'text', scroll: 0 };
|
|
483
|
-
return strip(0);
|
|
407
|
+
if (confirm) return { ...list, quit: key === 'q' };
|
|
408
|
+
if (focus === 'pane' && sub === 'agent') return TOGGLE.includes(key) ? list : { ...state, send: true };
|
|
409
|
+
if (key === 'q') return running ? { ...list, confirm: true } : { ...list, quit: true };
|
|
410
|
+
if (RIGHT.includes(key)) return { ...state, subs: turn(1), scroll: 0 };
|
|
411
|
+
if (LEFT.includes(key)) return { ...state, subs: turn(-1), scroll: 0 };
|
|
412
|
+
if (focus === 'pane') {
|
|
413
|
+
const at = (s) => ({ ...state, scroll: Math.max(0, Math.min(limit, s)) });
|
|
414
|
+
if (UP.includes(key)) return at(scroll - 1);
|
|
415
|
+
if (DOWN.includes(key)) return at(scroll + 1);
|
|
416
|
+
if (key === '\x1b[5~') return at(scroll - step);
|
|
417
|
+
if (key === '\x1b[6~') return at(scroll + step);
|
|
418
|
+
if (key === '\r' || key === '\x1b') return list;
|
|
419
|
+
return state;
|
|
484
420
|
}
|
|
485
|
-
if (
|
|
486
|
-
return
|
|
421
|
+
if (UP.includes(key)) return { ...list, cursor: Math.max(0, cursor - 1), scroll: 0 };
|
|
422
|
+
if (DOWN.includes(key)) return { ...list, cursor: Math.min(count - 1, cursor + 1), scroll: 0 };
|
|
423
|
+
if (key === '\r') return { ...list, focus: 'pane', scroll: 0 };
|
|
424
|
+
if (key === 'r') return open.includes(row.rank) ? { ...list, modal: restartModal(row.rank) } : list;
|
|
425
|
+
if (key === 'h') return { ...list, modal: harnessModal(row.rank, harnesses[cursor]) };
|
|
426
|
+
return list;
|
|
487
427
|
}
|
|
488
428
|
|
|
489
429
|
/** Runs the judges, shows the overview, and writes the report. Resolves when the engineer quits. */
|
|
490
|
-
export function runTable({ panel, runDir, harness, cwd, target, yolo = false }) {
|
|
430
|
+
export function runTable({ sid, panel, runDir, harness, cwd, target, yolo = false }) {
|
|
491
431
|
const out = process.stdout;
|
|
492
|
-
|
|
493
|
-
const runSeed = Math.floor(Math.random() * 2 ** 31);
|
|
494
|
-
const criteria = panel.criteria.map((c, i) => ({ rank: i + 1, seed: runSeed + i + 1, ...c }));
|
|
432
|
+
const criteria = panel.criteria.map((c, i) => ({ rank: i + 1, ...c }));
|
|
495
433
|
const judges = [];
|
|
496
|
-
// The harness of each judge: the run harness, until
|
|
434
|
+
// The harness of each judge: the run harness, until the harness modal picks another.
|
|
497
435
|
const harnesses = criteria.map(() => harness);
|
|
498
436
|
let rows = [];
|
|
499
|
-
let
|
|
500
|
-
let ui = {
|
|
501
|
-
let stripTimer;
|
|
502
|
-
let lastData = 0;
|
|
437
|
+
let reportWritten = false;
|
|
438
|
+
let ui = { cursor: 0, focus: 'list', subs: {}, scroll: 0, modal: null, confirm: false };
|
|
503
439
|
const started = Date.now();
|
|
504
440
|
|
|
505
441
|
const judgeDir = (rank) => join(runDir, 'judges', String(rank));
|
|
506
|
-
// A judge gets the lines above the tab strip.
|
|
507
|
-
const size = () => ({ cols: out.columns, rows: out.rows - STRIP_ROWS });
|
|
508
442
|
const seconds = (from, to = Date.now()) => Math.floor((to - from) / 1000);
|
|
443
|
+
// Every judge's pseudo-terminal has the size of the Agent pane.
|
|
444
|
+
const size = () => paneSize(criteria, out.columns, out.rows);
|
|
509
445
|
|
|
510
446
|
function screenText(j) {
|
|
511
447
|
const buffer = j.term.buffer.active;
|
|
@@ -516,7 +452,7 @@ export function runTable({ panel, runDir, harness, cwd, target, yolo = false })
|
|
|
516
452
|
|
|
517
453
|
function state(c, i) {
|
|
518
454
|
const j = judges[i];
|
|
519
|
-
const facts = {
|
|
455
|
+
const facts = { time: j ? seconds(j.started, j.finished) : null, tokens: j?.tokens ?? null };
|
|
520
456
|
const file = join(judgeDir(c.rank), 'answer.json');
|
|
521
457
|
if (existsSync(file)) {
|
|
522
458
|
if (j && !j.finished) j.finished = Date.now();
|
|
@@ -526,6 +462,7 @@ export function runTable({ panel, runDir, harness, cwd, target, yolo = false })
|
|
|
526
462
|
return errors.length ? { ...facts, state: 'error', error: `invalid answer: ${errors.join(' ')}` } : { ...facts, state: 'done', result: answer };
|
|
527
463
|
}
|
|
528
464
|
if (j === undefined) return { ...facts, state: 'queued' };
|
|
465
|
+
if (j.exited) return { ...facts, state: 'error', error: 'the judge exited without an answer.json' };
|
|
529
466
|
if (Date.now() - j.started > MINUTES_PER_CRITERION * 60_000) return { ...facts, state: 'error', error: `no answer.json after ${MINUTES_PER_CRITERION} minutes` };
|
|
530
467
|
const screen = screenText(j);
|
|
531
468
|
const tokens = tokensOnScreen(screen);
|
|
@@ -533,28 +470,20 @@ export function runTable({ panel, runDir, harness, cwd, target, yolo = false })
|
|
|
533
470
|
return { ...facts, tokens: j.tokens ?? null, state: 'running', trust: TRUST_PROMPT.test(screen) };
|
|
534
471
|
}
|
|
535
472
|
|
|
536
|
-
function start(rank
|
|
473
|
+
function start(rank) {
|
|
537
474
|
const [command, ...argv] = writeJudge({ dir: judgeDir(rank), criterion: criteria[rank - 1], harness: harnesses[rank - 1], cwd, target, yolo });
|
|
538
475
|
const { cols, rows: lines } = size();
|
|
539
476
|
const term = new xterm.Terminal({ cols, rows: lines, allowProposedApi: true });
|
|
540
|
-
const serializer = new SerializeAddon();
|
|
541
|
-
term.loadAddon(serializer);
|
|
542
477
|
const proc = pty.spawn(command, argv, { name: 'xterm-256color', cols, rows: lines, cwd, env: process.env });
|
|
543
|
-
const j = { started: Date.now(), finished: undefined,
|
|
544
|
-
|
|
545
|
-
proc.onData((data) => term.write(data, () => {
|
|
546
|
-
if (ui.view !== rank || judges[rank - 1] !== j) return;
|
|
547
|
-
out.write(data);
|
|
548
|
-
lastData = Date.now();
|
|
549
|
-
stripSoon();
|
|
550
|
-
}));
|
|
478
|
+
const j = { started: Date.now(), finished: undefined, tokens: null, term, proc, cursorShown: true, exited: false, dirty: true, screen: null };
|
|
479
|
+
proc.onData((data) => term.write(data, () => { j.dirty = true; }));
|
|
551
480
|
proc.onExit(() => { j.exited = true; });
|
|
552
481
|
// The headless terminal answers the queries of the harness, such as the cursor position.
|
|
553
482
|
term.onData((data) => proc.write(data));
|
|
554
|
-
// The
|
|
483
|
+
// The judge shows or hides its cursor with DECTCEM (mode 25); the Agent pane follows it.
|
|
555
484
|
for (const final of ['h', 'l']) {
|
|
556
485
|
term.parser.registerCsiHandler({ prefix: '?', final }, (params) => {
|
|
557
|
-
|
|
486
|
+
if (params.includes(25)) j.cursorShown = final === 'h';
|
|
558
487
|
return false;
|
|
559
488
|
});
|
|
560
489
|
}
|
|
@@ -570,64 +499,56 @@ export function runTable({ panel, runDir, harness, cwd, target, yolo = false })
|
|
|
570
499
|
stop(judges[rank - 1]);
|
|
571
500
|
const file = join(judgeDir(rank), 'answer.json');
|
|
572
501
|
if (existsSync(file)) unlinkSync(file);
|
|
573
|
-
start(rank
|
|
574
|
-
|
|
575
|
-
}
|
|
576
|
-
|
|
577
|
-
// In a judge, the strip hides the cursor while it draws and then puts it back where the judge had it. A cursor
|
|
578
|
-
// past the last column waits for the next character to wrap; a move there would lose that, so the strip waits.
|
|
579
|
-
function drawStrip() {
|
|
580
|
-
const lines = renderStrip({ rows, view: ui.view, frame, width: out.columns, focus: ui.focus, elapsed: seconds(started), confirm: ui.confirm });
|
|
581
|
-
const top = out.rows - STRIP_ROWS + 1;
|
|
582
|
-
const strip = lines.map((l, i) => `\x1b[${top + i};1H${l}\x1b[K`).join('');
|
|
583
|
-
if (isPage(ui.view)) return void out.write(`${RESET}${strip}`);
|
|
584
|
-
const j = judges[ui.view - 1];
|
|
585
|
-
const { cursorX, cursorY } = j.term.buffer.active;
|
|
586
|
-
if (cursorX >= out.columns) return;
|
|
587
|
-
out.write(`\x1b[?25l${RESET}${strip}\x1b[${cursorY + 1};${cursorX + 1}H\x1b[?25${j.modes[25]}`);
|
|
588
|
-
}
|
|
589
|
-
|
|
590
|
-
// Waits for a pause in the output, so the strip does not land inside an escape sequence that a chunk split.
|
|
591
|
-
function stripSoon() {
|
|
592
|
-
clearTimeout(stripTimer);
|
|
593
|
-
stripTimer = setTimeout(drawStrip, 50);
|
|
594
|
-
}
|
|
595
|
-
|
|
596
|
-
function showJudge(rank) {
|
|
597
|
-
const j = judges[rank - 1];
|
|
598
|
-
j.term.write('', () => {
|
|
599
|
-
if (ui.view !== rank) return;
|
|
600
|
-
// The overview already holds the alternate screen, so a judge on its alternate screen gets a clear after
|
|
601
|
-
// the normal screen, which the serializer writes first. The scroll region keeps the judge above the strip.
|
|
602
|
-
const screen = j.serializer.serialize({ scrollback: 0 }).replace('\x1b[?1049h\x1b[H', '$&\x1b[2J');
|
|
603
|
-
out.write(`${RESET}\x1b[H\x1b[2J\x1b[1;${out.rows - STRIP_ROWS}r${screen}${Object.entries(j.modes).map(([mode, final]) => `\x1b[?${mode}${final}`).join('')}`);
|
|
604
|
-
drawStrip();
|
|
605
|
-
});
|
|
502
|
+
start(rank);
|
|
503
|
+
reportWritten = false;
|
|
606
504
|
}
|
|
607
505
|
|
|
608
506
|
function finish() {
|
|
609
507
|
const outcomes = rows.map((r) => (r.state === 'done' ? { answer: r.result } : { error: r.error }));
|
|
610
|
-
const report = buildReport({ panel, target, outcomes });
|
|
508
|
+
const report = buildReport({ sid, panel, target, outcomes });
|
|
611
509
|
writeFileSync(join(runDir, 'report.json'), `${JSON.stringify(report, null, 2)}\n`);
|
|
612
|
-
|
|
510
|
+
reportWritten = true;
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
/** The lines of the Agent pane of judge j, rendered again only when the judge wrote output or the size changed. */
|
|
514
|
+
function agentLines(j) {
|
|
515
|
+
const { cols, rows: lines } = size();
|
|
516
|
+
if (j.dirty || !j.screen || j.screen.cols !== cols || j.screen.rows !== lines) {
|
|
517
|
+
j.screen = { cols, rows: lines, lines: screenLines(j.term, cols, lines) };
|
|
518
|
+
j.dirty = false;
|
|
519
|
+
}
|
|
520
|
+
return j.screen.lines;
|
|
613
521
|
}
|
|
614
522
|
|
|
615
523
|
let frame = 0;
|
|
616
524
|
function draw() {
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
525
|
+
if (!rows.length) return;
|
|
526
|
+
const row = rows[ui.cursor];
|
|
527
|
+
const sub = subOf(ui.subs, row);
|
|
528
|
+
const j = judges[ui.cursor];
|
|
529
|
+
const lines = renderScreen({
|
|
530
|
+
title: `${panel.owner}/${panel.name} v${panel.version}`, target, harness, elapsed: seconds(started), rows, cursor: ui.cursor, sub,
|
|
531
|
+
agent: sub === 'agent' && j ? agentLines(j) : null, focus: ui.focus, scroll: ui.scroll, keys: hint({ ...ui, sub }), width: out.columns, height: out.rows, frame,
|
|
532
|
+
});
|
|
533
|
+
while (lines.length < out.rows) lines.push('');
|
|
534
|
+
let screen = `\x1b[?25l${lines.slice(0, out.rows).map((l, i) => `\x1b[${i + 1};1H${l}\x1b[K`).join('')}`;
|
|
620
535
|
// The modal sits over the overview; the next draw without it shows the overview again.
|
|
621
536
|
if (ui.modal) {
|
|
622
|
-
const { top, left, lines: box } = renderModal({ modal: ui.modal, current: harnesses[ui.modal.rank - 1], width: out.columns, height: out.rows
|
|
623
|
-
|
|
537
|
+
const { top, left, lines: box } = renderModal({ modal: ui.modal, current: harnesses[ui.modal.rank - 1], width: out.columns, height: out.rows });
|
|
538
|
+
screen += box.map((l, i) => `\x1b[${top + i + 1};${left + 1}H${l}`).join('');
|
|
539
|
+
}
|
|
540
|
+
// In the Agent pane with the focus, the real cursor sits where the judge's cursor is, when the judge shows it.
|
|
541
|
+
if (ui.focus === 'pane' && sub === 'agent' && j && j.cursorShown && !ui.modal) {
|
|
542
|
+
const { row: r, col } = paneOrigin(rows, out.columns, out.rows);
|
|
543
|
+
const { cursorX, cursorY } = j.term.buffer.active;
|
|
544
|
+
screen += `\x1b[${r + cursorY + 1};${col + Math.min(cursorX, size().cols - 1) + 1}H\x1b[?25h`;
|
|
624
545
|
}
|
|
625
|
-
|
|
546
|
+
out.write(screen);
|
|
626
547
|
}
|
|
627
548
|
|
|
628
549
|
function tick() {
|
|
629
550
|
rows = criteria.map((c, i) => ({ ...c, ...state(c, i) }));
|
|
630
|
-
if (!
|
|
551
|
+
if (!reportWritten) {
|
|
631
552
|
const live = rows.filter((r) => r.state === 'running').length;
|
|
632
553
|
const first = judges[0];
|
|
633
554
|
const hold = first && !first.exited && (Date.now() - first.started < FIRST_JUDGE_HEAD_START || rows[0].trust);
|
|
@@ -636,60 +557,57 @@ export function runTable({ panel, runDir, harness, cwd, target, yolo = false })
|
|
|
636
557
|
start(r.rank);
|
|
637
558
|
r.state = 'running';
|
|
638
559
|
}
|
|
639
|
-
if (rows.every((r) => r.state
|
|
560
|
+
if (rows.every((r) => completed(r.state))) finish();
|
|
640
561
|
}
|
|
641
|
-
|
|
562
|
+
draw();
|
|
642
563
|
}
|
|
643
564
|
|
|
644
565
|
return new Promise((done) => {
|
|
645
566
|
const timer = setInterval(tick, 1000);
|
|
646
|
-
const
|
|
647
|
-
frame += 1;
|
|
648
|
-
if (isPage(ui.view)) draw();
|
|
649
|
-
else if (Date.now() - lastData > 60) drawStrip();
|
|
650
|
-
}, 100);
|
|
567
|
+
const frames = setInterval(() => { frame += 1; draw(); }, 100);
|
|
651
568
|
out.write('\x1b[?1049h\x1b[?25l');
|
|
652
569
|
process.stdin.setRawMode(true);
|
|
653
|
-
|
|
654
|
-
const
|
|
655
|
-
const
|
|
656
|
-
const
|
|
657
|
-
|
|
658
|
-
const next = press(ui, key, { open: judges.flatMap((j, i) => (j ? [i + 1] : [])), running: rows.some((r) => r.state === 'running'), count: criteria.length, maxScroll: limit, page: Math.max(1, height - PANE_FRAME), harnesses });
|
|
659
|
-
ui = { view: next.view, confirm: next.confirm, focus: next.focus, cursor: next.cursor, sub: next.sub, scroll: next.scroll, modal: next.modal };
|
|
570
|
+
const onKey = (key) => {
|
|
571
|
+
const sub = rows.length ? subOf(ui.subs, rows[ui.cursor]) : 'verdict';
|
|
572
|
+
const limit = rows.length ? scrollMax({ rows, cursor: ui.cursor, sub, width: out.columns, height: out.rows }) : 0;
|
|
573
|
+
const next = press(ui, key, { open: judges.flatMap((j, i) => (j ? [i + 1] : [])), running: rows.some((r) => r.state === 'running'), count: criteria.length, states: rows.map((r) => r.state), harnesses, maxScroll: limit, page: Math.max(1, size().rows - 1) });
|
|
574
|
+
ui = { cursor: next.cursor, focus: next.focus, subs: next.subs, scroll: next.scroll, modal: next.modal, confirm: next.confirm };
|
|
660
575
|
if (next.send) {
|
|
661
|
-
judges[
|
|
576
|
+
judges[ui.cursor]?.proc.write(key);
|
|
662
577
|
} else if (next.quit) {
|
|
663
578
|
clearInterval(timer);
|
|
664
|
-
clearInterval(
|
|
579
|
+
clearInterval(frames);
|
|
665
580
|
for (const j of judges) stop(j);
|
|
666
|
-
out.write(`${
|
|
581
|
+
out.write(`${RESET}\x1b[?25h\x1b[?1049l`);
|
|
667
582
|
process.stdin.setRawMode(false);
|
|
668
583
|
process.stdin.pause();
|
|
669
584
|
done();
|
|
670
585
|
} else if (next.restart) {
|
|
586
|
+
delete ui.subs[next.restart];
|
|
587
|
+
ui.scroll = 0;
|
|
671
588
|
restart(next.restart);
|
|
672
589
|
tick();
|
|
673
590
|
} else if (next.harness) {
|
|
674
591
|
harnesses[next.harness.rank - 1] = next.harness.value;
|
|
675
592
|
tick();
|
|
676
|
-
} else if (!isPage(ui.view) && ui.view === from) {
|
|
677
|
-
drawStrip();
|
|
678
593
|
} else {
|
|
679
|
-
|
|
680
|
-
if (!isPage(from)) out.write(TABLE_MODES);
|
|
681
|
-
if (isPage(ui.view)) draw();
|
|
682
|
-
else showJudge(ui.view);
|
|
594
|
+
draw();
|
|
683
595
|
}
|
|
596
|
+
};
|
|
597
|
+
process.stdin.on('data', (data) => {
|
|
598
|
+
const text = data.toString();
|
|
599
|
+
const sub = rows.length ? subOf(ui.subs, rows[ui.cursor]) : 'verdict';
|
|
600
|
+
if (ui.focus === 'pane' && sub === 'agent') onKey(text);
|
|
601
|
+
else for (const key of text.match(/\x1b\[[0-?]*[ -/]*[@-~]|\x1bO.|[\s\S]/g)) onKey(key);
|
|
684
602
|
});
|
|
685
603
|
out.on('resize', () => {
|
|
686
604
|
const { cols, rows: lines } = size();
|
|
687
605
|
for (const j of judges) {
|
|
688
606
|
if (j && !j.exited) j.proc.resize(cols, lines);
|
|
689
607
|
j?.term.resize(cols, lines);
|
|
608
|
+
if (j) j.dirty = true;
|
|
690
609
|
}
|
|
691
|
-
|
|
692
|
-
else showJudge(ui.view);
|
|
610
|
+
draw();
|
|
693
611
|
});
|
|
694
612
|
tick();
|
|
695
613
|
});
|