@xsft/panels 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +58 -13
- package/bin/panels.js +8 -4
- package/lib/args.js +18 -4
- package/lib/auth.js +74 -0
- package/lib/judge.js +14 -4
- package/lib/panel.js +10 -9
- package/lib/report.js +12 -8
- package/lib/table.js +335 -344
- package/package.json +1 -2
- package/prompts/judge-system-prompt.txt +11 -13
- package/schema/report.json +7 -2
package/lib/table.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
// The terminal mode of `panels run`: the overview
|
|
2
|
-
//
|
|
1
|
+
// The terminal mode of `panels run`: one screen, the overview. The criteria at the left, and at the right the
|
|
2
|
+
// detail of the cursor criterion: its verdict, its judge's live terminal, or its text. Each judge runs its
|
|
3
|
+
// interactive harness in a pseudo-terminal, and a headless terminal keeps its screen; the Agent pane draws
|
|
4
|
+
// that screen from the buffer, so no mode of a judge reaches the real terminal.
|
|
3
5
|
import { existsSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs';
|
|
4
6
|
import { join } from 'node:path';
|
|
5
7
|
import pty from '@homebridge/node-pty-prebuilt-multiarch';
|
|
6
|
-
import { SerializeAddon } from '@xterm/addon-serialize';
|
|
7
8
|
import xterm from '@xterm/headless';
|
|
8
9
|
import { check, MINUTES_PER_CRITERION, parseAnswer, writeJudge } from './judge.js';
|
|
9
10
|
import { buildReport, VERSION } from './report.js';
|
|
@@ -17,29 +18,33 @@ const ORANGE = '\x1b[38;2;255;106;61m';
|
|
|
17
18
|
const GREY = '\x1b[90m';
|
|
18
19
|
const BOLD = '\x1b[1m';
|
|
19
20
|
const RESET = '\x1b[0m';
|
|
20
|
-
// The colour of a score, from the 16-colour palette: 4 green, 3
|
|
21
|
-
const SCORE_COLOUR = { 4: '\x1b[
|
|
21
|
+
// The colour of a score, from the 16-colour palette: 4 bright green, 3 green, 2 yellow, 1 red.
|
|
22
|
+
const SCORE_COLOUR = { 4: '\x1b[92m', 3: '\x1b[32m', 2: '\x1b[33m', 1: '\x1b[31m' };
|
|
23
|
+
// Inline code in the markdown of a verdict or a body: ink on a dark grey ground.
|
|
24
|
+
const CODE = '\x1b[48;5;236m';
|
|
25
|
+
const CODE_END = '\x1b[49m';
|
|
26
|
+
const BOLD_END = '\x1b[22m';
|
|
22
27
|
const LEVELS = { 1: 'Fails', 2: 'Falls short', 3: 'Almost met', 4: 'Met' };
|
|
23
|
-
// ctrl-q
|
|
28
|
+
// ctrl-q brings the focus back from the Agent pane. A terminal in the kitty keyboard mode sends it as CSI 113;5u.
|
|
24
29
|
const TOGGLE = ['\x11', '\x1b[113;5u'];
|
|
25
30
|
const RIGHT = ['\x1b[C', '\x1bOC'];
|
|
26
31
|
const LEFT = ['\x1b[D', '\x1bOD'];
|
|
27
32
|
const UP = ['\x1b[A', '\x1bOA'];
|
|
28
33
|
const DOWN = ['\x1b[B', '\x1bOB'];
|
|
29
|
-
// Enter, Esc, or ctrl-q give the focus back to the judge of the active tab.
|
|
30
|
-
const BACK = ['\r', '\x1b', ...TOGGLE];
|
|
31
|
-
// Turns off each mode that the screen of a judge can turn on, so the overview gets the keys and the screen as before.
|
|
32
|
-
const TABLE_MODES = '\x1b[?1049h\x1b[r\x1b[0m\x1b[4l\x1b[?6l\x1b[?7h\x1b[?1l\x1b>\x1b[?1000l\x1b[?1002l\x1b[?1003l\x1b[?1006l\x1b[?1004l\x1b[?2004l\x1b[<u\x1b[?25l\x1b[2J';
|
|
33
34
|
// The glimmer on the text of the running judges: grey text, and one soft wave three columns wide that moves one
|
|
34
35
|
// column per frame from the left edge of the screen to the right edge and starts over. A running label lights
|
|
35
36
|
// up only where the wave passes over its own columns, so every running label lights at the same columns.
|
|
36
37
|
const GLIMMER = { base: '\x1b[38;5;244m', mid: '\x1b[38;5;247m', high: '\x1b[38;5;250m' };
|
|
37
38
|
const GLIMMER_ORANGE = { base: ORANGE, mid: '\x1b[38;2;255;140;100m', high: '\x1b[38;2;255;175;145m' };
|
|
38
39
|
const WAVE = 3;
|
|
39
|
-
// The tab strip takes the last three lines of the terminal: a hairline, the tabs, then the status and the keys.
|
|
40
|
-
const STRIP_ROWS = 3;
|
|
41
40
|
// The overview shows two columns from this width up, and the list above the detail below it.
|
|
42
41
|
const TWO_COLUMNS = 100;
|
|
42
|
+
// The sub tabs of the detail pane, in the order the arrows walk them.
|
|
43
|
+
export const SUBS = ['verdict', 'agent', 'criterion'];
|
|
44
|
+
// The header line and the hairline under it; the panes take the rest of the screen.
|
|
45
|
+
const TOP_ROWS = 2;
|
|
46
|
+
// The detail pane holds the sub tab bar and its hairline, then the content.
|
|
47
|
+
const PANE_FRAME = 2;
|
|
43
48
|
|
|
44
49
|
/**
|
|
45
50
|
* text with the glimmer at frame, where the text starts at the screen column start (0-based) on a screen width
|
|
@@ -77,7 +82,6 @@ export function tokensOnScreen(text) {
|
|
|
77
82
|
|
|
78
83
|
const plainLength = (s) => s.replace(/\x1b\[[0-9;]*m/g, '').length;
|
|
79
84
|
const padTo = (s, width) => s + ' '.repeat(Math.max(0, width - plainLength(s)));
|
|
80
|
-
const centre = (s, width) => ' '.repeat(Math.max(0, Math.floor((width - plainLength(s)) / 2))) + s;
|
|
81
85
|
|
|
82
86
|
/** The first width visible characters of a styled line, with every escape sequence kept, so the line never wraps. */
|
|
83
87
|
export function clip(line, width) {
|
|
@@ -86,67 +90,40 @@ export function clip(line, width) {
|
|
|
86
90
|
for (const part of line.split(/(\x1b\[[0-9;]*m)/)) {
|
|
87
91
|
if (part.startsWith('\x1b')) { out += part; continue; }
|
|
88
92
|
const room = Math.max(0, width - seen);
|
|
89
|
-
|
|
90
|
-
|
|
93
|
+
const characters = [...part];
|
|
94
|
+
out += characters.slice(0, room).join('');
|
|
95
|
+
seen += Math.min(characters.length, room);
|
|
91
96
|
}
|
|
92
97
|
return out;
|
|
93
98
|
}
|
|
94
99
|
|
|
95
|
-
/** The words of text in lines of at most width characters. */
|
|
100
|
+
/** The words of text in lines of at most width visible characters. A word can carry escape sequences. */
|
|
96
101
|
export function wrap(text, width) {
|
|
97
102
|
const lines = [];
|
|
98
103
|
let line = '';
|
|
99
104
|
for (const word of text.split(/\s+/).filter(Boolean)) {
|
|
100
|
-
if (line && line
|
|
105
|
+
if (line && plainLength(line) + 1 + plainLength(word) > width) { lines.push(line); line = word; }
|
|
101
106
|
else line = line ? `${line} ${word}` : word;
|
|
102
107
|
}
|
|
103
108
|
if (line) lines.push(line);
|
|
104
109
|
return lines;
|
|
105
110
|
}
|
|
106
111
|
|
|
107
|
-
/** The scores added as quarters: 1/4 + 1/4 + 2/4 is "1". No trailing zeros. */
|
|
108
|
-
const quarters = (sum) => String(Math.round((sum / 4) * 100) / 100);
|
|
109
|
-
|
|
110
|
-
// The Cross mark: a half disc inside a square frame, drawn in a halftone screen. Two sizes, rows by columns.
|
|
111
|
-
const MARK_SIZES = [[11, 22], [7, 14]];
|
|
112
|
-
const HALFTONE = [' ', '░', '▒', '▓'];
|
|
113
|
-
const BREATH = 20;
|
|
114
|
-
|
|
115
112
|
/**
|
|
116
|
-
*
|
|
117
|
-
*
|
|
118
|
-
*
|
|
119
|
-
*
|
|
113
|
+
* Markdown as styled lines for a pane: each paragraph wraps on its own, one blank line stays where the text has
|
|
114
|
+
* one or more, a heading becomes a grey upper-case label, a list item keeps its dash with a hanging indent, a
|
|
115
|
+
* fenced block is indented two spaces as it is, a table line stays as it is, inline code is ink on a dark grey
|
|
116
|
+
* ground, bold is bold, and link marks go.
|
|
120
117
|
*/
|
|
121
|
-
export function
|
|
122
|
-
const
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
const radius = inner.rows / 2;
|
|
126
|
-
const pulse = Math.sin((2 * Math.PI * frame) / BREATH);
|
|
127
|
-
const lines = [`${GREY}┌${'─'.repeat(inner.cols)}┐${RESET}`];
|
|
128
|
-
for (let y = 0; y < inner.rows; y++) {
|
|
129
|
-
let cells = '';
|
|
130
|
-
for (let x = 0; x < inner.cols; x++) {
|
|
131
|
-
const d = Math.hypot((x + 0.5) / 2, y - cy) / radius;
|
|
132
|
-
cells += d > 1 ? ' ' : HALFTONE[Math.max(1, Math.min(3, Math.round(1.5 + (1 - d) * 1.5 + pulse)))];
|
|
118
|
+
export function renderMarkdown(markdown, width) {
|
|
119
|
+
const inline = (text, heading = false) => text.split(/(`[^`]+`)/).map((part) => {
|
|
120
|
+
if (part.startsWith('`') && part.endsWith('`')) {
|
|
121
|
+
return part.slice(1, -1).split(/(\s+)/).map((word) => /^\s*$/.test(word) ? word : `${CODE}${word}${CODE_END}`).join('');
|
|
133
122
|
}
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
/** The size of the mark that fits a pane of columnWidth by lines, or null when only the one-line form fits. */
|
|
141
|
-
export const markSize = (columnWidth, lines) => MARK_SIZES.find(([r, c]) => r + 2 <= lines && c <= columnWidth) ?? null;
|
|
142
|
-
|
|
143
|
-
/**
|
|
144
|
-
* The body of a criterion as plain lines for the text pane: a heading becomes a grey upper-case label, a list
|
|
145
|
-
* item keeps its dash, a fenced block is indented four spaces as it is, a table line stays as it is, and the
|
|
146
|
-
* other text is wrapped to width with the emphasis, code, and link marks removed.
|
|
147
|
-
*/
|
|
148
|
-
export function plainMarkdown(markdown, width) {
|
|
149
|
-
const inline = (t) => t.replace(/\*\*(.+?)\*\*/g, '$1').replace(/__(.+?)__/g, '$1').replace(/`([^`]+)`/g, '$1').replace(/\*(\S[^*]*?)\*/g, '$1').replace(/\[([^\]]+)\]\([^)]*\)/g, '$1');
|
|
123
|
+
const words = heading ? part.toUpperCase() : part;
|
|
124
|
+
return words.replace(/\*\*(.+?)\*\*/g, `${BOLD}$1${BOLD_END}`).replace(/__(.+?)__/g, `${BOLD}$1${BOLD_END}`)
|
|
125
|
+
.replace(/(^|\s)\*(\S[^*]*?)\*(?=\s|$|[.,;:])/g, '$1$2').replace(/\[([^\]]+)\]\(([^)]*)\)/g, '$1 ($2)');
|
|
126
|
+
}).join('');
|
|
150
127
|
const lines = [];
|
|
151
128
|
let paragraph = [];
|
|
152
129
|
let fence = false;
|
|
@@ -154,15 +131,21 @@ export function plainMarkdown(markdown, width) {
|
|
|
154
131
|
if (paragraph.length) lines.push(...wrap(inline(paragraph.join(' ')), width));
|
|
155
132
|
paragraph = [];
|
|
156
133
|
};
|
|
157
|
-
for (const raw of markdown.split('\n')) {
|
|
134
|
+
for (const raw of (markdown ?? '').split('\n')) {
|
|
158
135
|
const line = raw.trimEnd();
|
|
159
136
|
if (/^\s*```/.test(line)) { flush(); fence = !fence; continue; }
|
|
160
|
-
if (fence) { lines.push(`
|
|
161
|
-
if (!line.trim()) { flush(); if (lines.at(-1) !== '') lines.push(''); continue; }
|
|
137
|
+
if (fence) { lines.push(` ${line}`); continue; }
|
|
138
|
+
if (!line.trim()) { flush(); if (lines.length && lines.at(-1) !== '') lines.push(''); continue; }
|
|
162
139
|
const heading = /^#{1,6}\s+(.*)$/.exec(line);
|
|
163
|
-
if (heading) { flush(); if (lines.length && lines.at(-1) !== '') lines.push(''); lines.push(`${GREY}${inline(heading[1])
|
|
164
|
-
const item = /^\s*(
|
|
165
|
-
if (item) {
|
|
140
|
+
if (heading) { flush(); if (lines.length && lines.at(-1) !== '') lines.push(''); lines.push(`${GREY}${inline(heading[1], true)}${RESET}`); continue; }
|
|
141
|
+
const item = /^\s*([-*+]|\d+\.)\s+(.*)$/.exec(line);
|
|
142
|
+
if (item) {
|
|
143
|
+
flush();
|
|
144
|
+
const dash = item[1].length === 1 ? '-' : item[1];
|
|
145
|
+
const [first, ...rest] = wrap(inline(item[2]), width - dash.length - 1);
|
|
146
|
+
lines.push(`${dash} ${first ?? ''}`, ...rest.map((l) => `${' '.repeat(dash.length + 1)}${l}`));
|
|
147
|
+
continue;
|
|
148
|
+
}
|
|
166
149
|
if (/^\s*\|/.test(line)) { flush(); lines.push(inline(line)); continue; }
|
|
167
150
|
paragraph.push(line.trim());
|
|
168
151
|
}
|
|
@@ -172,73 +155,111 @@ export function plainMarkdown(markdown, width) {
|
|
|
172
155
|
return lines;
|
|
173
156
|
}
|
|
174
157
|
|
|
175
|
-
/** The
|
|
176
|
-
|
|
158
|
+
/** The scores added as quarters: 1/4 + 1/4 + 2/4 is "1". No trailing zeros. */
|
|
159
|
+
const quarters = (sum) => String(Math.round((sum / 4) * 100) / 100);
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* The lines of the screen of a judge, from the active buffer of its headless terminal: each row's cells with
|
|
163
|
+
* their attributes (bold, dim, italic, underline, inverse, strikethrough, and the colours in the default,
|
|
164
|
+
* palette, or rgb modes), runs with equal attributes merged, padded or clipped to width. A wide character
|
|
165
|
+
* takes its two cells.
|
|
166
|
+
*/
|
|
167
|
+
export function screenLines(term, width, rows) {
|
|
168
|
+
const buffer = term.buffer.active;
|
|
169
|
+
const sgr = (c) => {
|
|
170
|
+
const codes = [];
|
|
171
|
+
if (c.isBold()) codes.push('1');
|
|
172
|
+
if (c.isDim()) codes.push('2');
|
|
173
|
+
if (c.isItalic()) codes.push('3');
|
|
174
|
+
if (c.isUnderline()) codes.push('4');
|
|
175
|
+
if (c.isInverse()) codes.push('7');
|
|
176
|
+
if (c.isStrikethrough()) codes.push('9');
|
|
177
|
+
if (c.isFgPalette()) codes.push(`38;5;${c.getFgColor()}`);
|
|
178
|
+
else if (c.isFgRGB()) { const v = c.getFgColor(); codes.push(`38;2;${(v >> 16) & 255};${(v >> 8) & 255};${v & 255}`); }
|
|
179
|
+
if (c.isBgPalette()) codes.push(`48;5;${c.getBgColor()}`);
|
|
180
|
+
else if (c.isBgRGB()) { const v = c.getBgColor(); codes.push(`48;2;${(v >> 16) & 255};${(v >> 8) & 255};${v & 255}`); }
|
|
181
|
+
return codes.join(';');
|
|
182
|
+
};
|
|
177
183
|
const lines = [];
|
|
178
|
-
|
|
179
|
-
|
|
184
|
+
for (let y = 0; y < rows; y++) {
|
|
185
|
+
const line = buffer.getLine(buffer.viewportY + y);
|
|
186
|
+
let out = '';
|
|
187
|
+
let run = null;
|
|
188
|
+
let seen = 0;
|
|
189
|
+
for (let x = 0; line && x < line.length && seen < width; x++) {
|
|
190
|
+
const cell = line.getCell(x);
|
|
191
|
+
const chars = cell.getChars();
|
|
192
|
+
const w = cell.getWidth();
|
|
193
|
+
if (w === 0) continue;
|
|
194
|
+
if (seen + w > width) break;
|
|
195
|
+
const attrs = sgr(cell);
|
|
196
|
+
if (attrs !== run) { out += `${RESET}${attrs ? `\x1b[${attrs}m` : ''}`; run = attrs; }
|
|
197
|
+
out += chars || ' ';
|
|
198
|
+
seen += w;
|
|
199
|
+
}
|
|
200
|
+
lines.push(`${out}${RESET}${' '.repeat(Math.max(0, width - seen))}`);
|
|
201
|
+
}
|
|
180
202
|
return lines;
|
|
181
203
|
}
|
|
182
204
|
|
|
183
|
-
const
|
|
184
|
-
// The name column is as wide as the longest name, and at least as wide as the word total under it.
|
|
185
|
-
const leftWidth = Math.max(5, ...rows.map((r) => r.name.length)) + 27;
|
|
186
|
-
return width >= TWO_COLUMNS ? { leftWidth, columnWidth: width - leftWidth - 3, start: leftWidth + 3 } : { leftWidth, columnWidth: width - 2, start: 2 };
|
|
187
|
-
};
|
|
205
|
+
const completed = (state) => state === 'done' || state === 'error';
|
|
188
206
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
const
|
|
192
|
-
|
|
193
|
-
|
|
207
|
+
/** A manual tab choice lasts until the judge completes or restarts. subs maps ranks to { sub, state }. */
|
|
208
|
+
export function subOf(subs, row) {
|
|
209
|
+
const choice = subs[row.rank];
|
|
210
|
+
if (choice && completed(choice.state) === completed(row.state)) return choice.sub;
|
|
211
|
+
return completed(row.state) ? 'verdict' : 'agent';
|
|
212
|
+
}
|
|
194
213
|
|
|
195
|
-
const
|
|
214
|
+
const layout = (rows, width) => {
|
|
215
|
+
// The name column is as wide as the longest name, and at least as wide as the words of the total row.
|
|
216
|
+
const desired = Math.max(12, ...rows.map((r) => r.name.length)) + 27;
|
|
217
|
+
const leftWidth = Math.min(desired, width >= TWO_COLUMNS ? Math.floor(width * 0.48) : width);
|
|
218
|
+
return width >= TWO_COLUMNS ? { leftWidth, columnWidth: width - leftWidth - 3 } : { leftWidth, columnWidth: width - 2 };
|
|
219
|
+
};
|
|
196
220
|
|
|
197
|
-
/** The
|
|
198
|
-
export function
|
|
221
|
+
/** The size of the panes: the width of the detail pane and the rows of its content, for a screen width by height. */
|
|
222
|
+
export function paneSize(rows, width, height) {
|
|
199
223
|
const { columnWidth } = layout(rows, width);
|
|
200
|
-
const
|
|
201
|
-
|
|
224
|
+
const body = height - TOP_ROWS;
|
|
225
|
+
const paneRows = width >= TWO_COLUMNS ? body : body - Math.floor(body / 2) - 1;
|
|
226
|
+
return { cols: Math.max(1, columnWidth), rows: Math.max(1, paneRows - PANE_FRAME) };
|
|
202
227
|
}
|
|
203
228
|
|
|
204
|
-
/**
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
const fill = (top, bottom, lines) => {
|
|
212
|
-
const n = lines === undefined || lines < top.length + bottom.length ? 0 : lines - top.length - bottom.length;
|
|
213
|
-
return [...top, ...Array(n).fill(''), ...bottom];
|
|
214
|
-
};
|
|
215
|
-
const lines = [header, `${GREY}${'─'.repeat(width)}${RESET}`];
|
|
216
|
-
const body = height === undefined ? undefined : height - 2;
|
|
217
|
-
if (width >= TWO_COLUMNS) {
|
|
218
|
-
const leftLines = fill(left.lines, leftBottom, body);
|
|
219
|
-
const rightLines = pane(body);
|
|
220
|
-
for (let i = 0; i < Math.max(leftLines.length, rightLines.length); i++) lines.push(`${padTo(leftLines[i] ?? '', leftWidth)} ${edge} ${rightLines[i] ?? ''}`);
|
|
221
|
-
} else {
|
|
222
|
-
const half = body === undefined ? undefined : Math.floor(body / 2);
|
|
223
|
-
const list = fill(left.lines, leftBottom, half);
|
|
224
|
-
lines.push(...list, '', ...pane(body === undefined ? undefined : body - list.length - 1).map((l) => `${edge === `${GREY}│${RESET}` ? ' ' : edge} ${l}`));
|
|
229
|
+
/** The lines of the Verdict or Criterion content of a row, for a pane of columnWidth. */
|
|
230
|
+
function paneText(row, sub, columnWidth) {
|
|
231
|
+
if (sub === 'criterion') {
|
|
232
|
+
const lines = [];
|
|
233
|
+
if (row.description) lines.push(...wrap(row.description, columnWidth));
|
|
234
|
+
if (row.body) lines.push('', ...renderMarkdown(row.body, columnWidth));
|
|
235
|
+
return lines;
|
|
225
236
|
}
|
|
226
|
-
|
|
237
|
+
if (row.state === 'done') return renderMarkdown(row.result.verdict, columnWidth);
|
|
238
|
+
if (row.state === 'error') return renderMarkdown(row.error, columnWidth);
|
|
239
|
+
return [`${GREY}waiting for the verdict${RESET}`];
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/** The number of lines of the Verdict or Criterion content that the scroll can hide. */
|
|
243
|
+
export function scrollMax({ rows, cursor, sub, width, height }) {
|
|
244
|
+
if (sub === 'agent') return 0;
|
|
245
|
+
const { columnWidth } = layout(rows, width);
|
|
246
|
+
const window = paneSize(rows, width, height).rows;
|
|
247
|
+
return Math.max(0, paneText(rows[cursor], sub, columnWidth).length - window);
|
|
227
248
|
}
|
|
228
249
|
|
|
229
250
|
/**
|
|
230
|
-
*
|
|
231
|
-
*
|
|
232
|
-
*
|
|
233
|
-
* cursor is the index of the row whose detail shows at the right. The header line holds the run: panel and target
|
|
234
|
-
* at the left, the version, the harness, and the elapsed time at the right edge. Two columns from TWO_COLUMNS wide,
|
|
235
|
-
* the list above the detail below that. height is the number of lines the overview may fill: the column divider
|
|
236
|
-
* runs the whole height, the rows and the detail header stay at the top, and the total row and the key and fact
|
|
237
|
-
* lines sit at the bottom. The detail of a running judge is the Cross mark, centred in the pane, with the time
|
|
238
|
-
* under it. Without a height, or when the content is taller than the height, the lines follow each other.
|
|
251
|
+
* Draws the criterion list and selected detail pane. cursor selects a row; scroll skips lines of its text.
|
|
252
|
+
* The selected row stays visible above the totals. At TWO_COLUMNS or more, the panes sit side by side.
|
|
253
|
+
* At smaller widths, the list sits above the detail. height can be omitted to include all content.
|
|
239
254
|
*/
|
|
240
|
-
export function
|
|
241
|
-
const
|
|
255
|
+
export function renderScreen({ title, target, harness, elapsed, rows, cursor, sub, agent = null, focus = 'list', scroll = 0, keys = '', width, height, frame = 0, version = VERSION }) {
|
|
256
|
+
const run = ` ${BOLD}panels run${RESET} ${title}${GREY} ${target} ${version} · ${harness} · ${clock(elapsed)}${RESET}`;
|
|
257
|
+
const parts = keys.split(' · ');
|
|
258
|
+
while (parts.length > 1 && parts.join(' · ').length > width - 24) parts.pop();
|
|
259
|
+
const keysText = parts.join(' · ');
|
|
260
|
+
const runWidth = Math.max(0, width - keysText.length - 3);
|
|
261
|
+
const header = `${padTo(clip(run, runWidth), runWidth)}${RESET} ${GREY}${keysText}${RESET}`;
|
|
262
|
+
const { leftWidth, columnWidth } = layout(rows, width);
|
|
242
263
|
const pad = leftWidth - 27;
|
|
243
264
|
const cells = (score, time, tokens) => ` ${score.padStart(3)} ${time.padStart(5)} ${tokens.padStart(6)}`;
|
|
244
265
|
const left = rows.map((r, i) => {
|
|
@@ -248,195 +269,179 @@ export function renderOverview({ title, target, harness, elapsed, rows, cursor,
|
|
|
248
269
|
const time = r.time === null ? '' : clock(r.time);
|
|
249
270
|
const tokens = tokensText(r.tokens);
|
|
250
271
|
if (r.state === 'running' && !r.trust) {
|
|
251
|
-
const name = glimmer(r.name.padEnd(pad), frame, 6, width, current ? GLIMMER_ORANGE : GLIMMER);
|
|
272
|
+
const name = glimmer(clip(r.name, pad).padEnd(pad), frame, 6, width, current ? GLIMMER_ORANGE : GLIMMER);
|
|
252
273
|
return `${mark}${current ? ORANGE : ''}${rank}${RESET}${name}${current ? ORANGE : GREY}${cells('·/4', time, tokens)}${RESET}`;
|
|
253
274
|
}
|
|
254
275
|
// A row without a score yet holds the place of the score with ·/4.
|
|
255
276
|
const score = r.state === 'done' ? `${r.result.score}/4` : r.state === 'error' ? '✗' : '·/4';
|
|
256
|
-
const colour =
|
|
257
|
-
const name = current ? `${ORANGE}${r.name.padEnd(pad)}${RESET}` : r.state === 'queued' ? `${GREY}${r.name.padEnd(pad)}${RESET}` : r.name.padEnd(pad);
|
|
258
|
-
const rest = r.state === 'running' ? `${ORANGE}
|
|
277
|
+
const colour = r.state === 'done' ? SCORE_COLOUR[r.result.score] : r.state === 'error' ? SCORE_COLOUR[1] : current ? ORANGE : GREY;
|
|
278
|
+
const name = current ? `${ORANGE}${clip(r.name, pad).padEnd(pad)}${RESET}` : r.state === 'queued' ? `${GREY}${clip(r.name, pad).padEnd(pad)}${RESET}` : clip(r.name, pad).padEnd(pad);
|
|
279
|
+
const rest = r.state === 'running' ? `${ORANGE}enter Agent to answer trust prompt${RESET}` : `${colour}${cells(score, time, tokens)}${RESET}`;
|
|
259
280
|
return `${mark}${current ? ORANGE : ''}${rank}${RESET}${name}${rest}`;
|
|
260
281
|
});
|
|
261
282
|
const done = rows.filter((r) => r.state === 'done');
|
|
283
|
+
const finished = rows.filter((r) => completed(r.state)).length;
|
|
262
284
|
const sum = done.length ? `${quarters(done.reduce((n, r) => n + r.result.score, 0))}/${rows.length}` : `·/${rows.length}`;
|
|
263
285
|
const total = cells(sum, clock(Math.max(0, ...rows.map((r) => r.time ?? 0))), tokensText(rows.some((r) => r.tokens !== null) ? rows.reduce((n, r) => n + (r.tokens ?? 0), 0) : null));
|
|
264
|
-
const totalLine = `${padTo(
|
|
286
|
+
const totalLine = `${padTo(`${GREY} ${finished} of ${rows.length} done${RESET}`, leftWidth - total.length)}${done.length ? total : total.replace(sum, `${GREY}${sum}${RESET}`)}`;
|
|
287
|
+
const leftBottom = [`${GREY}${'─'.repeat(leftWidth)}${RESET}`, totalLine];
|
|
265
288
|
|
|
266
289
|
const row = rows[cursor];
|
|
267
290
|
const level = row.state === 'done' ? `${row.result.score}/4 ${LEVELS[row.result.score]}` : row.state === 'error' ? 'error' : row.state;
|
|
268
291
|
const name = `${row.rank} ${row.name.toUpperCase()}`;
|
|
269
|
-
const
|
|
270
|
-
const
|
|
271
|
-
const bottom = [`${ORANGE}r${RESET} restart ${ORANGE}h${RESET} harness ${row.harness} ${ORANGE}${row.rank}${RESET} watch`, '', `${GREY}${facts.join(' · ')}${RESET}`];
|
|
272
|
-
// The pane: the header, the content, then the bottom lines at the bottom. A running judge shows the mark centred.
|
|
292
|
+
const subTab = (label, key) => `${sub === key ? (focus === 'pane' ? `${ORANGE}${BOLD}` : BOLD) : GREY}${label}${RESET}`;
|
|
293
|
+
const levelColour = row.state === 'done' ? SCORE_COLOUR[row.result.score] : row.state === 'error' ? SCORE_COLOUR[1] : GREY;
|
|
273
294
|
const pane = (lines) => {
|
|
274
|
-
|
|
275
|
-
const window = lines === undefined ? undefined : lines - 2 - 1 - bottom.length;
|
|
295
|
+
const window = lines === undefined ? undefined : Math.max(0, lines - PANE_FRAME);
|
|
276
296
|
let content;
|
|
277
|
-
if (row.state === '
|
|
278
|
-
|
|
279
|
-
const
|
|
280
|
-
const
|
|
281
|
-
|
|
282
|
-
const block = [...art, '', line].map((l) => centre(l, columnWidth));
|
|
283
|
-
const above = window === undefined ? 0 : Math.max(0, Math.floor((window - block.length) / 2));
|
|
284
|
-
content = [...Array(above).fill(''), ...block];
|
|
285
|
-
} else {
|
|
286
|
-
const text = row.state === 'done' ? row.result.verdict : row.state === 'error' ? row.error : row.state === 'queued' ? 'The judge has not started.' : `Press ${row.rank} and answer the trust prompt.`;
|
|
287
|
-
content = wrap(text, columnWidth);
|
|
297
|
+
if (sub === 'agent') content = agent ?? [`${GREY}${row.state === 'queued' ? 'the judge has not started' : 'no screen yet'}${RESET}`];
|
|
298
|
+
else {
|
|
299
|
+
const text = paneText(row, sub, columnWidth);
|
|
300
|
+
const from = window === undefined ? 0 : Math.min(scroll, Math.max(0, text.length - window));
|
|
301
|
+
content = window === undefined ? text : text.slice(from, from + window);
|
|
288
302
|
}
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
303
|
+
// The sub tab bar at the left, the name and the level of the row at the right, then a hairline.
|
|
304
|
+
const bar = `${subTab('Verdict', 'verdict')}${GREY} │ ${RESET}${subTab('Agent', 'agent')}${GREY} │ ${RESET}${subTab('Criterion', 'criterion')}`;
|
|
305
|
+
const tail = `${BOLD}${name}${RESET} ${levelColour}${level}${RESET}`;
|
|
306
|
+
const space = columnWidth - plainLength(bar) - 1;
|
|
307
|
+
const detail = space >= plainLength(tail) ? tail : `${levelColour}${level}${RESET}`;
|
|
308
|
+
const top = [`${bar}${' '.repeat(Math.max(1, columnWidth - plainLength(bar) - plainLength(detail)))}${space >= plainLength(detail) ? detail : ''}`, `${GREY}${'─'.repeat(columnWidth)}${RESET}`];
|
|
309
|
+
if (window === undefined) return [...top, ...content];
|
|
310
|
+
return [...top, ...content.slice(0, window), ...Array(Math.max(0, window - content.length)).fill('')];
|
|
292
311
|
};
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
312
|
+
const edge = focus === 'pane' ? `${ORANGE}▌${RESET}` : `${GREY}│${RESET}`;
|
|
313
|
+
const lines = [header, `${GREY}${'─'.repeat(width)}${RESET}`];
|
|
314
|
+
const body = height === undefined ? undefined : height - TOP_ROWS;
|
|
315
|
+
const fill = (top, bottom, n) => {
|
|
316
|
+
if (n === undefined) return [...top, ...bottom];
|
|
317
|
+
const visible = Math.max(0, n - bottom.length);
|
|
318
|
+
const from = Math.max(0, Math.min(cursor - visible + 1, top.length - visible));
|
|
319
|
+
const entries = top.slice(from, from + visible);
|
|
320
|
+
return [...entries, ...Array(Math.max(0, visible - entries.length)).fill(''), ...bottom].slice(0, n);
|
|
321
|
+
};
|
|
322
|
+
if (width >= TWO_COLUMNS) {
|
|
323
|
+
const leftLines = fill(left, leftBottom, body);
|
|
324
|
+
const rightLines = pane(body);
|
|
325
|
+
for (let i = 0; i < Math.max(leftLines.length, rightLines.length); i++) lines.push(`${padTo(clip(leftLines[i] ?? '', leftWidth), leftWidth)} ${edge} ${rightLines[i] ?? ''}`);
|
|
326
|
+
} else {
|
|
327
|
+
const half = body === undefined ? undefined : Math.floor(body / 2);
|
|
328
|
+
const list = fill(left, leftBottom, half);
|
|
329
|
+
lines.push(...list, '', ...pane(body === undefined ? undefined : body - list.length - 1).map((l) => `${focus === 'pane' ? `${ORANGE}▌${RESET}` : ' '} ${l}`));
|
|
330
|
+
}
|
|
331
|
+
return lines.map((l) => clip(l, width));
|
|
302
332
|
}
|
|
303
333
|
|
|
304
|
-
/**
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
const { leftWidth, columnWidth } = layout(rows, width);
|
|
311
|
-
const pad = leftWidth - 27;
|
|
312
|
-
const left = rows.map((r, i) => (i === cursor ? `${ORANGE}▸ ${String(r.rank).padStart(2)} ${r.name.padEnd(pad)}${RESET}` : ` ${String(r.rank).padStart(2)} ${r.name.padEnd(pad)}`));
|
|
313
|
-
const row = rows[cursor];
|
|
314
|
-
const name = `${row.rank} ${row.name.toUpperCase()}`;
|
|
315
|
-
const header = `${focus === 'text' ? `${ORANGE}${BOLD}` : BOLD}${name}${RESET}`;
|
|
316
|
-
const text = criterionText(row, columnWidth);
|
|
317
|
-
const pane = (lines) => {
|
|
318
|
-
const window = lines === undefined ? text.length : lines - 2;
|
|
319
|
-
const from = Math.min(scroll, Math.max(0, text.length - window));
|
|
320
|
-
return [header, '', ...text.slice(from, from + Math.max(0, window))];
|
|
321
|
-
};
|
|
322
|
-
return page({
|
|
323
|
-
header: headerLine(title, target, harness, elapsed, width, version),
|
|
324
|
-
width,
|
|
325
|
-
height,
|
|
326
|
-
left: { rows, lines: left },
|
|
327
|
-
leftBottom: [],
|
|
328
|
-
pane,
|
|
329
|
-
edge: focus === 'text' ? `${ORANGE}▌${RESET}` : `${GREY}│${RESET}`,
|
|
330
|
-
});
|
|
334
|
+
/** Where the Agent pane's content starts on the screen: { row, col }, 0-based, for the cursor of the judge. */
|
|
335
|
+
export function paneOrigin(rows, width, height) {
|
|
336
|
+
const { leftWidth } = layout(rows, width);
|
|
337
|
+
if (width >= TWO_COLUMNS) return { row: TOP_ROWS + PANE_FRAME, col: leftWidth + 3 };
|
|
338
|
+
const body = height - TOP_ROWS;
|
|
339
|
+
return { row: TOP_ROWS + Math.floor(body / 2) + 1 + PANE_FRAME, col: 2 };
|
|
331
340
|
}
|
|
332
341
|
|
|
333
|
-
|
|
334
|
-
export const PANEL = 'panel';
|
|
335
|
-
export const isPage = (view) => view === 0 || view === PANEL;
|
|
342
|
+
const HARNESSES = ['claude', 'codex'];
|
|
336
343
|
|
|
337
344
|
/**
|
|
338
|
-
* The
|
|
339
|
-
*
|
|
340
|
-
* name of each criterion, with a grey │ between the tabs. A running tab glimmers. The names step down to the
|
|
341
|
-
* first word when they do not fit the width. focus is 'strip' (the active tab is bold orange, the arrows move
|
|
342
|
-
* it), 'text' (the text pane of the Panel tab scrolls), or 'judge' (the active tab is bold ink, the keys go to
|
|
343
|
-
* the judge). confirm replaces the progress with the quit question.
|
|
345
|
+
* The modal states: { kind: 'harness', rank, options, cursor } picks the harness of judge rank; { kind: 'restart',
|
|
346
|
+
* rank, options, cursor } asks before a restart. options are the rows, cursor the index of the orange row.
|
|
344
347
|
*/
|
|
345
|
-
export
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
const
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
return [`${GREY}${'─'.repeat(width)}${RESET}`, clip(line, width), `${GREY}${clip(status, width)}${RESET}`];
|
|
348
|
+
export const harnessModal = (rank, current) => ({ kind: 'harness', rank, options: HARNESSES, cursor: Math.max(0, HARNESSES.indexOf(current)) });
|
|
349
|
+
export const restartModal = (rank) => ({ kind: 'restart', rank, options: ['restart', 'cancel'], cursor: 1 });
|
|
350
|
+
|
|
351
|
+
/**
|
|
352
|
+
* The lines of a modal and where they go: { top, left, lines }, centred on a screen width by height. A grey
|
|
353
|
+
* box-drawing frame, a bold title, a grey note when there is one, then one row per option, the cursor row orange
|
|
354
|
+
* with ▸. current is the harness the judge uses now, marked in the harness picker.
|
|
355
|
+
*/
|
|
356
|
+
export function renderModal({ modal, current, width, height }) {
|
|
357
|
+
const title = modal.kind === 'harness' ? `Harness for judge ${modal.rank}` : `Restart judge ${modal.rank}?`;
|
|
358
|
+
const note = modal.kind === 'restart' ? `with ${current} · the current answer is discarded` : null;
|
|
359
|
+
const rows = modal.options.map((o) => (modal.kind === 'harness' && o === current ? `${o} (current)` : o));
|
|
360
|
+
const inner = Math.max(title.length, note?.length ?? 0, ...rows.map((r) => r.length + 2)) + 2;
|
|
361
|
+
const line = (text) => `${GREY}│${RESET} ${padTo(text, inner - 2)} ${GREY}│${RESET}`;
|
|
362
|
+
const lines = [`${GREY}┌${'─'.repeat(inner)}┐${RESET}`, line(`${BOLD}${title}${RESET}`)];
|
|
363
|
+
if (note) lines.push(line(`${GREY}${note}${RESET}`));
|
|
364
|
+
lines.push(line(''));
|
|
365
|
+
rows.forEach((r, i) => lines.push(line(i === modal.cursor ? `${ORANGE}▸ ${r}${RESET}` : ` ${r}`)));
|
|
366
|
+
lines.push(`${GREY}└${'─'.repeat(inner)}┘${RESET}`);
|
|
367
|
+
return { top: Math.max(0, Math.floor((height - lines.length) / 2)), left: Math.max(0, Math.floor((width - inner - 2) / 2)), lines };
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
/** The key hint for the state: the list, the pane on a text sub tab, the Agent pane, a modal, or the quit question. */
|
|
371
|
+
export function hint({ focus, sub, modal, confirm }) {
|
|
372
|
+
if (confirm) return 'judges still run · q again quits · another key stays';
|
|
373
|
+
if (modal) return '↑ ↓ · enter pick · esc close';
|
|
374
|
+
if (focus === 'pane') return sub === 'agent' ? 'ctrl-q list · keys go to judge' : '← → tabs · ↑ ↓ scroll · esc list · q quit';
|
|
375
|
+
return '↑ ↓ rows · ← → tabs · enter pane · r restart · h harness · q quit';
|
|
374
376
|
}
|
|
375
377
|
|
|
376
378
|
/**
|
|
377
|
-
* The next {
|
|
378
|
-
* quit: true, restart: N
|
|
379
|
-
*
|
|
380
|
-
*
|
|
381
|
-
*
|
|
382
|
-
*
|
|
383
|
-
*
|
|
384
|
-
*
|
|
385
|
-
*
|
|
386
|
-
* the text pane, where up and down scroll one line, PgUp and PgDn a page, and Esc or Enter return the focus.
|
|
387
|
-
* On a judge tab, Enter, Esc, or ctrl-q give the focus to the judge.
|
|
379
|
+
* The next state after one key: { cursor, focus, subs, scroll, modal, confirm }, plus send: true when the key goes
|
|
380
|
+
* to the judge of the cursor row, quit: true, restart: N, or harness: { rank, value }. focus is 'list' or 'pane'.
|
|
381
|
+
* subs maps a rank to the explicit { sub, state } choice of the engineer. ctx holds open (the ranks of the judges
|
|
382
|
+
* that have started), running, count, states (by index), harnesses, maxScroll, and page.
|
|
383
|
+
* The list: up and down move the cursor, left and right the sub tab of the cursor row, Enter focuses the pane,
|
|
384
|
+
* r opens the restart modal, h the harness modal, q quits (a second q while a judge runs). The pane on Verdict or
|
|
385
|
+
* Criterion: up and down scroll, PgUp and PgDn a page, left and right the sub tab, Esc or Enter back, q quits.
|
|
386
|
+
* The pane on Agent: every key goes to the judge, except ctrl-q, which returns to the list. A modal takes every
|
|
387
|
+
* key until Enter or Esc.
|
|
388
388
|
*/
|
|
389
|
-
export function press(
|
|
390
|
-
const
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
if (focus === 'text') {
|
|
405
|
-
const text = (s) => ({ view, confirm: false, focus: 'text', cursor, scroll: Math.max(0, Math.min(limit, s)) });
|
|
406
|
-
if (UP.includes(key)) return text(scroll - 1);
|
|
407
|
-
if (DOWN.includes(key)) return text(scroll + 1);
|
|
408
|
-
if (key === '\x1b[5~') return text(scroll - step);
|
|
409
|
-
if (key === '\x1b[6~') return text(scroll + step);
|
|
410
|
-
if (key === '\r' || key === '\x1b') return strip(view);
|
|
411
|
-
return text(scroll);
|
|
389
|
+
export function press(state, key, { open, running, count, states, harnesses = [], maxScroll: limit = 0, page: step = 10 }) {
|
|
390
|
+
const { cursor, focus, subs, scroll, modal, confirm } = state;
|
|
391
|
+
const row = { rank: cursor + 1, state: states[cursor] };
|
|
392
|
+
const sub = subOf(subs, row);
|
|
393
|
+
const list = { cursor, focus: 'list', subs, scroll, modal: null, confirm: false };
|
|
394
|
+
const turn = (dir) => ({ ...subs, [row.rank]: { sub: SUBS[(SUBS.indexOf(sub) + dir + SUBS.length) % SUBS.length], state: row.state } });
|
|
395
|
+
if (modal) {
|
|
396
|
+
// The modal takes every key: up and down move its cursor, Enter picks the row, Esc closes it, the rest is ignored.
|
|
397
|
+
if (UP.includes(key)) return { ...state, modal: { ...modal, cursor: Math.max(0, modal.cursor - 1) } };
|
|
398
|
+
if (DOWN.includes(key)) return { ...state, modal: { ...modal, cursor: Math.min(modal.options.length - 1, modal.cursor + 1) } };
|
|
399
|
+
if (key === '\x1b') return list;
|
|
400
|
+
if (key === '\r') {
|
|
401
|
+
const pick = modal.options[modal.cursor];
|
|
402
|
+
if (modal.kind === 'harness') return { ...list, harness: { rank: modal.rank, value: pick } };
|
|
403
|
+
return pick === 'restart' ? { ...list, restart: modal.rank } : list;
|
|
412
404
|
}
|
|
413
|
-
|
|
414
|
-
if (DOWN.includes(key)) return { ...strip(view), cursor: Math.min(count - 1, cursor + 1), scroll: 0 };
|
|
415
|
-
if (view === PANEL && key === '\r') return { view, confirm: false, focus: 'text', cursor, scroll: 0 };
|
|
416
|
-
return strip(view);
|
|
405
|
+
return state;
|
|
417
406
|
}
|
|
418
|
-
if (
|
|
419
|
-
return
|
|
407
|
+
if (confirm) return { ...list, quit: key === 'q' };
|
|
408
|
+
if (focus === 'pane' && sub === 'agent') return TOGGLE.includes(key) ? list : { ...state, send: true };
|
|
409
|
+
if (key === 'q') return running ? { ...list, confirm: true } : { ...list, quit: true };
|
|
410
|
+
if (RIGHT.includes(key)) return { ...state, subs: turn(1), scroll: 0 };
|
|
411
|
+
if (LEFT.includes(key)) return { ...state, subs: turn(-1), scroll: 0 };
|
|
412
|
+
if (focus === 'pane') {
|
|
413
|
+
const at = (s) => ({ ...state, scroll: Math.max(0, Math.min(limit, s)) });
|
|
414
|
+
if (UP.includes(key)) return at(scroll - 1);
|
|
415
|
+
if (DOWN.includes(key)) return at(scroll + 1);
|
|
416
|
+
if (key === '\x1b[5~') return at(scroll - step);
|
|
417
|
+
if (key === '\x1b[6~') return at(scroll + step);
|
|
418
|
+
if (key === '\r' || key === '\x1b') return list;
|
|
419
|
+
return state;
|
|
420
|
+
}
|
|
421
|
+
if (UP.includes(key)) return { ...list, cursor: Math.max(0, cursor - 1), scroll: 0 };
|
|
422
|
+
if (DOWN.includes(key)) return { ...list, cursor: Math.min(count - 1, cursor + 1), scroll: 0 };
|
|
423
|
+
if (key === '\r') return { ...list, focus: 'pane', scroll: 0 };
|
|
424
|
+
if (key === 'r') return open.includes(row.rank) ? { ...list, modal: restartModal(row.rank) } : list;
|
|
425
|
+
if (key === 'h') return { ...list, modal: harnessModal(row.rank, harnesses[cursor]) };
|
|
426
|
+
return list;
|
|
420
427
|
}
|
|
421
428
|
|
|
422
429
|
/** Runs the judges, shows the overview, and writes the report. Resolves when the engineer quits. */
|
|
423
|
-
export function runTable({ panel, runDir, harness, cwd, target, yolo = false }) {
|
|
430
|
+
export function runTable({ sid, panel, runDir, harness, cwd, target, yolo = false }) {
|
|
424
431
|
const out = process.stdout;
|
|
425
432
|
const criteria = panel.criteria.map((c, i) => ({ rank: i + 1, ...c }));
|
|
426
433
|
const judges = [];
|
|
427
|
-
// The harness of each judge: the run harness, until
|
|
434
|
+
// The harness of each judge: the run harness, until the harness modal picks another.
|
|
428
435
|
const harnesses = criteria.map(() => harness);
|
|
429
436
|
let rows = [];
|
|
430
|
-
let
|
|
431
|
-
let ui = {
|
|
432
|
-
let stripTimer;
|
|
433
|
-
let lastData = 0;
|
|
437
|
+
let reportWritten = false;
|
|
438
|
+
let ui = { cursor: 0, focus: 'list', subs: {}, scroll: 0, modal: null, confirm: false };
|
|
434
439
|
const started = Date.now();
|
|
435
440
|
|
|
436
441
|
const judgeDir = (rank) => join(runDir, 'judges', String(rank));
|
|
437
|
-
// A judge gets the lines above the tab strip.
|
|
438
|
-
const size = () => ({ cols: out.columns, rows: out.rows - STRIP_ROWS });
|
|
439
442
|
const seconds = (from, to = Date.now()) => Math.floor((to - from) / 1000);
|
|
443
|
+
// Every judge's pseudo-terminal has the size of the Agent pane.
|
|
444
|
+
const size = () => paneSize(criteria, out.columns, out.rows);
|
|
440
445
|
|
|
441
446
|
function screenText(j) {
|
|
442
447
|
const buffer = j.term.buffer.active;
|
|
@@ -447,7 +452,7 @@ export function runTable({ panel, runDir, harness, cwd, target, yolo = false })
|
|
|
447
452
|
|
|
448
453
|
function state(c, i) {
|
|
449
454
|
const j = judges[i];
|
|
450
|
-
const facts = {
|
|
455
|
+
const facts = { time: j ? seconds(j.started, j.finished) : null, tokens: j?.tokens ?? null };
|
|
451
456
|
const file = join(judgeDir(c.rank), 'answer.json');
|
|
452
457
|
if (existsSync(file)) {
|
|
453
458
|
if (j && !j.finished) j.finished = Date.now();
|
|
@@ -457,6 +462,7 @@ export function runTable({ panel, runDir, harness, cwd, target, yolo = false })
|
|
|
457
462
|
return errors.length ? { ...facts, state: 'error', error: `invalid answer: ${errors.join(' ')}` } : { ...facts, state: 'done', result: answer };
|
|
458
463
|
}
|
|
459
464
|
if (j === undefined) return { ...facts, state: 'queued' };
|
|
465
|
+
if (j.exited) return { ...facts, state: 'error', error: 'the judge exited without an answer.json' };
|
|
460
466
|
if (Date.now() - j.started > MINUTES_PER_CRITERION * 60_000) return { ...facts, state: 'error', error: `no answer.json after ${MINUTES_PER_CRITERION} minutes` };
|
|
461
467
|
const screen = screenText(j);
|
|
462
468
|
const tokens = tokensOnScreen(screen);
|
|
@@ -464,28 +470,20 @@ export function runTable({ panel, runDir, harness, cwd, target, yolo = false })
|
|
|
464
470
|
return { ...facts, tokens: j.tokens ?? null, state: 'running', trust: TRUST_PROMPT.test(screen) };
|
|
465
471
|
}
|
|
466
472
|
|
|
467
|
-
function start(rank
|
|
473
|
+
function start(rank) {
|
|
468
474
|
const [command, ...argv] = writeJudge({ dir: judgeDir(rank), criterion: criteria[rank - 1], harness: harnesses[rank - 1], cwd, target, yolo });
|
|
469
475
|
const { cols, rows: lines } = size();
|
|
470
476
|
const term = new xterm.Terminal({ cols, rows: lines, allowProposedApi: true });
|
|
471
|
-
const serializer = new SerializeAddon();
|
|
472
|
-
term.loadAddon(serializer);
|
|
473
477
|
const proc = pty.spawn(command, argv, { name: 'xterm-256color', cols, rows: lines, cwd, env: process.env });
|
|
474
|
-
const j = { started: Date.now(), finished: undefined,
|
|
475
|
-
|
|
476
|
-
proc.onData((data) => term.write(data, () => {
|
|
477
|
-
if (ui.view !== rank || judges[rank - 1] !== j) return;
|
|
478
|
-
out.write(data);
|
|
479
|
-
lastData = Date.now();
|
|
480
|
-
stripSoon();
|
|
481
|
-
}));
|
|
478
|
+
const j = { started: Date.now(), finished: undefined, tokens: null, term, proc, cursorShown: true, exited: false, dirty: true, screen: null };
|
|
479
|
+
proc.onData((data) => term.write(data, () => { j.dirty = true; }));
|
|
482
480
|
proc.onExit(() => { j.exited = true; });
|
|
483
481
|
// The headless terminal answers the queries of the harness, such as the cursor position.
|
|
484
482
|
term.onData((data) => proc.write(data));
|
|
485
|
-
// The
|
|
483
|
+
// The judge shows or hides its cursor with DECTCEM (mode 25); the Agent pane follows it.
|
|
486
484
|
for (const final of ['h', 'l']) {
|
|
487
485
|
term.parser.registerCsiHandler({ prefix: '?', final }, (params) => {
|
|
488
|
-
|
|
486
|
+
if (params.includes(25)) j.cursorShown = final === 'h';
|
|
489
487
|
return false;
|
|
490
488
|
});
|
|
491
489
|
}
|
|
@@ -501,60 +499,56 @@ export function runTable({ panel, runDir, harness, cwd, target, yolo = false })
|
|
|
501
499
|
stop(judges[rank - 1]);
|
|
502
500
|
const file = join(judgeDir(rank), 'answer.json');
|
|
503
501
|
if (existsSync(file)) unlinkSync(file);
|
|
504
|
-
start(rank
|
|
505
|
-
|
|
506
|
-
}
|
|
507
|
-
|
|
508
|
-
// In a judge, the strip hides the cursor while it draws and then puts it back where the judge had it. A cursor
|
|
509
|
-
// past the last column waits for the next character to wrap; a move there would lose that, so the strip waits.
|
|
510
|
-
function drawStrip() {
|
|
511
|
-
const lines = renderStrip({ rows, view: ui.view, frame, width: out.columns, focus: ui.focus, elapsed: seconds(started), confirm: ui.confirm });
|
|
512
|
-
const top = out.rows - STRIP_ROWS + 1;
|
|
513
|
-
const strip = lines.map((l, i) => `\x1b[${top + i};1H${l}\x1b[K`).join('');
|
|
514
|
-
if (isPage(ui.view)) return void out.write(`${RESET}${strip}`);
|
|
515
|
-
const j = judges[ui.view - 1];
|
|
516
|
-
const { cursorX, cursorY } = j.term.buffer.active;
|
|
517
|
-
if (cursorX >= out.columns) return;
|
|
518
|
-
out.write(`\x1b[?25l${RESET}${strip}\x1b[${cursorY + 1};${cursorX + 1}H\x1b[?25${j.modes[25]}`);
|
|
519
|
-
}
|
|
520
|
-
|
|
521
|
-
// Waits for a pause in the output, so the strip does not land inside an escape sequence that a chunk split.
|
|
522
|
-
function stripSoon() {
|
|
523
|
-
clearTimeout(stripTimer);
|
|
524
|
-
stripTimer = setTimeout(drawStrip, 50);
|
|
525
|
-
}
|
|
526
|
-
|
|
527
|
-
function showJudge(rank) {
|
|
528
|
-
const j = judges[rank - 1];
|
|
529
|
-
j.term.write('', () => {
|
|
530
|
-
if (ui.view !== rank) return;
|
|
531
|
-
// The overview already holds the alternate screen, so a judge on its alternate screen gets a clear after
|
|
532
|
-
// the normal screen, which the serializer writes first. The scroll region keeps the judge above the strip.
|
|
533
|
-
const screen = j.serializer.serialize({ scrollback: 0 }).replace('\x1b[?1049h\x1b[H', '$&\x1b[2J');
|
|
534
|
-
out.write(`${RESET}\x1b[H\x1b[2J\x1b[1;${out.rows - STRIP_ROWS}r${screen}${Object.entries(j.modes).map(([mode, final]) => `\x1b[?${mode}${final}`).join('')}`);
|
|
535
|
-
drawStrip();
|
|
536
|
-
});
|
|
502
|
+
start(rank);
|
|
503
|
+
reportWritten = false;
|
|
537
504
|
}
|
|
538
505
|
|
|
539
506
|
function finish() {
|
|
540
507
|
const outcomes = rows.map((r) => (r.state === 'done' ? { answer: r.result } : { error: r.error }));
|
|
541
|
-
const report = buildReport({ panel, target, outcomes });
|
|
508
|
+
const report = buildReport({ sid, panel, target, outcomes });
|
|
542
509
|
writeFileSync(join(runDir, 'report.json'), `${JSON.stringify(report, null, 2)}\n`);
|
|
543
|
-
|
|
510
|
+
reportWritten = true;
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
/** The lines of the Agent pane of judge j, rendered again only when the judge wrote output or the size changed. */
|
|
514
|
+
function agentLines(j) {
|
|
515
|
+
const { cols, rows: lines } = size();
|
|
516
|
+
if (j.dirty || !j.screen || j.screen.cols !== cols || j.screen.rows !== lines) {
|
|
517
|
+
j.screen = { cols, rows: lines, lines: screenLines(j.term, cols, lines) };
|
|
518
|
+
j.dirty = false;
|
|
519
|
+
}
|
|
520
|
+
return j.screen.lines;
|
|
544
521
|
}
|
|
545
522
|
|
|
546
523
|
let frame = 0;
|
|
547
524
|
function draw() {
|
|
548
|
-
|
|
549
|
-
const
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
525
|
+
if (!rows.length) return;
|
|
526
|
+
const row = rows[ui.cursor];
|
|
527
|
+
const sub = subOf(ui.subs, row);
|
|
528
|
+
const j = judges[ui.cursor];
|
|
529
|
+
const lines = renderScreen({
|
|
530
|
+
title: `${panel.owner}/${panel.name} v${panel.version}`, target, harness, elapsed: seconds(started), rows, cursor: ui.cursor, sub,
|
|
531
|
+
agent: sub === 'agent' && j ? agentLines(j) : null, focus: ui.focus, scroll: ui.scroll, keys: hint({ ...ui, sub }), width: out.columns, height: out.rows, frame,
|
|
532
|
+
});
|
|
533
|
+
while (lines.length < out.rows) lines.push('');
|
|
534
|
+
let screen = `\x1b[?25l${lines.slice(0, out.rows).map((l, i) => `\x1b[${i + 1};1H${l}\x1b[K`).join('')}`;
|
|
535
|
+
// The modal sits over the overview; the next draw without it shows the overview again.
|
|
536
|
+
if (ui.modal) {
|
|
537
|
+
const { top, left, lines: box } = renderModal({ modal: ui.modal, current: harnesses[ui.modal.rank - 1], width: out.columns, height: out.rows });
|
|
538
|
+
screen += box.map((l, i) => `\x1b[${top + i + 1};${left + 1}H${l}`).join('');
|
|
539
|
+
}
|
|
540
|
+
// In the Agent pane with the focus, the real cursor sits where the judge's cursor is, when the judge shows it.
|
|
541
|
+
if (ui.focus === 'pane' && sub === 'agent' && j && j.cursorShown && !ui.modal) {
|
|
542
|
+
const { row: r, col } = paneOrigin(rows, out.columns, out.rows);
|
|
543
|
+
const { cursorX, cursorY } = j.term.buffer.active;
|
|
544
|
+
screen += `\x1b[${r + cursorY + 1};${col + Math.min(cursorX, size().cols - 1) + 1}H\x1b[?25h`;
|
|
545
|
+
}
|
|
546
|
+
out.write(screen);
|
|
553
547
|
}
|
|
554
548
|
|
|
555
549
|
function tick() {
|
|
556
550
|
rows = criteria.map((c, i) => ({ ...c, ...state(c, i) }));
|
|
557
|
-
if (!
|
|
551
|
+
if (!reportWritten) {
|
|
558
552
|
const live = rows.filter((r) => r.state === 'running').length;
|
|
559
553
|
const first = judges[0];
|
|
560
554
|
const hold = first && !first.exited && (Date.now() - first.started < FIRST_JUDGE_HEAD_START || rows[0].trust);
|
|
@@ -563,60 +557,57 @@ export function runTable({ panel, runDir, harness, cwd, target, yolo = false })
|
|
|
563
557
|
start(r.rank);
|
|
564
558
|
r.state = 'running';
|
|
565
559
|
}
|
|
566
|
-
if (rows.every((r) => r.state
|
|
560
|
+
if (rows.every((r) => completed(r.state))) finish();
|
|
567
561
|
}
|
|
568
|
-
|
|
562
|
+
draw();
|
|
569
563
|
}
|
|
570
564
|
|
|
571
565
|
return new Promise((done) => {
|
|
572
566
|
const timer = setInterval(tick, 1000);
|
|
573
|
-
const
|
|
574
|
-
frame += 1;
|
|
575
|
-
if (isPage(ui.view)) draw();
|
|
576
|
-
else if (Date.now() - lastData > 60) drawStrip();
|
|
577
|
-
}, 100);
|
|
567
|
+
const frames = setInterval(() => { frame += 1; draw(); }, 100);
|
|
578
568
|
out.write('\x1b[?1049h\x1b[?25l');
|
|
579
569
|
process.stdin.setRawMode(true);
|
|
580
|
-
|
|
581
|
-
const
|
|
582
|
-
const
|
|
583
|
-
const
|
|
584
|
-
|
|
585
|
-
const next = press(ui, key, { open: judges.flatMap((j, i) => (j ? [i + 1] : [])), running: rows.some((r) => r.state === 'running'), count: criteria.length, maxScroll: limit, page: Math.max(1, height - 7) });
|
|
586
|
-
ui = { view: next.view, confirm: next.confirm, focus: next.focus, cursor: next.cursor, scroll: next.scroll };
|
|
570
|
+
const onKey = (key) => {
|
|
571
|
+
const sub = rows.length ? subOf(ui.subs, rows[ui.cursor]) : 'verdict';
|
|
572
|
+
const limit = rows.length ? scrollMax({ rows, cursor: ui.cursor, sub, width: out.columns, height: out.rows }) : 0;
|
|
573
|
+
const next = press(ui, key, { open: judges.flatMap((j, i) => (j ? [i + 1] : [])), running: rows.some((r) => r.state === 'running'), count: criteria.length, states: rows.map((r) => r.state), harnesses, maxScroll: limit, page: Math.max(1, size().rows - 1) });
|
|
574
|
+
ui = { cursor: next.cursor, focus: next.focus, subs: next.subs, scroll: next.scroll, modal: next.modal, confirm: next.confirm };
|
|
587
575
|
if (next.send) {
|
|
588
|
-
judges[
|
|
576
|
+
judges[ui.cursor]?.proc.write(key);
|
|
589
577
|
} else if (next.quit) {
|
|
590
578
|
clearInterval(timer);
|
|
591
|
-
clearInterval(
|
|
579
|
+
clearInterval(frames);
|
|
592
580
|
for (const j of judges) stop(j);
|
|
593
|
-
out.write(`${
|
|
581
|
+
out.write(`${RESET}\x1b[?25h\x1b[?1049l`);
|
|
594
582
|
process.stdin.setRawMode(false);
|
|
595
583
|
process.stdin.pause();
|
|
596
584
|
done();
|
|
597
585
|
} else if (next.restart) {
|
|
586
|
+
delete ui.subs[next.restart];
|
|
587
|
+
ui.scroll = 0;
|
|
598
588
|
restart(next.restart);
|
|
599
589
|
tick();
|
|
600
|
-
} else if (next.
|
|
601
|
-
harnesses[next.
|
|
590
|
+
} else if (next.harness) {
|
|
591
|
+
harnesses[next.harness.rank - 1] = next.harness.value;
|
|
602
592
|
tick();
|
|
603
|
-
} else if (!isPage(ui.view) && ui.view === from) {
|
|
604
|
-
drawStrip();
|
|
605
593
|
} else {
|
|
606
|
-
|
|
607
|
-
if (!isPage(from)) out.write(TABLE_MODES);
|
|
608
|
-
if (isPage(ui.view)) draw();
|
|
609
|
-
else showJudge(ui.view);
|
|
594
|
+
draw();
|
|
610
595
|
}
|
|
596
|
+
};
|
|
597
|
+
process.stdin.on('data', (data) => {
|
|
598
|
+
const text = data.toString();
|
|
599
|
+
const sub = rows.length ? subOf(ui.subs, rows[ui.cursor]) : 'verdict';
|
|
600
|
+
if (ui.focus === 'pane' && sub === 'agent') onKey(text);
|
|
601
|
+
else for (const key of text.match(/\x1b\[[0-?]*[ -/]*[@-~]|\x1bO.|[\s\S]/g)) onKey(key);
|
|
611
602
|
});
|
|
612
603
|
out.on('resize', () => {
|
|
613
604
|
const { cols, rows: lines } = size();
|
|
614
605
|
for (const j of judges) {
|
|
615
606
|
if (j && !j.exited) j.proc.resize(cols, lines);
|
|
616
607
|
j?.term.resize(cols, lines);
|
|
608
|
+
if (j) j.dirty = true;
|
|
617
609
|
}
|
|
618
|
-
|
|
619
|
-
else showJudge(ui.view);
|
|
610
|
+
draw();
|
|
620
611
|
});
|
|
621
612
|
tick();
|
|
622
613
|
});
|