@xsft/panels 0.6.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -4
- package/bin/panels.js +13 -10
- package/lib/args.js +5 -1
- package/lib/judge.js +42 -57
- package/lib/report.js +16 -37
- package/lib/table.js +336 -108
- package/package.json +19 -6
- package/prompts/judge-system-prompt.txt +16 -36
- package/schema/report.json +11 -97
package/lib/table.js
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
|
-
// The terminal mode of `panels run`: the
|
|
1
|
+
// The terminal mode of `panels run`: the overview, and a view of each judge. Each judge runs its interactive
|
|
2
2
|
// harness in a pseudo-terminal, and a headless terminal keeps its screen for the switch to its view.
|
|
3
|
-
import { existsSync, readFileSync, writeFileSync } from 'node:fs';
|
|
4
|
-
import { join
|
|
3
|
+
import { existsSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
5
|
import pty from '@homebridge/node-pty-prebuilt-multiarch';
|
|
6
6
|
import { SerializeAddon } from '@xterm/addon-serialize';
|
|
7
7
|
import xterm from '@xterm/headless';
|
|
8
|
+
import figlet from 'figlet';
|
|
8
9
|
import { check, MINUTES_PER_CRITERION, parseAnswer, writeJudge } from './judge.js';
|
|
9
|
-
import { buildReport,
|
|
10
|
+
import { buildReport, VERSION } from './report.js';
|
|
10
11
|
|
|
11
12
|
const LIVE_JUDGES = 10;
|
|
12
13
|
// A harness asks this in a folder it has not seen. Only the first judge starts until its screen no longer asks,
|
|
@@ -17,26 +18,29 @@ const ORANGE = '\x1b[38;2;255;106;61m';
|
|
|
17
18
|
const GREY = '\x1b[90m';
|
|
18
19
|
const BOLD = '\x1b[1m';
|
|
19
20
|
const RESET = '\x1b[0m';
|
|
21
|
+
// The colour of a score, from the 16-colour palette: 4 green, 3 the default ink, 2 yellow, 1 red.
|
|
22
|
+
const SCORE_COLOUR = { 4: '\x1b[32m', 3: '', 2: '\x1b[33m', 1: '\x1b[31m' };
|
|
23
|
+
const LEVELS = { 1: 'Fails', 2: 'Falls short', 3: 'Almost met', 4: 'Met' };
|
|
20
24
|
// ctrl-q moves the focus between a judge and the tab strip. A terminal in the kitty keyboard mode sends it as CSI 113;5u.
|
|
21
25
|
const TOGGLE = ['\x11', '\x1b[113;5u'];
|
|
22
26
|
const RIGHT = ['\x1b[C', '\x1bOC'];
|
|
23
27
|
const LEFT = ['\x1b[D', '\x1bOD'];
|
|
28
|
+
const UP = ['\x1b[A', '\x1bOA'];
|
|
29
|
+
const DOWN = ['\x1b[B', '\x1bOB'];
|
|
24
30
|
// Enter, Esc, or ctrl-q give the focus back to the judge of the active tab.
|
|
25
31
|
const BACK = ['\r', '\x1b', ...TOGGLE];
|
|
26
|
-
// Turns off each mode that the screen of a judge can turn on, so the
|
|
32
|
+
// Turns off each mode that the screen of a judge can turn on, so the overview gets the keys and the screen as before.
|
|
27
33
|
const TABLE_MODES = '\x1b[?1049h\x1b[r\x1b[0m\x1b[4l\x1b[?6l\x1b[?7h\x1b[?1l\x1b>\x1b[?1000l\x1b[?1002l\x1b[?1003l\x1b[?1006l\x1b[?1004l\x1b[?2004l\x1b[<u\x1b[?25l\x1b[2J';
|
|
28
|
-
|
|
29
|
-
/**
|
|
30
|
-
* The lines of the table. rows hold { rank, name, state, result, trust }, and state is queued, running, done, or error.
|
|
31
|
-
* trust is true while the harness of a running judge asks whether to trust the folder.
|
|
32
|
-
* final is null until the report exists, then { status, path }. confirm is true while q waits for y.
|
|
33
|
-
*/
|
|
34
34
|
// The glimmer on the text of the running judges: grey text, and one soft wave three columns wide that moves one
|
|
35
35
|
// column per frame from the left edge of the screen to the right edge and starts over. A running label lights
|
|
36
36
|
// up only where the wave passes over its own columns, so every running label lights at the same columns.
|
|
37
37
|
const GLIMMER = { base: '\x1b[38;5;244m', mid: '\x1b[38;5;247m', high: '\x1b[38;5;250m' };
|
|
38
38
|
const GLIMMER_ORANGE = { base: ORANGE, mid: '\x1b[38;2;255;140;100m', high: '\x1b[38;2;255;175;145m' };
|
|
39
39
|
const WAVE = 3;
|
|
40
|
+
// The tab strip takes the last three lines of the terminal: a hairline, the tabs, then the status and the keys.
|
|
41
|
+
const STRIP_ROWS = 3;
|
|
42
|
+
// The overview shows two columns from this width up, and the list above the detail below it.
|
|
43
|
+
const TWO_COLUMNS = 100;
|
|
40
44
|
|
|
41
45
|
/**
|
|
42
46
|
* text with the glimmer at frame, where the text starts at the screen column start (0-based) on a screen width
|
|
@@ -49,62 +53,241 @@ export function glimmer(text, frame, start, width, colours = GLIMMER, prefix = '
|
|
|
49
53
|
return `${prefix}${d === 0 ? colours.high : d === 1 ? colours.mid : colours.base}${c}`;
|
|
50
54
|
}).join('') + RESET;
|
|
51
55
|
}
|
|
52
|
-
// The tab strip takes the last three lines of the terminal: a hairline, the tabs, then the status and the keys.
|
|
53
|
-
const STRIP_ROWS = 3;
|
|
54
56
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
57
|
+
/** m:ss for a number of seconds. */
|
|
58
|
+
export const clock = (seconds) => `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, '0')}`;
|
|
59
|
+
|
|
60
|
+
/** "12.4k" for 12400 tokens, "0.3k" for 300, "1.2M" from a million, and "" for none. */
|
|
61
|
+
export function tokensText(n) {
|
|
62
|
+
if (n === null || n === undefined) return '';
|
|
63
|
+
return n >= 1e6 ? `${(n / 1e6).toFixed(1)}M` : `${(n / 1000).toFixed(1)}k`;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* The largest token count on the screen of a judge, or null. Claude shows "↓ 354 tokens" and a context count
|
|
68
|
+
* such as "12532 tokens"; Codex shows its count with a k or M suffix. The largest number before "tokens" wins.
|
|
69
|
+
*/
|
|
70
|
+
export function tokensOnScreen(text) {
|
|
71
|
+
let best = null;
|
|
72
|
+
for (const m of text.matchAll(/(\d[\d,]*(?:\.\d+)?)\s*([kKmM])?\s*tokens/g)) {
|
|
73
|
+
const n = Number(m[1].replaceAll(',', '')) * (m[2] ? { k: 1e3, m: 1e6 }[m[2].toLowerCase()] : 1);
|
|
74
|
+
if (best === null || n > best) best = n;
|
|
75
|
+
}
|
|
76
|
+
return best;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
let artLines;
|
|
80
|
+
/** The word "judging" as big letters, in the ANSI Shadow font (Standard when that font is missing). Rendered once. */
|
|
81
|
+
export function art() {
|
|
82
|
+
if (!artLines) {
|
|
83
|
+
const font = figlet.fontsSync().includes('ANSI Shadow') ? 'ANSI Shadow' : 'Standard';
|
|
84
|
+
artLines = figlet.textSync('judging', { font }).split('\n').filter((l) => l.trim());
|
|
85
|
+
}
|
|
86
|
+
return artLines;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const plainLength = (s) => s.replace(/\x1b\[[0-9;]*m/g, '').length;
|
|
90
|
+
const padTo = (s, width) => s + ' '.repeat(Math.max(0, width - plainLength(s)));
|
|
91
|
+
|
|
92
|
+
/** The first width visible characters of a styled line, with every escape sequence kept, so the line never wraps. */
|
|
93
|
+
export function clip(line, width) {
|
|
94
|
+
let seen = 0;
|
|
95
|
+
let out = '';
|
|
96
|
+
for (const part of line.split(/(\x1b\[[0-9;]*m)/)) {
|
|
97
|
+
if (part.startsWith('\x1b')) { out += part; continue; }
|
|
98
|
+
const room = Math.max(0, width - seen);
|
|
99
|
+
out += [...part].slice(0, room).join('');
|
|
100
|
+
seen += Math.min([...part].length, room);
|
|
66
101
|
}
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
102
|
+
return out;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** The words of text in lines of at most width characters. */
|
|
106
|
+
export function wrap(text, width) {
|
|
107
|
+
const lines = [];
|
|
108
|
+
let line = '';
|
|
109
|
+
for (const word of text.split(/\s+/).filter(Boolean)) {
|
|
110
|
+
if (line && line.length + 1 + word.length > width) { lines.push(line); line = word; }
|
|
111
|
+
else line = line ? `${line} ${word}` : word;
|
|
71
112
|
}
|
|
72
|
-
if (
|
|
113
|
+
if (line) lines.push(line);
|
|
73
114
|
return lines;
|
|
74
115
|
}
|
|
75
116
|
|
|
117
|
+
/** The scores added as quarters: 1/4 + 1/4 + 2/4 is "1". No trailing zeros. */
|
|
118
|
+
const quarters = (sum) => String(Math.round((sum / 4) * 100) / 100);
|
|
119
|
+
|
|
76
120
|
/**
|
|
77
|
-
* The
|
|
78
|
-
*
|
|
79
|
-
*
|
|
80
|
-
* 4), ✗ for an error, · while queued, or ! while the harness asks whether to trust the folder. The names step
|
|
81
|
-
* down to fit the width: the whole name, the first word, or no name. focus is 'strip' (the active tab is bold
|
|
82
|
-
* orange, the arrows move it) or 'judge' (the active tab is bold ink, the keys go to the judge).
|
|
121
|
+
* The body of a criterion as plain lines for the detail pane: a heading becomes a grey upper-case label, a list
|
|
122
|
+
* item keeps its dash, a fenced block is indented four spaces as it is, a table line stays as it is, and the
|
|
123
|
+
* other text is wrapped to width with the emphasis, code, and link marks removed.
|
|
83
124
|
*/
|
|
84
|
-
export function
|
|
85
|
-
const
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
125
|
+
export function plainMarkdown(markdown, width) {
|
|
126
|
+
const inline = (t) => t.replace(/\*\*(.+?)\*\*/g, '$1').replace(/__(.+?)__/g, '$1').replace(/`([^`]+)`/g, '$1').replace(/\*(\S[^*]*?)\*/g, '$1').replace(/\[([^\]]+)\]\([^)]*\)/g, '$1');
|
|
127
|
+
const lines = [];
|
|
128
|
+
let paragraph = [];
|
|
129
|
+
let fence = false;
|
|
130
|
+
const flush = () => {
|
|
131
|
+
if (paragraph.length) lines.push(...wrap(inline(paragraph.join(' ')), width));
|
|
132
|
+
paragraph = [];
|
|
90
133
|
};
|
|
134
|
+
for (const raw of markdown.split('\n')) {
|
|
135
|
+
const line = raw.trimEnd();
|
|
136
|
+
if (/^\s*```/.test(line)) { flush(); fence = !fence; continue; }
|
|
137
|
+
if (fence) { lines.push(` ${line}`); continue; }
|
|
138
|
+
if (!line.trim()) { flush(); if (lines.at(-1) !== '') lines.push(''); continue; }
|
|
139
|
+
const heading = /^#{1,6}\s+(.*)$/.exec(line);
|
|
140
|
+
if (heading) { flush(); if (lines.length && lines.at(-1) !== '') lines.push(''); lines.push(`${GREY}${inline(heading[1]).toUpperCase()}${RESET}`); continue; }
|
|
141
|
+
const item = /^\s*(?:[-*+]|\d+\.)\s+(.*)$/.exec(line);
|
|
142
|
+
if (item) { flush(); const [first, ...rest] = wrap(inline(item[1]), width - 2); lines.push(`- ${first ?? ''}`, ...rest.map((l) => ` ${l}`)); continue; }
|
|
143
|
+
if (/^\s*\|/.test(line)) { flush(); lines.push(inline(line)); continue; }
|
|
144
|
+
paragraph.push(line.trim());
|
|
145
|
+
}
|
|
146
|
+
flush();
|
|
147
|
+
while (lines.at(-1) === '') lines.pop();
|
|
148
|
+
while (lines[0] === '') lines.shift();
|
|
149
|
+
return lines;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** The lines of the detail pane under its header: the verdict or the judging art, then the criterion text. */
|
|
153
|
+
function detailContent(row, columnWidth, start, width, frame) {
|
|
154
|
+
const lines = [''];
|
|
155
|
+
if (row.state === 'running' && !row.trust) {
|
|
156
|
+
// The judge works: the word judging in big letters with the wave over it, or in one line in a narrow pane.
|
|
157
|
+
const letters = art();
|
|
158
|
+
lines.push(...(letters[0].length <= columnWidth ? letters : ['judging']).map((l) => glimmer(l, frame, start, width)));
|
|
159
|
+
const since = row.restarted !== null ? `restarted with ${row.harness} · ` : '';
|
|
160
|
+
lines.push(`${GREY}${since}${clock(row.time ?? 0)} · ${row.rank} to watch${RESET}`);
|
|
161
|
+
} else {
|
|
162
|
+
const text = row.state === 'done' ? row.result.verdict
|
|
163
|
+
: row.state === 'error' ? row.error
|
|
164
|
+
: row.state === 'queued' ? 'The judge has not started.'
|
|
165
|
+
: `Press ${row.rank} and answer the trust prompt.`;
|
|
166
|
+
lines.push(...wrap(text, columnWidth));
|
|
167
|
+
}
|
|
168
|
+
if (row.description) lines.push('', ...wrap(row.description, columnWidth));
|
|
169
|
+
if (row.body) lines.push('', ...plainMarkdown(row.body, columnWidth));
|
|
170
|
+
return lines;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
const layout = (rows, width) => {
|
|
174
|
+
// The name column is as wide as the longest name, and at least as wide as the word total under it.
|
|
175
|
+
const leftWidth = Math.max(5, ...rows.map((r) => r.name.length)) + 27;
|
|
176
|
+
return width >= TWO_COLUMNS ? { leftWidth, columnWidth: width - leftWidth - 3, start: leftWidth + 3 } : { leftWidth, columnWidth: width - 2, start: 2 };
|
|
177
|
+
};
|
|
178
|
+
|
|
179
|
+
/** The number of lines of the detail pane that the scroll can hide: the content past the window of the pane. */
|
|
180
|
+
export function maxScroll({ rows, cursor, width, height }) {
|
|
181
|
+
const { columnWidth, start } = layout(rows, width);
|
|
182
|
+
const content = detailContent(rows[cursor], columnWidth, start, width, 0);
|
|
183
|
+
const body = height - 2;
|
|
184
|
+
const paneHeight = width >= TWO_COLUMNS ? body : body - Math.floor(body / 2) - 1;
|
|
185
|
+
// The pane holds the header, the content window, a blank line, and the three bottom lines.
|
|
186
|
+
const window = paneHeight - 5;
|
|
187
|
+
return window < 1 ? 0 : Math.max(0, content.length - window);
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* The lines of the overview. rows hold { rank, name, description, body, state, result, error, trust, time, tokens,
|
|
192
|
+
* harness, restarted }, and state is queued, running, done, or error. time is the seconds the judge has run,
|
|
193
|
+
* tokens the count it used (null for none), harness the one its next restart uses, restarted the seconds since
|
|
194
|
+
* a restart (null for none). cursor is the index of the row whose detail shows at the right; focus is 'list'
|
|
195
|
+
* or 'detail' (the detail pane shows an orange bar and its header turns orange); scroll hides that many lines
|
|
196
|
+
* of the detail content under its header. The header line holds the run: panel and target at the left, the
|
|
197
|
+
* version, the harness, and the elapsed time at the right edge. Two columns from TWO_COLUMNS wide, the list
|
|
198
|
+
* above the detail below that. height is the number of lines the overview may fill: the column divider runs
|
|
199
|
+
* the whole height, the rows and the detail header stay at the top, and the total row and the key and fact
|
|
200
|
+
* lines sit at the bottom. Without a height, or when the content is taller than the height, the lines follow
|
|
201
|
+
* each other and nothing scrolls.
|
|
202
|
+
*/
|
|
203
|
+
export function renderOverview({ title, target, harness, elapsed, rows, cursor, width, height, frame = 0, version = VERSION, focus = 'list', scroll = 0 }) {
|
|
204
|
+
const run = ` panels run ${title} ${target}`;
|
|
205
|
+
const meta = `${version} · ${harness} · ${clock(elapsed)} `;
|
|
206
|
+
const header = ` ${BOLD}panels run${RESET} ${title}${GREY} ${target}${' '.repeat(Math.max(2, width - run.length - meta.length))}${meta}${RESET}`;
|
|
207
|
+
const rule = `${GREY}${'─'.repeat(width)}${RESET}`;
|
|
208
|
+
const { leftWidth, columnWidth, start } = layout(rows, width);
|
|
209
|
+
const pad = leftWidth - 27;
|
|
210
|
+
const cells = (score, time, tokens) => ` ${score.padStart(3)} ${time.padStart(5)} ${tokens.padStart(6)}`;
|
|
211
|
+
const left = rows.map((r, i) => {
|
|
212
|
+
const current = i === cursor;
|
|
213
|
+
const mark = current ? `${ORANGE}▸${RESET} ` : ' ';
|
|
214
|
+
const rank = `${String(r.rank).padStart(2)} `;
|
|
215
|
+
const time = r.time === null ? '' : clock(r.time);
|
|
216
|
+
const tokens = tokensText(r.tokens);
|
|
217
|
+
if (r.state === 'running' && !r.trust) {
|
|
218
|
+
const name = glimmer(r.name.padEnd(pad), frame, 6, width, current ? GLIMMER_ORANGE : GLIMMER);
|
|
219
|
+
return `${mark}${current ? ORANGE : ''}${rank}${RESET}${name}${current ? ORANGE : GREY}${cells('·/4', time, tokens)}${RESET}`;
|
|
220
|
+
}
|
|
221
|
+
// A row without a score yet holds the place of the score with ·/4.
|
|
222
|
+
const score = r.state === 'done' ? `${r.result.score}/4` : r.state === 'error' ? '✗' : '·/4';
|
|
223
|
+
const colour = current ? ORANGE : r.state === 'done' ? SCORE_COLOUR[r.result.score] : r.state === 'error' ? SCORE_COLOUR[1] : GREY;
|
|
224
|
+
const name = current ? `${ORANGE}${r.name.padEnd(pad)}${RESET}` : r.state === 'queued' ? `${GREY}${r.name.padEnd(pad)}${RESET}` : r.name.padEnd(pad);
|
|
225
|
+
const rest = r.state === 'running' ? `${ORANGE}press ${r.rank} and answer the trust prompt${RESET}` : `${colour}${cells(score, time, tokens)}${RESET}`;
|
|
226
|
+
return `${mark}${current ? ORANGE : ''}${rank}${RESET}${name}${rest}`;
|
|
227
|
+
});
|
|
228
|
+
const done = rows.filter((r) => r.state === 'done');
|
|
229
|
+
const sum = done.length ? `${quarters(done.reduce((n, r) => n + r.result.score, 0))}/${rows.length}` : `·/${rows.length}`;
|
|
230
|
+
const total = cells(sum, clock(Math.max(0, ...rows.map((r) => r.time ?? 0))), tokensText(rows.some((r) => r.tokens !== null) ? rows.reduce((n, r) => n + (r.tokens ?? 0), 0) : null));
|
|
231
|
+
const totalLine = `${padTo(' total', leftWidth - total.length)}${done.length ? total : total.replace(sum, `${GREY}${sum}${RESET}`)}`;
|
|
232
|
+
const leftBottom = [`${GREY}${'─'.repeat(leftWidth)}${RESET}`, totalLine];
|
|
233
|
+
|
|
234
|
+
const row = rows[cursor];
|
|
235
|
+
const level = row.state === 'done' ? `${row.result.score}/4 ${LEVELS[row.result.score]}` : row.state === 'error' ? 'error' : row.state;
|
|
236
|
+
const name = `${row.rank} ${row.name.toUpperCase()}`;
|
|
237
|
+
const detailHeader = `${focus === 'detail' ? `${ORANGE}${BOLD}` : BOLD}${name}${RESET}${' '.repeat(Math.max(1, columnWidth - name.length - level.length))}${GREY}${level}${RESET}`;
|
|
238
|
+
const content = detailContent(row, columnWidth, start, width, frame);
|
|
239
|
+
const facts = [`judge ${row.rank}`, row.harness, ...(row.time === null ? [] : [clock(row.time)]), ...(row.state === 'done' || row.state === 'error' ? [`judges/${row.rank}/answer.json`] : [])];
|
|
240
|
+
const bottom = [`${ORANGE}r${RESET} restart ${ORANGE}h${RESET} harness ${row.harness} ${ORANGE}${row.rank}${RESET} watch`, '', `${GREY}${facts.join(' · ')}${RESET}`];
|
|
241
|
+
|
|
242
|
+
// top, gap blank lines, then bottom; with a height that holds them, the blank lines grow so bottom ends at the height.
|
|
243
|
+
const pane = (top, bottomLines, lines, gap) => {
|
|
244
|
+
const fill = lines === undefined || lines < top.length + gap + bottomLines.length ? gap : lines - top.length - bottomLines.length;
|
|
245
|
+
return [...top, ...Array(fill).fill(''), ...bottomLines];
|
|
246
|
+
};
|
|
247
|
+
// The detail pane: the header, a window of the content that the scroll moves, a blank line, then the bottom lines.
|
|
248
|
+
const detailPane = (lines) => {
|
|
249
|
+
const window = lines === undefined ? 0 : lines - 5;
|
|
250
|
+
if (window < 1) return pane([detailHeader, ...content], bottom, lines, 1);
|
|
251
|
+
const from = Math.min(scroll, Math.max(0, content.length - window));
|
|
252
|
+
return [detailHeader, ...content.slice(from, from + window), ...Array(Math.max(0, window - content.length)).fill(''), '', ...bottom];
|
|
253
|
+
};
|
|
254
|
+
const edge = focus === 'detail' ? `${ORANGE}▌${RESET}` : `${GREY}│${RESET}`;
|
|
255
|
+
|
|
256
|
+
const lines = [header, rule];
|
|
257
|
+
const body = height === undefined ? undefined : height - 2;
|
|
258
|
+
if (width >= TWO_COLUMNS) {
|
|
259
|
+
const leftLines = pane(left, leftBottom, body, 0);
|
|
260
|
+
const rightLines = detailPane(body);
|
|
261
|
+
for (let i = 0; i < Math.max(leftLines.length, rightLines.length); i++) lines.push(`${padTo(leftLines[i] ?? '', leftWidth)} ${edge} ${rightLines[i] ?? ''}`);
|
|
262
|
+
} else {
|
|
263
|
+
const half = body === undefined ? undefined : Math.floor(body / 2);
|
|
264
|
+
const list = pane(left, leftBottom, half, 0);
|
|
265
|
+
lines.push(...list, '', ...detailPane(body === undefined ? undefined : body - list.length - 1).map((l) => `${focus === 'detail' ? `${ORANGE}▌${RESET}` : ' '} ${l}`));
|
|
266
|
+
}
|
|
267
|
+
return lines.map((l) => clip(l, width));
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/**
|
|
271
|
+
* The three lines of the tab strip: a grey hairline, the tabs, then a grey status line with the progress and
|
|
272
|
+
* the elapsed time at the left and the keys of the focus at the right. The first tab is Overview, then the
|
|
273
|
+
* name of each criterion, with a grey │ between the tabs. A running tab glimmers. The names step down to the
|
|
274
|
+
* first word when they do not fit the width. focus is 'strip' (the active tab is bold orange, the arrows move
|
|
275
|
+
* it), 'detail' (the detail pane of the overview scrolls), or 'judge' (the active tab is bold ink, the keys go to
|
|
276
|
+
* the judge). confirm replaces the progress with the quit question.
|
|
277
|
+
*/
|
|
278
|
+
export function renderStrip({ rows, view, frame = 0, width, focus = 'strip', elapsed = 0, confirm = false }) {
|
|
91
279
|
const finished = rows.filter((r) => r.state === 'done' || r.state === 'error').length;
|
|
92
|
-
const
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
];
|
|
98
|
-
const plain = (t) => [t.rank, t.name, t.mark].filter(Boolean).join(' ');
|
|
99
|
-
const used = (list) => 1 + list.reduce((n, t) => n + plain(t).length, 0) + 2 * (list.length - 1);
|
|
100
|
-
const list = [0, 1, 2].map(tabs).find((l) => used(l) <= width) ?? tabs(2);
|
|
280
|
+
const keys = focus === 'judge' ? 'ctrl-q tabs'
|
|
281
|
+
: focus === 'detail' ? '↑ ↓ scroll · enter list · ← → tabs · q q quit'
|
|
282
|
+
: view === 0 ? '↑ ↓ criterion · r restart · h harness · ← → tabs · q q quit' : '← → tabs · enter back · q quit';
|
|
283
|
+
const names = (level) => ['Overview', ...rows.map((r) => (level === 0 ? r.name : r.name.split(' ')[0]))];
|
|
284
|
+
const used = (list) => 1 + list.reduce((n, t) => n + t.length, 0) + 3 * (list.length - 1);
|
|
285
|
+
const list = [0, 1].map(names).find((l) => used(l) <= width) ?? names(1);
|
|
101
286
|
let line = ' ';
|
|
102
287
|
let col = 1;
|
|
103
|
-
list.forEach((
|
|
104
|
-
if (i) { line +=
|
|
288
|
+
list.forEach((text, i) => {
|
|
289
|
+
if (i) { line += `${GREY} │ ${RESET}`; col += 3; }
|
|
105
290
|
const active = i === view;
|
|
106
|
-
const base = active ? (focus === 'strip' ? `${ORANGE}${BOLD}` : BOLD) : GREY;
|
|
107
|
-
const text = plain(t);
|
|
108
291
|
const row = rows[i - 1];
|
|
109
292
|
const start = col;
|
|
110
293
|
col += text.length;
|
|
@@ -112,49 +295,70 @@ export function renderStrip({ rows, view, final, frame = 0, width, focus = 'stri
|
|
|
112
295
|
line += glimmer(text, frame, start, width, active && focus === 'strip' ? GLIMMER_ORANGE : GLIMMER, active ? BOLD : '');
|
|
113
296
|
return;
|
|
114
297
|
}
|
|
115
|
-
|
|
116
|
-
if (t.rank) pieces.push(`${base}${t.rank}${RESET}`);
|
|
117
|
-
if (t.name) pieces.push(`${base}${t.name}${RESET}`);
|
|
118
|
-
if (t.mark) pieces.push(`${row.state === 'done' && row.result.score === 4 ? `${base}${BOLD}` : base}${t.mark}${RESET}`);
|
|
119
|
-
line += pieces.join(' ');
|
|
298
|
+
line += `${active ? (focus === 'strip' ? `${ORANGE}${BOLD}` : BOLD) : GREY}${text}${RESET}`;
|
|
120
299
|
});
|
|
121
|
-
const progress = `${finished} of ${rows.length} done · ${clock}`;
|
|
122
|
-
|
|
123
|
-
|
|
300
|
+
const progress = confirm ? 'judges still run. q again quits, another key stays' : `${finished} of ${rows.length} done · ${clock(elapsed)}`;
|
|
301
|
+
// A narrow terminal drops the keys before the progress, and then the end of the progress.
|
|
302
|
+
const gap = width - progress.length - keys.length - 2;
|
|
303
|
+
const status = gap >= 2 ? ` ${progress}${' '.repeat(gap)}${keys}` : ` ${progress}`;
|
|
304
|
+
return [`${GREY}${'─'.repeat(width)}${RESET}`, clip(line, width), `${GREY}${clip(status, width)}${RESET}`];
|
|
124
305
|
}
|
|
125
306
|
|
|
126
307
|
/**
|
|
127
|
-
* The next { view, confirm, focus } after one key, plus send: true when the key goes to the judge,
|
|
128
|
-
*
|
|
308
|
+
* The next { view, confirm, focus, cursor, scroll } after one key, plus send: true when the key goes to the judge,
|
|
309
|
+
* quit: true, restart: N to restart judge N, or flip: N to flip the harness of judge N. view is 0 for the
|
|
310
|
+
* overview, or N for judge N. open holds the ranks of the judges that have started, count the number of
|
|
311
|
+
* criteria, maxScroll the lines the detail pane can hide, page the lines one PgUp or PgDn moves. With the
|
|
129
312
|
* focus on a judge, every key but ctrl-q goes to the judge, and ctrl-q moves the focus to the strip. With the
|
|
130
|
-
* focus on the strip, the arrows move the active tab and switch the view, 1-9 jump to a started
|
|
131
|
-
*
|
|
132
|
-
*
|
|
313
|
+
* focus on the strip, the arrows left and right move the active tab and switch the view, 1-9 jump to a started
|
|
314
|
+
* judge, 0 to the overview, Enter, Esc, or ctrl-q give the focus back to the judge, and q quits (a second q
|
|
315
|
+
* confirms while a judge runs). On the overview, which takes no text, up and down move the cursor, r restarts
|
|
316
|
+
* the judge of the cursor row, h flips its harness, and Enter moves the focus into the detail pane: there up
|
|
317
|
+
* and down scroll one line, PgUp and PgDn a page, and Esc or Enter return the focus to the list.
|
|
133
318
|
*/
|
|
134
|
-
export function press({ view, confirm, focus }, key, { open, running }) {
|
|
135
|
-
const strip = (v) => ({ view: v, confirm: false, focus: 'strip' });
|
|
136
|
-
if (focus === 'judge') return TOGGLE.includes(key) ? strip(view) : { view, confirm, focus, send: true };
|
|
319
|
+
export function press({ view, confirm, focus, cursor, scroll = 0 }, key, { open, running, count, maxScroll: limit = 0, page = 10 }) {
|
|
320
|
+
const strip = (v) => ({ view: v, confirm: false, focus: 'strip', cursor, scroll });
|
|
321
|
+
if (focus === 'judge') return TOGGLE.includes(key) ? strip(view) : { view, confirm, focus, cursor, scroll, send: true };
|
|
137
322
|
if (confirm) return { ...strip(0), quit: key === 'q' };
|
|
138
|
-
if (key === 'q') return running ? { view: 0, confirm: true, focus: 'strip' } : { ...strip(0), quit: true };
|
|
323
|
+
if (key === 'q') return running ? { view: 0, confirm: true, focus: 'strip', cursor, scroll } : { ...strip(0), quit: true };
|
|
139
324
|
if (RIGHT.includes(key) || LEFT.includes(key)) {
|
|
140
325
|
const cycle = [0, ...open];
|
|
141
326
|
const step = RIGHT.includes(key) ? 1 : -1;
|
|
142
327
|
return strip(cycle[(cycle.indexOf(view) + step + cycle.length) % cycle.length]);
|
|
143
328
|
}
|
|
144
|
-
if (/^[1-9]$/.test(key)) return open.includes(Number(key)) ? strip(Number(key)) : strip(view);
|
|
329
|
+
if (/^[1-9]$/.test(key)) return open.includes(Number(key)) ? strip(Number(key)) : { ...strip(view), focus };
|
|
145
330
|
if (key === '0') return strip(0);
|
|
146
|
-
if (
|
|
331
|
+
if (view === 0) {
|
|
332
|
+
const detail = (s) => ({ view: 0, confirm: false, focus: 'detail', cursor, scroll: Math.max(0, Math.min(limit, s)) });
|
|
333
|
+
if (key === 'r') return open.includes(cursor + 1) ? { ...strip(0), focus, restart: cursor + 1 } : { ...strip(0), focus };
|
|
334
|
+
if (key === 'h') return { ...strip(0), focus, flip: cursor + 1 };
|
|
335
|
+
if (focus === 'detail') {
|
|
336
|
+
if (UP.includes(key)) return detail(scroll - 1);
|
|
337
|
+
if (DOWN.includes(key)) return detail(scroll + 1);
|
|
338
|
+
if (key === '\x1b[5~') return detail(scroll - page);
|
|
339
|
+
if (key === '\x1b[6~') return detail(scroll + page);
|
|
340
|
+
if (key === '\r' || key === '\x1b') return strip(0);
|
|
341
|
+
return detail(scroll);
|
|
342
|
+
}
|
|
343
|
+
if (UP.includes(key)) return { ...strip(0), cursor: Math.max(0, cursor - 1), scroll: 0 };
|
|
344
|
+
if (DOWN.includes(key)) return { ...strip(0), cursor: Math.min(count - 1, cursor + 1), scroll: 0 };
|
|
345
|
+
if (key === '\r') return detail(0);
|
|
346
|
+
return strip(0);
|
|
347
|
+
}
|
|
348
|
+
if (BACK.includes(key)) return { view, confirm: false, focus: 'judge', cursor, scroll };
|
|
147
349
|
return strip(view);
|
|
148
350
|
}
|
|
149
351
|
|
|
150
|
-
/** Runs the judges, shows the
|
|
151
|
-
export function runTable({ panel, runDir, harness, cwd, target, brief, sources }) {
|
|
352
|
+
/** Runs the judges, shows the overview, and writes the report. Resolves when the engineer quits. */
|
|
353
|
+
export function runTable({ panel, runDir, harness, cwd, target, brief, sources, yolo = false }) {
|
|
152
354
|
const out = process.stdout;
|
|
153
355
|
const criteria = panel.criteria.map((c, i) => ({ rank: i + 1, ...c }));
|
|
154
356
|
const judges = [];
|
|
357
|
+
// The harness of each judge: the run harness, until h flips it.
|
|
358
|
+
const harnesses = criteria.map(() => harness);
|
|
155
359
|
let rows = [];
|
|
156
360
|
let final = null;
|
|
157
|
-
let ui = { view: 0, confirm: false, focus: 'strip' };
|
|
361
|
+
let ui = { view: 0, confirm: false, focus: 'strip', cursor: 0, scroll: 0 };
|
|
158
362
|
let stripTimer;
|
|
159
363
|
let lastData = 0;
|
|
160
364
|
const started = Date.now();
|
|
@@ -162,37 +366,45 @@ export function runTable({ panel, runDir, harness, cwd, target, brief, sources }
|
|
|
162
366
|
const judgeDir = (rank) => join(runDir, 'judges', String(rank));
|
|
163
367
|
// A judge gets the lines above the tab strip.
|
|
164
368
|
const size = () => ({ cols: out.columns, rows: out.rows - STRIP_ROWS });
|
|
369
|
+
const seconds = (from, to = Date.now()) => Math.floor((to - from) / 1000);
|
|
370
|
+
|
|
371
|
+
function screenText(j) {
|
|
372
|
+
const buffer = j.term.buffer.active;
|
|
373
|
+
const lines = [];
|
|
374
|
+
for (let y = 0; y < buffer.length; y++) lines.push(buffer.getLine(y).translateToString(true));
|
|
375
|
+
return lines.join(' ');
|
|
376
|
+
}
|
|
165
377
|
|
|
166
378
|
function state(c, i) {
|
|
379
|
+
const j = judges[i];
|
|
380
|
+
const facts = { harness: harnesses[i], time: j ? seconds(j.started, j.finished) : null, tokens: j?.tokens ?? null, restarted: j?.restartedAt ? seconds(j.restartedAt) : null };
|
|
167
381
|
const file = join(judgeDir(c.rank), 'answer.json');
|
|
168
382
|
if (existsSync(file)) {
|
|
383
|
+
if (j && !j.finished) j.finished = Date.now();
|
|
384
|
+
facts.time = j ? seconds(j.started, j.finished) : null;
|
|
169
385
|
const answer = parseAnswer(readFileSync(file, 'utf8'));
|
|
170
|
-
const errors = check(answer,
|
|
171
|
-
return errors.length ? { state: 'error', error: `invalid answer: ${errors.join(' ')}` } : { state: 'done', result: answer
|
|
386
|
+
const errors = check(answer, c);
|
|
387
|
+
return errors.length ? { ...facts, state: 'error', error: `invalid answer: ${errors.join(' ')}` } : { ...facts, state: 'done', result: answer };
|
|
172
388
|
}
|
|
173
|
-
if (
|
|
174
|
-
if (Date.now() -
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
const buffer = j.term.buffer.active;
|
|
180
|
-
const lines = [];
|
|
181
|
-
for (let y = 0; y < buffer.length; y++) lines.push(buffer.getLine(y).translateToString(true));
|
|
182
|
-
return TRUST_PROMPT.test(lines.join(' '));
|
|
389
|
+
if (j === undefined) return { ...facts, state: 'queued' };
|
|
390
|
+
if (Date.now() - j.started > MINUTES_PER_CRITERION * 60_000) return { ...facts, state: 'error', error: `no answer.json after ${MINUTES_PER_CRITERION} minutes` };
|
|
391
|
+
const screen = screenText(j);
|
|
392
|
+
const tokens = tokensOnScreen(screen);
|
|
393
|
+
if (tokens !== null) j.tokens = tokens;
|
|
394
|
+
return { ...facts, tokens: j.tokens ?? null, state: 'running', trust: TRUST_PROMPT.test(screen) };
|
|
183
395
|
}
|
|
184
396
|
|
|
185
|
-
function start(rank) {
|
|
186
|
-
const [command, ...argv] = writeJudge({ dir: judgeDir(rank),
|
|
397
|
+
function start(rank, restartedAt = null) {
|
|
398
|
+
const [command, ...argv] = writeJudge({ dir: judgeDir(rank), criterion: criteria[rank - 1], harness: harnesses[rank - 1], cwd, target, brief, sources, yolo });
|
|
187
399
|
const { cols, rows: lines } = size();
|
|
188
400
|
const term = new xterm.Terminal({ cols, rows: lines, allowProposedApi: true });
|
|
189
401
|
const serializer = new SerializeAddon();
|
|
190
402
|
term.loadAddon(serializer);
|
|
191
403
|
const proc = pty.spawn(command, argv, { name: 'xterm-256color', cols, rows: lines, cwd, env: process.env });
|
|
192
|
-
const j = { started: Date.now(), term, serializer, proc, modes: { 25: 'h', 1006: 'l' }, exited: false };
|
|
404
|
+
const j = { started: Date.now(), finished: undefined, restartedAt, tokens: null, term, serializer, proc, modes: { 25: 'h', 1006: 'l' }, exited: false };
|
|
193
405
|
// The write callback keeps the order of the live output and of the screen that showJudge writes.
|
|
194
406
|
proc.onData((data) => term.write(data, () => {
|
|
195
|
-
if (ui.view !== rank) return;
|
|
407
|
+
if (ui.view !== rank || judges[rank - 1] !== j) return;
|
|
196
408
|
out.write(data);
|
|
197
409
|
lastData = Date.now();
|
|
198
410
|
stripSoon();
|
|
@@ -210,10 +422,23 @@ export function runTable({ panel, runDir, harness, cwd, target, brief, sources }
|
|
|
210
422
|
judges[rank - 1] = j;
|
|
211
423
|
}
|
|
212
424
|
|
|
425
|
+
// The harness on PATH can be a shell script that runs the real binary as a child and waits for it, so
|
|
426
|
+
// SIGHUP goes to the process group of the judge. The pty makes each judge the leader of its own group.
|
|
427
|
+
const stop = (j) => { if (j && !j.exited) process.kill(-j.proc.pid, 'SIGHUP'); };
|
|
428
|
+
|
|
429
|
+
/** Stops judge rank, drops its answer, and starts it again with its harness. The report is written again later. */
|
|
430
|
+
function restart(rank) {
|
|
431
|
+
stop(judges[rank - 1]);
|
|
432
|
+
const file = join(judgeDir(rank), 'answer.json');
|
|
433
|
+
if (existsSync(file)) unlinkSync(file);
|
|
434
|
+
start(rank, Date.now());
|
|
435
|
+
final = null;
|
|
436
|
+
}
|
|
437
|
+
|
|
213
438
|
// In a judge, the strip hides the cursor while it draws and then puts it back where the judge had it. A cursor
|
|
214
439
|
// past the last column waits for the next character to wrap; a move there would lose that, so the strip waits.
|
|
215
440
|
function drawStrip() {
|
|
216
|
-
const lines = renderStrip({ rows, view: ui.view,
|
|
441
|
+
const lines = renderStrip({ rows, view: ui.view, frame, width: out.columns, focus: ui.focus, elapsed: seconds(started), confirm: ui.confirm });
|
|
217
442
|
const top = out.rows - STRIP_ROWS + 1;
|
|
218
443
|
const strip = lines.map((l, i) => `\x1b[${top + i};1H${l}\x1b[K`).join('');
|
|
219
444
|
if (!ui.view) return void out.write(`${RESET}${strip}`);
|
|
@@ -233,7 +458,7 @@ export function runTable({ panel, runDir, harness, cwd, target, brief, sources }
|
|
|
233
458
|
const j = judges[rank - 1];
|
|
234
459
|
j.term.write('', () => {
|
|
235
460
|
if (ui.view !== rank) return;
|
|
236
|
-
// The
|
|
461
|
+
// The overview already holds the alternate screen, so a judge on its alternate screen gets a clear after
|
|
237
462
|
// the normal screen, which the serializer writes first. The scroll region keeps the judge above the strip.
|
|
238
463
|
const screen = j.serializer.serialize({ scrollback: 0 }).replace('\x1b[?1049h\x1b[H', '$&\x1b[2J');
|
|
239
464
|
out.write(`${RESET}\x1b[H\x1b[2J\x1b[1;${out.rows - STRIP_ROWS}r${screen}${Object.entries(j.modes).map(([mode, final]) => `\x1b[?${mode}${final}`).join('')}`);
|
|
@@ -242,25 +467,23 @@ export function runTable({ panel, runDir, harness, cwd, target, brief, sources }
|
|
|
242
467
|
}
|
|
243
468
|
|
|
244
469
|
function finish() {
|
|
245
|
-
const outcomes = rows.map((r) => (r.state === 'done' ? {
|
|
246
|
-
const report = buildReport({ panel, target,
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`);
|
|
250
|
-
final = { status: report.status, path: relative(cwd, reportPath) };
|
|
470
|
+
const outcomes = rows.map((r) => (r.state === 'done' ? { answer: r.result } : { error: r.error }));
|
|
471
|
+
const report = buildReport({ panel, target, outcomes });
|
|
472
|
+
writeFileSync(join(runDir, 'report.json'), `${JSON.stringify(report, null, 2)}\n`);
|
|
473
|
+
final = { status: report.status };
|
|
251
474
|
}
|
|
252
475
|
|
|
253
476
|
let frame = 0;
|
|
254
477
|
function draw() {
|
|
255
|
-
const lines =
|
|
478
|
+
const lines = renderOverview({ title: `${panel.owner}/${panel.name} v${panel.version}`, target, harness, elapsed: seconds(started), rows, cursor: ui.cursor, width: out.columns, height: out.rows - STRIP_ROWS, frame, focus: ui.focus === 'detail' ? 'detail' : 'list', scroll: ui.scroll });
|
|
256
479
|
while (lines.length < out.rows - STRIP_ROWS) lines.push('');
|
|
257
480
|
out.write(lines.slice(0, out.rows - STRIP_ROWS).map((l, i) => `\x1b[${i + 1};1H${l}\x1b[K`).join(''));
|
|
258
481
|
drawStrip();
|
|
259
482
|
}
|
|
260
483
|
|
|
261
484
|
function tick() {
|
|
485
|
+
rows = criteria.map((c, i) => ({ ...c, ...state(c, i) }));
|
|
262
486
|
if (!final) {
|
|
263
|
-
rows = criteria.map((c, i) => ({ ...c, ...state(c, i) }));
|
|
264
487
|
const live = rows.filter((r) => r.state === 'running').length;
|
|
265
488
|
const first = judges[0];
|
|
266
489
|
const hold = first && !first.exited && (Date.now() - first.started < FIRST_JUDGE_HEAD_START || rows[0].trust);
|
|
@@ -278,7 +501,6 @@ export function runTable({ panel, runDir, harness, cwd, target, brief, sources }
|
|
|
278
501
|
const timer = setInterval(tick, 1000);
|
|
279
502
|
const spinner = setInterval(() => {
|
|
280
503
|
frame += 1;
|
|
281
|
-
if (final) return;
|
|
282
504
|
if (!ui.view) draw();
|
|
283
505
|
else if (Date.now() - lastData > 60) drawStrip();
|
|
284
506
|
}, 100);
|
|
@@ -287,20 +509,26 @@ export function runTable({ panel, runDir, harness, cwd, target, brief, sources }
|
|
|
287
509
|
process.stdin.on('data', (data) => {
|
|
288
510
|
const key = data.toString();
|
|
289
511
|
const from = ui.view;
|
|
290
|
-
const
|
|
291
|
-
|
|
512
|
+
const height = out.rows - STRIP_ROWS;
|
|
513
|
+
const limit = rows.length ? maxScroll({ rows, cursor: ui.cursor, width: out.columns, height }) : 0;
|
|
514
|
+
const next = press(ui, key, { open: judges.flatMap((j, i) => (j ? [i + 1] : [])), running: rows.some((r) => r.state === 'running'), count: criteria.length, maxScroll: limit, page: Math.max(1, height - 7) });
|
|
515
|
+
ui = { view: next.view, confirm: next.confirm, focus: next.focus, cursor: next.cursor, scroll: next.scroll };
|
|
292
516
|
if (next.send) {
|
|
293
517
|
judges[from - 1].proc.write(key);
|
|
294
518
|
} else if (next.quit) {
|
|
295
519
|
clearInterval(timer);
|
|
296
520
|
clearInterval(spinner);
|
|
297
|
-
|
|
298
|
-
// SIGHUP goes to the process group of the judge. The pty makes each judge the leader of its own group.
|
|
299
|
-
for (const j of judges) if (j && !j.exited) process.kill(-j.proc.pid, 'SIGHUP');
|
|
521
|
+
for (const j of judges) stop(j);
|
|
300
522
|
out.write(`${TABLE_MODES}\x1b[?25h\x1b[?1049l`);
|
|
301
523
|
process.stdin.setRawMode(false);
|
|
302
524
|
process.stdin.pause();
|
|
303
525
|
done();
|
|
526
|
+
} else if (next.restart) {
|
|
527
|
+
restart(next.restart);
|
|
528
|
+
tick();
|
|
529
|
+
} else if (next.flip) {
|
|
530
|
+
harnesses[next.flip - 1] = harnesses[next.flip - 1] === 'claude' ? 'codex' : 'claude';
|
|
531
|
+
tick();
|
|
304
532
|
} else if (ui.view && ui.view === from) {
|
|
305
533
|
drawStrip();
|
|
306
534
|
} else {
|