@xsft/panels 0.1.0 → 0.3.1

package/README.md

@@ -1,6 +1,6 @@
 # @xsft/panels
 
-`panels run` runs a panel from cross.software/panels against a target. Each judge is a `claude` or `codex` session that scores one or more criteria of the panel. You need Node 20 or newer and one of the two harnesses.
+`panels run` runs a panel from cross.software/panels against a target. Each judge is a `claude` or `codex` session that scores one criterion of the panel. You need Node 20 or newer and one of the two harnesses.
 
 ```text
 panels run <panel> [target...] [flags]
@@ -8,7 +8,6 @@ panels run <panel> [target...] [flags]
 <panel>           owner/name, or owner/name/vN to pin version N
 [target...]       free words, joined with single spaces into one target
 
---size <n>        number of judges                   default: the number of criteria
 --harness <name>  claude | codex                     default: claude if installed, else codex
 --brief <file>    the brief                          default: none
 --sources <path>  a doc folder, a repository, or an app URL; repeat the flag for each
@@ -26,7 +25,13 @@ npx @xsft/panels run xsft/marketing-site https://example.com --brief docs/brief.
 npx @xsft/panels run xsft/marketing-site/v6 . at http://localhost:4321 --sources docs --harness codex
 ```
 
-Run the command in the folder of the product. Each judge starts in that folder, so `.`, a branch, and relative paths resolve there. The CLI does not test, fetch, or change the target. Four judges run at a time. A judge gets 20 minutes for each of its criteria.
+Run the command in the folder of the product. Each judge starts in that folder, so `.`, a branch, and relative paths resolve there. The CLI does not test, fetch, or change the target. Four judges run at a time. A judge gets 20 minutes.
+
+## Terminal mode
+
+On a terminal, `panels run` shows the score table, and each judge runs in its interactive harness in a pseudo-terminal inside the command. Each judge writes `judges/N/answer.json` and stays open. The table writes the report when each judge has an answer or an error, and the command exits from `report.json` when you quit. Without a terminal, the judges run in the background (`claude -p`, `codex exec`).
+
+Keys in the table: `1` to `9` open the screen of that judge, and `q` quits and asks `y` while a judge still runs. In a judge, each key goes to the judge, and `ctrl-]` returns to the table.
 
 ## Run folder
 
@@ -34,7 +39,7 @@ Each run writes to `.panels/<owner>-<name>/<UTC time>/`:
 
 - `panel.json`: the panel as the api gave it.
 - `report.md` and `report.json`: the scores, the evidence, the fixes, and the errors. `schema/report.json` is the schema of `report.json`.
-- `judges/<k>/`: the prompt, the output schema, and the raw answer of judge k.
+- `judges/<k>/`: the prompt, the output schema, and the raw answer of judge k. In terminal mode, `answer.json` takes the place of the raw answer.
 
 The last line on stdout is the path of `report.json`.
 
@@ -48,4 +53,4 @@ The last line on stdout is the path of `report.json`.
 - `--model`: each harness uses its default.
 - A panel URL as `<panel>`, a `PANELS_API` override, and a panel cache.
 - A login: an owner cannot run an unpublished panel.
-- A limit on `--size`, cost totals, and `cap` and `gap` fields.
+- Cost totals, and `cap` and `gap` fields.

package/bin/panels.js

@@ -1,11 +1,11 @@
 #!/usr/bin/env node
-import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
 import { delimiter, join } from 'node:path';
 import { parse, SYNOPSIS } from '../lib/args.js';
 import { runJudge } from '../lib/judge.js';
 import { fetchPanel } from '../lib/panel.js';
 import { buildReport, renderMarkdown } from '../lib/report.js';
-import { split } from '../lib/split.js';
+import { runTable } from '../lib/table.js';
 
 // More parallel sessions on one account get 429 answers from the model api.
 const PARALLEL_JUDGES = 4;
@@ -25,6 +25,17 @@ async function pool(items, size, fn) {
   return out;
 }
 
+const EXIT = { pass: 0, fail: 1, error: 2 };
+
+/** Runs the judges in the score table, and exits from report.json when the engineer quits. */
+async function runOnTerminal({ panel, runDir, harness, cwd, args }) {
+  await runTable({ panel, runDir, harness, cwd, target: args.target, brief: args.brief, sources: args.sources });
+  const reportPath = join(runDir, 'report.json');
+  if (!existsSync(reportPath)) return 2;
+  console.log(reportPath);
+  return EXIT[JSON.parse(readFileSync(reportPath, 'utf8')).status];
+}
+
 async function main() {
   const args = parse(process.argv.slice(2));
   if (args.help) {
@@ -51,10 +62,12 @@ async function main() {
   }
   const { panel } = fetched;
   const cwd = process.cwd();
-  const judges = split(panel.criteria.length, args.size ?? panel.criteria.length);
-  const runDir = join(cwd, '.panels', `${panel.owner}-${panel.name}`, new Date().toISOString().replaceAll(':', '-'));
+  const judges = panel.criteria.map((_, i) => [i]);
+  const time = new Date().toISOString().replaceAll(':', '-');
+  const runDir = join(cwd, '.panels', `${panel.owner}-${panel.name}`, time);
   mkdirSync(runDir, { recursive: true });
   writeFileSync(join(runDir, 'panel.json'), JSON.stringify(panel, null, 2));
+  if (process.stdout.isTTY && process.stdin.isTTY) return runOnTerminal({ panel, runDir, harness, cwd, args });
   const outcomes = await pool(judges, PARALLEL_JUDGES, (indexes, j) => runJudge({
     dir: join(runDir, 'judges', String(j + 1)),
     criteria: indexes.map((i) => ({ rank: i + 1, ...panel.criteria[i] })),
@@ -71,7 +84,7 @@ async function main() {
   writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`);
   console.log(markdown);
   console.log(reportPath);
-  return { pass: 0, fail: 1, error: 2 }[report.status];
+  return EXIT[report.status];
 }
 
 // Node exits 1 on a crash, and 1 means "fail" to an agent loop, so a crash prints its stack and exits 2.

package/lib/args.js

@@ -5,7 +5,6 @@ export const SYNOPSIS = `panels run <panel> [target...] [flags]
 <panel>           owner/name, or owner/name/vN to pin version N
 [target...]       free words, joined with single spaces into one target
 
---size <n>        number of judges                   default: the number of criteria
 --harness <name>  claude | codex                     default: claude if installed, else codex
 --brief <file>    the brief                          default: none
 --sources <path>  a doc folder, a repository, or an app URL; repeat the flag for each
@@ -16,7 +15,6 @@ exit 1  fail: every judge answered, and one or more criteria scored below 4
 exit 2  error: bad flags, no target, panel not found, harness missing, or a judge failed`;
 
 const options = {
-  size: { type: 'string' },
   harness: { type: 'string' },
   brief: { type: 'string' },
   sources: { type: 'string', multiple: true },
@@ -39,12 +37,10 @@ export function parse(argv) {
   const match = PANEL.exec(panel);
   if (!match) return { error: `bad panel ${panel}: use owner/name or owner/name/vN` };
   if (words.length === 0) return { error: 'no target: type what the panel reviews after the panel name, such as a URL, a path, or a branch' };
-  if (values.size !== undefined && !/^[1-9]\d*$/.test(values.size)) return { error: `--size needs a whole number of 1 or more, not ${values.size}` };
   if (values.harness !== undefined && !['claude', 'codex'].includes(values.harness)) return { error: `--harness needs claude or codex, not ${values.harness}` };
   return {
     panel: { owner: match[1], name: match[2], version: match[3] && Number(match[3]) },
     target: words.join(' '),
-    size: values.size && Number(values.size),
     harness: values.harness,
     brief: values.brief,
     sources: values.sources ?? [],

package/lib/judge.js

@@ -4,7 +4,7 @@ import { dirname, join, relative, resolve, sep } from 'node:path';
 
 const { judgeResult } = JSON.parse(readFileSync(new URL('../schema/report.json', import.meta.url), 'utf8')).$defs;
 const SYSTEM_PROMPT = readFileSync(new URL('../prompts/judge-system-prompt.txt', import.meta.url), 'utf8');
-const MINUTES_PER_CRITERION = 20;
+export const MINUTES_PER_CRITERION = 20;
 const TOOLS = 'Read,Glob,Grep,Bash,WebFetch,WebSearch';
 // With --permission-mode dontAsk, Claude refuses every tool call that this list does not allow, so Bash runs git reads only.
 const ALLOWED_TOOLS = 'Read,Glob,Grep,WebFetch,WebSearch,Bash(git log:*),Bash(git show:*),Bash(git diff:*),Bash(git status:*),Bash(git ls-files:*),Bash(git rev-parse:*),Bash(git branch --list:*)';
@@ -112,7 +112,7 @@ async function askCodex(j, prompt, thread) {
   return { answer: parseAnswer(readFileSync(raw, 'utf8')), session: thread ?? JSON.parse(started).thread_id };
 }
 
-function parseAnswer(text) {
+export function parseAnswer(text) {
   try {
     return JSON.parse(text);
   } catch {
@@ -143,3 +143,29 @@ export async function runJudge({ dir, criteria, harness, cwd, target, brief, sou
   }
   return errors.length ? { error: `invalid answer after one re-ask: ${errors.join(' ')}` } : { results: reply.answer.results };
 }
+
+const TOML_ESCAPES = { '"': '\\"', '\\': '\\\\', '\n': '\\n' };
+
+/** A TOML basic string, for the value of a codex -c flag. */
+export const tomlString = (s) => `"${s.replace(/["\\\x00-\x1f\x7f]/g, (c) => TOML_ESCAPES[c] ?? `\\u${c.charCodeAt(0).toString(16).padStart(4, '0')}`)}"`;
+
+/**
+ * Writes prompt.md and schema.json of one judge of the terminal mode, and returns the command and the
+ * arguments that start the judge in its interactive harness.
+ */
+export function writeJudge({ dir, criteria, harness, cwd, target, brief, sources }) {
+  mkdirSync(dir, { recursive: true });
+  const closing = `The engineer ran the command in ${cwd}. As your last step, write your answer JSON, which matches the schema in ${join(dir, 'schema.json')}, to ${join(dir, 'answer.json')}. Write that file yourself with a tool, and do not start sub-agents: the JSON in your reply does not count. Then stop. These two files are the only exceptions to the rules on writing files and on the .panels folder.\n`;
+  const prompt = `${userMessage(criteria, { target, brief, sources })}\n${closing}`;
+  writeFileSync(join(dir, 'prompt.md'), prompt);
+  writeFileSync(join(dir, 'schema.json'), JSON.stringify(judgeSchema(criteria), null, 2));
+  if (harness === 'codex') {
+    // Codex ranks the developer instructions above the prompt, and the system prompt says to return the JSON and to
+    // write no files. Without the closing in the developer instructions, the judge prints the JSON and writes no answer.json.
+    return ['codex', '-C', dir, '-s', 'workspace-write', '--disable', 'multi_agent', '--disable', 'apps', '-c', `developer_instructions=${tomlString(`${SYSTEM_PROMPT}\n${closing}`)}`, prompt];
+  }
+  // Claude matches file paths only in Edit rules, which cover Write too; // starts an absolute path.
+  const allowed = `${ALLOWED_TOOLS},Edit(/${join(dir, 'answer.json')})`;
+  const add = addDirs(cwd, brief, sources).flatMap((d) => ['--add-dir', d]);
+  return ['claude', '--safe-mode', '--system-prompt', SYSTEM_PROMPT, '--tools', `${TOOLS},Write`, ...add, '--allowedTools', allowed, '--permission-mode', 'dontAsk', prompt];
+}

package/lib/table.js

@@ -0,0 +1,200 @@
+// The terminal mode of `panels run`: the score table, and a view of each judge. Each judge runs its interactive
+// harness in a pseudo-terminal, and a headless terminal keeps its screen for the switch to its view.
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { join, relative } from 'node:path';
+import pty from '@homebridge/node-pty-prebuilt-multiarch';
+import { SerializeAddon } from '@xterm/addon-serialize';
+import xterm from '@xterm/headless';
+import { check, MINUTES_PER_CRITERION, parseAnswer, writeJudge } from './judge.js';
+import { buildReport, renderMarkdown } from './report.js';
+
+// More parallel sessions on one account get 429 answers from the model api.
+const LIVE_JUDGES = 4;
+const ORANGE = '\x1b[38;2;255;106;61m';
+const GREY = '\x1b[90m';
+const BOLD = '\x1b[1m';
+const RESET = '\x1b[0m';
+// ctrl-] returns from a judge to the table. A terminal in the kitty keyboard mode sends it as CSI 93;5u.
+const BACK = ['\x1d', '\x1b[93;5u'];
+// Turns off each mode that the screen of a judge can turn on, so the table gets the keys and the screen as before.
+const TABLE_MODES = '\x1b[?1049h\x1b[r\x1b[0m\x1b[4l\x1b[?6l\x1b[?7h\x1b[?1l\x1b>\x1b[?1000l\x1b[?1002l\x1b[?1003l\x1b[?1006l\x1b[?1004l\x1b[?2004l\x1b[<u\x1b[?25l\x1b[2J';
+
+/**
+ * The lines of the table. rows hold { rank, name, state, result }, and state is queued, running, done, or error.
+ * final is null until the report exists, then { status, path }. confirm is true while q waits for y.
+ */
+export function render({ title, rows, final, confirm, width }) {
+  const rule = `${GREY}${'─'.repeat(width)}${RESET}`;
+  const pad = Math.max(...rows.map((r) => r.name.length)) + 2;
+  const lines = [` ${BOLD}panels run${RESET}  ${title}`, rule];
+  for (const r of rows) {
+    const head = `${String(r.rank).padStart(3)}  ${r.name.padEnd(pad)}`;
+    if (r.state === 'running') lines.push(`${ORANGE}${head}…${RESET}`);
+    else if (r.state === 'queued') lines.push(`${head}${GREY}queued${RESET}`);
+    else if (r.state === 'error') lines.push(`${head}error`);
+    else lines.push(`${head}${r.result.score} ${GREY}b${r.result.band}${RESET}`);
+  }
+  lines.push(rule);
+  if (final) {
+    const at4 = rows.filter((r) => r.state === 'done' && r.result.score === 4).length;
+    lines.push(` ${BOLD}${final.status.toUpperCase()}${RESET}  ${at4} of ${rows.length} at 4`, ` report ${final.path}`);
+  }
+  lines.push(confirm ? ' judges still run. y quits, another key stays' : `${GREY} 1-${Math.min(rows.length, 9)} open judge   q quit${RESET}`);
+  return lines;
+}
+
+/**
+ * The next { view, confirm } after one key, plus send: true when the key goes to the judge, or quit: true.
+ * view is 0 for the table, or N for judge N. open holds the ranks of the judges that have started.
+ */
+export function press({ view, confirm }, key, { open, running }) {
+  if (view) return BACK.includes(key) ? { view: 0, confirm: false } : { view, confirm, send: true };
+  if (confirm) return { view: 0, confirm: false, quit: key === 'y' };
+  if (key === 'q') return running ? { view: 0, confirm: true } : { view: 0, confirm: false, quit: true };
+  if (/^[1-9]$/.test(key) && open.includes(Number(key))) return { view: Number(key), confirm: false };
+  return { view: 0, confirm: false };
+}
+
+/** Runs the judges, shows the table, and writes the report. Resolves when the engineer quits. */
+export function runTable({ panel, runDir, harness, cwd, target, brief, sources }) {
+  const out = process.stdout;
+  const criteria = panel.criteria.map((c, i) => ({ rank: i + 1, ...c }));
+  const judges = [];
+  let rows = [];
+  let final = null;
+  let ui = { view: 0, confirm: false };
+  let footerTimer;
+
+  const judgeDir = (rank) => join(runDir, 'judges', String(rank));
+  // A judge gets one line less than the terminal. The last line holds the footer.
+  const size = () => ({ cols: out.columns, rows: out.rows - 1 });
+
+  function state(c, i) {
+    const file = join(judgeDir(c.rank), 'answer.json');
+    if (existsSync(file)) {
+      const answer = parseAnswer(readFileSync(file, 'utf8'));
+      const errors = check(answer, [c]);
+      return errors.length ? { state: 'error', error: `invalid answer: ${errors.join(' ')}` } : { state: 'done', result: answer.results[0] };
+    }
+    if (judges[i] === undefined) return { state: 'queued' };
+    if (Date.now() - judges[i].started > MINUTES_PER_CRITERION * 60_000) return { state: 'error', error: `no answer.json after ${MINUTES_PER_CRITERION} minutes` };
+    return { state: 'running' };
+  }
+
+  function start(rank) {
+    const [command, ...argv] = writeJudge({ dir: judgeDir(rank), criteria: [criteria[rank - 1]], harness, cwd, target, brief, sources });
+    const { cols, rows: lines } = size();
+    const term = new xterm.Terminal({ cols, rows: lines, allowProposedApi: true });
+    const serializer = new SerializeAddon();
+    term.loadAddon(serializer);
+    const proc = pty.spawn(command, argv, { name: 'xterm-256color', cols, rows: lines, cwd, env: process.env });
+    const j = { started: Date.now(), term, serializer, proc, modes: { 25: 'h', 1006: 'l' }, exited: false };
+    // The write callback keeps the order of the live output and of the screen that showJudge writes.
+    proc.onData((data) => term.write(data, () => {
+      if (ui.view !== rank) return;
+      out.write(data);
+      footer();
+    }));
+    proc.onExit(() => { j.exited = true; });
+    // The headless terminal answers the queries of the harness, such as the cursor position.
+    term.onData((data) => proc.write(data));
+    // The serializer does not keep the cursor visibility (25) or the mouse encoding (1006), so each judge keeps them here.
+    for (const final of ['h', 'l']) {
+      term.parser.registerCsiHandler({ prefix: '?', final }, (params) => {
+        for (const p of params) if (p in j.modes) j.modes[p] = final;
+        return false;
+      });
+    }
+    judges[rank - 1] = j;
+  }
+
+  function drawFooter() {
+    if (!ui.view) return;
+    const { cursorX, cursorY } = judges[ui.view - 1].term.buffer.active;
+    out.write(`\x1b[${out.rows};1H${RESET}${GREY} judge ${ui.view}   ctrl-] scores${RESET}\x1b[K\x1b[${cursorY + 1};${cursorX + 1}H`);
+  }
+
+  // Waits for a pause in the output, so the footer does not land inside an escape sequence that a chunk split.
+  function footer() {
+    clearTimeout(footerTimer);
+    footerTimer = setTimeout(drawFooter, 50);
+  }
+
+  function showJudge(rank) {
+    const j = judges[rank - 1];
+    j.term.write('', () => {
+      if (ui.view !== rank) return;
+      // The table already holds the alternate screen, so a judge on its alternate screen gets a clear after
+      // the normal screen, which the serializer writes first. The scroll region keeps the judge off the footer line.
+      const screen = j.serializer.serialize({ scrollback: 0 }).replace('\x1b[?1049h\x1b[H', '$&\x1b[2J');
+      out.write(`${RESET}\x1b[H\x1b[2J\x1b[1;${out.rows - 1}r${screen}${Object.entries(j.modes).map(([mode, final]) => `\x1b[?${mode}${final}`).join('')}`);
+      drawFooter();
+    });
+  }
+
+  function finish() {
+    const outcomes = rows.map((r) => (r.state === 'done' ? { results: [r.result] } : { error: r.error }));
+    const report = buildReport({ panel, target, cwd, harness, judges: rows.map((_, i) => [i]), outcomes });
+    const reportPath = join(runDir, 'report.json');
+    writeFileSync(join(runDir, 'report.md'), renderMarkdown(report));
+    writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`);
+    final = { status: report.status, path: relative(cwd, reportPath) };
+  }
+
+  function draw() {
+    const lines = render({ title: `${panel.owner}/${panel.name} v${panel.version}`, rows, final, confirm: ui.confirm, width: Math.min(out.columns, 100) });
+    out.write(`\x1b[H${lines.map((l) => `${l}\x1b[K`).join('\n')}\x1b[J`);
+  }
+
+  function tick() {
+    if (!final) {
+      rows = criteria.map((c, i) => ({ ...c, ...state(c, i) }));
+      const live = rows.filter((r) => r.state === 'running').length;
+      for (const r of rows.filter((x) => x.state === 'queued').slice(0, LIVE_JUDGES - live)) {
+        start(r.rank);
+        r.state = 'running';
+      }
+      if (rows.every((r) => r.state === 'done' || r.state === 'error')) finish();
+    }
+    if (!ui.view) draw();
+  }
+
+  return new Promise((done) => {
+    const timer = setInterval(tick, 1000);
+    out.write('\x1b[?1049h\x1b[?25l');
+    process.stdin.setRawMode(true);
+    process.stdin.on('data', (data) => {
+      const key = data.toString();
+      const from = ui.view;
+      const next = press(ui, key, { open: judges.flatMap((j, i) => (j ? [i + 1] : [])), running: rows.some((r) => r.state === 'running') });
+      ui = { view: next.view, confirm: next.confirm };
+      if (next.send) {
+        judges[from - 1].proc.write(key);
+      } else if (next.quit) {
+        clearInterval(timer);
+        // The harness on PATH can be a shell script that runs the real binary as a child and waits for it, so
+        // SIGHUP goes to the process group of the judge. The pty makes each judge the leader of its own group.
+        for (const j of judges) if (j && !j.exited) process.kill(-j.proc.pid, 'SIGHUP');
+        out.write(`${TABLE_MODES}\x1b[?25h\x1b[?1049l`);
+        process.stdin.setRawMode(false);
+        process.stdin.pause();
+        done();
+      } else if (ui.view) {
+        showJudge(ui.view);
+      } else {
+        if (from) out.write(TABLE_MODES);
+        draw();
+      }
+    });
+    out.on('resize', () => {
+      const { cols, rows: lines } = size();
+      for (const j of judges) {
+        if (j && !j.exited) j.proc.resize(cols, lines);
+        j?.term.resize(cols, lines);
+      }
+      if (ui.view) showJudge(ui.view);
+      else draw();
+    });
+    tick();
+  });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xsft/panels",
-  "version": "0.1.0",
+  "version": "0.3.1",
   "private": false,
   "description": "Run a panel of AI judges from cross.software/panels against a target.",
   "type": "module",
@@ -8,5 +8,10 @@
   "files": ["bin", "lib", "prompts", "schema", "README.md"],
   "license": "UNLICENSED",
   "engines": { "node": ">=20" },
-  "scripts": { "test": "node --test test/*.test.js" }
+  "scripts": { "test": "node --test test/*.test.js" },
+  "dependencies": {
+    "@homebridge/node-pty-prebuilt-multiarch": "0.14.1",
+    "@xterm/addon-serialize": "0.14.0",
+    "@xterm/headless": "6.0.0"
+  }
 }

package/lib/split.js

@@ -1,14 +0,0 @@
-/** Returns, for each judge, the indexes of the criteria it scores. n is the number of criteria. */
-export function split(n, size) {
-  if (size <= n) {
-    const judges = [];
-    let start = 0;
-    for (let j = 0; j < size; j++) {
-      const length = Math.floor(n / size) + (j < n % size ? 1 : 0);
-      judges.push(Array.from({ length }, (_, x) => start + x));
-      start += length;
-    }
-    return judges;
-  }
-  return Array.from({ length: n }, (_, i) => Array(Math.floor(size / n) + (i < size % n ? 1 : 0)).fill([i])).flat();
-}