@xsft/panels 0.1.0 → 0.2.0

package/README.md

@@ -1,6 +1,6 @@
 # @xsft/panels
 
-`panels run` runs a panel from cross.software/panels against a target. Each judge is a `claude` or `codex` session that scores one or more criteria of the panel. You need Node 20 or newer and one of the two harnesses.
+`panels run` runs a panel from cross.software/panels against a target. Each judge is a `claude` or `codex` session that scores one criterion of the panel. You need Node 20 or newer and one of the two harnesses.
 
 ```text
 panels run <panel> [target...] [flags]
@@ -8,7 +8,6 @@ panels run <panel> [target...] [flags]
 <panel>           owner/name, or owner/name/vN to pin version N
 [target...]       free words, joined with single spaces into one target
 
---size <n>        number of judges                   default: the number of criteria
 --harness <name>  claude | codex                     default: claude if installed, else codex
 --brief <file>    the brief                          default: none
 --sources <path>  a doc folder, a repository, or an app URL; repeat the flag for each
@@ -26,7 +25,13 @@ npx @xsft/panels run xsft/marketing-site https://example.com --brief docs/brief.
 npx @xsft/panels run xsft/marketing-site/v6 . at http://localhost:4321 --sources docs --harness codex
 ```
 
-Run the command in the folder of the product. Each judge starts in that folder, so `.`, a branch, and relative paths resolve there. The CLI does not test, fetch, or change the target. Four judges run at a time. A judge gets 20 minutes for each of its criteria.
+Run the command in the folder of the product. Each judge starts in that folder, so `.`, a branch, and relative paths resolve there. The CLI does not test, fetch, or change the target. Four judges run at a time. A judge gets 20 minutes.
+
+## tmux mode
+
+On a terminal with `tmux` on PATH, `panels run` opens a tmux session: window 0 shows the scores, and window N runs judge N in the interactive harness. Each judge writes `judges/N/answer.json` and stays open; window 0 writes the report when each judge has an answer or an error, and the command exits from `report.json` when you leave the session. Without a terminal or tmux, the judges run in the background (`claude -p`, `codex exec`).
+
+Keys in window 0: `1` to `9` open the window of that judge (tmux `prefix 0` comes back), `q` ends the session and asks `y` while a judge still runs.
 
 ## Run folder
 
@@ -34,7 +39,8 @@ Each run writes to `.panels/<owner>-<name>/<UTC time>/`:
 
 - `panel.json`: the panel as the api gave it.
 - `report.md` and `report.json`: the scores, the evidence, the fixes, and the errors. `schema/report.json` is the schema of `report.json`.
-- `judges/<k>/`: the prompt, the output schema, and the raw answer of judge k.
+- `judges/<k>/`: the prompt, the output schema, and the raw answer of judge k. In tmux mode, `start.sh` and `answer.json` take the place of the raw answer.
+- `run.json`, in tmux mode: the session name, the harness, the folder, and the target, for window 0.
 
 The last line on stdout is the path of `report.json`.
 
@@ -48,4 +54,4 @@ The last line on stdout is the path of `report.json`.
 - `--model`: each harness uses its default.
 - A panel URL as `<panel>`, a `PANELS_API` override, and a panel cache.
 - A login: an owner cannot run an unpublished panel.
-- A limit on `--size`, cost totals, and `cap` and `gap` fields.
+- Cost totals, and `cap` and `gap` fields.

package/bin/panels.js

@@ -1,11 +1,12 @@
 #!/usr/bin/env node
-import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
 import { delimiter, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
 import { parse, SYNOPSIS } from '../lib/args.js';
-import { runJudge } from '../lib/judge.js';
+import { runJudge, writeJudge } from '../lib/judge.js';
 import { fetchPanel } from '../lib/panel.js';
 import { buildReport, renderMarkdown } from '../lib/report.js';
-import { split } from '../lib/split.js';
 
 // More parallel sessions on one account get 429 answers from the model api.
 const PARALLEL_JUDGES = 4;
@@ -25,6 +26,25 @@ async function pool(items, size, fn) {
   return out;
 }
 
+const EXIT = { pass: 0, fail: 1, error: 2 };
+
+/** Starts the scores window in a new tmux session, attaches to it, and exits from report.json when the attach returns. */
+function runInTmux({ panel, runDir, time, harness, cwd, args }) {
+  const session = `panels-${time.slice(11, 16)}`;
+  writeFileSync(join(runDir, 'run.json'), JSON.stringify({ session, harness, cwd, target: args.target }, null, 2));
+  panel.criteria.forEach((c, i) => writeJudge({ dir: join(runDir, 'judges', String(i + 1)), criteria: [{ rank: i + 1, ...c }], harness, cwd, target: args.target, brief: args.brief, sources: args.sources }));
+  execFileSync('tmux', ['new-session', '-d', '-s', session, '-n', 'scores', process.execPath, fileURLToPath(new URL('../lib/table.js', import.meta.url)), runDir]);
+  if (process.env.TMUX) {
+    execFileSync('tmux', ['switch-client', '-t', session]);
+    return 0;
+  }
+  spawnSync('tmux', ['attach-session', '-t', session], { stdio: 'inherit' });
+  const reportPath = join(runDir, 'report.json');
+  if (!existsSync(reportPath)) return 2;
+  console.log(reportPath);
+  return EXIT[JSON.parse(readFileSync(reportPath, 'utf8')).status];
+}
+
 async function main() {
   const args = parse(process.argv.slice(2));
   if (args.help) {
@@ -51,10 +71,12 @@ async function main() {
   }
   const { panel } = fetched;
   const cwd = process.cwd();
-  const judges = split(panel.criteria.length, args.size ?? panel.criteria.length);
-  const runDir = join(cwd, '.panels', `${panel.owner}-${panel.name}`, new Date().toISOString().replaceAll(':', '-'));
+  const judges = panel.criteria.map((_, i) => [i]);
+  const time = new Date().toISOString().replaceAll(':', '-');
+  const runDir = join(cwd, '.panels', `${panel.owner}-${panel.name}`, time);
   mkdirSync(runDir, { recursive: true });
   writeFileSync(join(runDir, 'panel.json'), JSON.stringify(panel, null, 2));
+  if (process.stdout.isTTY && process.stdin.isTTY && onPath('tmux')) return runInTmux({ panel, runDir, time, harness, cwd, args });
   const outcomes = await pool(judges, PARALLEL_JUDGES, (indexes, j) => runJudge({
     dir: join(runDir, 'judges', String(j + 1)),
     criteria: indexes.map((i) => ({ rank: i + 1, ...panel.criteria[i] })),
@@ -71,7 +93,7 @@ async function main() {
   writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`);
   console.log(markdown);
   console.log(reportPath);
-  return { pass: 0, fail: 1, error: 2 }[report.status];
+  return EXIT[report.status];
 }
 
 // Node exits 1 on a crash, and 1 means "fail" to an agent loop, so a crash prints its stack and exits 2.

package/lib/args.js

@@ -5,7 +5,6 @@ export const SYNOPSIS = `panels run <panel> [target...] [flags]
 <panel>           owner/name, or owner/name/vN to pin version N
 [target...]       free words, joined with single spaces into one target
 
---size <n>        number of judges                   default: the number of criteria
 --harness <name>  claude | codex                     default: claude if installed, else codex
 --brief <file>    the brief                          default: none
 --sources <path>  a doc folder, a repository, or an app URL; repeat the flag for each
@@ -16,7 +15,6 @@ exit 1  fail: every judge answered, and one or more criteria scored below 4
 exit 2  error: bad flags, no target, panel not found, harness missing, or a judge failed`;
 
 const options = {
-  size: { type: 'string' },
   harness: { type: 'string' },
   brief: { type: 'string' },
   sources: { type: 'string', multiple: true },
@@ -39,12 +37,10 @@ export function parse(argv) {
   const match = PANEL.exec(panel);
   if (!match) return { error: `bad panel ${panel}: use owner/name or owner/name/vN` };
   if (words.length === 0) return { error: 'no target: type what the panel reviews after the panel name, such as a URL, a path, or a branch' };
-  if (values.size !== undefined && !/^[1-9]\d*$/.test(values.size)) return { error: `--size needs a whole number of 1 or more, not ${values.size}` };
   if (values.harness !== undefined && !['claude', 'codex'].includes(values.harness)) return { error: `--harness needs claude or codex, not ${values.harness}` };
   return {
     panel: { owner: match[1], name: match[2], version: match[3] && Number(match[3]) },
     target: words.join(' '),
-    size: values.size && Number(values.size),
     harness: values.harness,
     brief: values.brief,
     sources: values.sources ?? [],

package/lib/judge.js

@@ -1,10 +1,12 @@
 import { spawn } from 'node:child_process';
 import { closeSync, mkdirSync, openSync, readFileSync, writeFileSync } from 'node:fs';
 import { dirname, join, relative, resolve, sep } from 'node:path';
+import { fileURLToPath } from 'node:url';
 
 const { judgeResult } = JSON.parse(readFileSync(new URL('../schema/report.json', import.meta.url), 'utf8')).$defs;
-const SYSTEM_PROMPT = readFileSync(new URL('../prompts/judge-system-prompt.txt', import.meta.url), 'utf8');
-const MINUTES_PER_CRITERION = 20;
+const SYSTEM_PROMPT_FILE = fileURLToPath(new URL('../prompts/judge-system-prompt.txt', import.meta.url));
+const SYSTEM_PROMPT = readFileSync(SYSTEM_PROMPT_FILE, 'utf8');
+export const MINUTES_PER_CRITERION = 20;
 const TOOLS = 'Read,Glob,Grep,Bash,WebFetch,WebSearch';
 // With --permission-mode dontAsk, Claude refuses every tool call that this list does not allow, so Bash runs git reads only.
 const ALLOWED_TOOLS = 'Read,Glob,Grep,WebFetch,WebSearch,Bash(git log:*),Bash(git show:*),Bash(git diff:*),Bash(git status:*),Bash(git ls-files:*),Bash(git rev-parse:*),Bash(git branch --list:*)';
@@ -112,7 +114,7 @@ async function askCodex(j, prompt, thread) {
   return { answer: parseAnswer(readFileSync(raw, 'utf8')), session: thread ?? JSON.parse(started).thread_id };
 }
 
-function parseAnswer(text) {
+export function parseAnswer(text) {
   try {
     return JSON.parse(text);
   } catch {
@@ -143,3 +145,32 @@ export async function runJudge({ dir, criteria, harness, cwd, target, brief, sou
   }
   return errors.length ? { error: `invalid answer after one re-ask: ${errors.join(' ')}` } : { results: reply.answer.results };
 }
+
+/** Quotes a string for sh. */
+export const sq = (s) => `'${s.replaceAll("'", "'\\''")}'`;
+
+const TOML_ESCAPES = { '"': '\\"', '\\': '\\\\', '\n': '\\n' };
+
+/** A TOML basic string, for the value of a codex -c flag. */
+export const tomlString = (s) => `"${s.replace(/["\\\x00-\x1f\x7f]/g, (c) => TOML_ESCAPES[c] ?? `\\u${c.charCodeAt(0).toString(16).padStart(4, '0')}`)}"`;
+
+/** The shell command that starts one judge of the tmux mode in its interactive harness. */
+export function startScript({ dir, harness, cwd, brief, sources }) {
+  const prompt = `"$(cat ${sq(join(dir, 'prompt.md'))})"`;
+  if (harness === 'codex') {
+    return `codex -C ${sq(dir)} -s workspace-write --disable multi_agent --disable apps -c ${sq(`developer_instructions=${tomlString(SYSTEM_PROMPT)}`)} ${prompt}\n`;
+  }
+  const add = addDirs(cwd, brief, sources).map((d) => ` --add-dir ${sq(d)}`).join('');
+  // Claude matches file paths only in Edit rules, which cover Write too; // starts an absolute path.
+  const allowed = `${ALLOWED_TOOLS},Edit(/${join(dir, 'answer.json')})`;
+  return `claude --safe-mode --system-prompt "$(cat ${sq(SYSTEM_PROMPT_FILE)})" --tools ${TOOLS},Write${add} --allowedTools ${sq(allowed)} --permission-mode dontAsk ${prompt}\n`;
+}
+
+/** Writes prompt.md, schema.json, and start.sh of one judge of the tmux mode. */
+export function writeJudge({ dir, criteria, harness, cwd, target, brief, sources }) {
+  mkdirSync(dir, { recursive: true });
+  const closing = `The engineer ran the command in ${cwd}. As your last step, write your answer JSON, which matches the schema in ${join(dir, 'schema.json')}, to ${join(dir, 'answer.json')}. Then stop. These two files are the only exceptions to the rules on writing files and on the .panels folder.\n`;
+  writeFileSync(join(dir, 'prompt.md'), `${userMessage(criteria, { target, brief, sources })}\n${closing}`);
+  writeFileSync(join(dir, 'schema.json'), JSON.stringify(judgeSchema(criteria), null, 2));
+  writeFileSync(join(dir, 'start.sh'), startScript({ dir, harness, cwd, brief, sources }));
+}

package/lib/table.js

@@ -0,0 +1,119 @@
+// Window 0 of the tmux mode: `node table.js <run folder>`. It starts the judge windows, shows the scores, and writes the report.
+import { execFileSync, spawnSync } from 'node:child_process';
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { join, relative } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { check, MINUTES_PER_CRITERION, parseAnswer } from './judge.js';
+import { buildReport, renderMarkdown } from './report.js';
+
+// More parallel sessions on one account get 429 answers from the model api.
+const LIVE_JUDGES = 4;
+const ORANGE = '\x1b[38;2;255;106;61m';
+const GREY = '\x1b[90m';
+const BOLD = '\x1b[1m';
+const RESET = '\x1b[0m';
+
+/**
+ * The lines of window 0. rows hold { rank, name, state, result }, and state is queued, running, done, or error.
+ * final is null until the report exists, then { status, path }. confirm is true while q waits for y.
+ */
+export function render({ title, rows, final, confirm, width }) {
+  const rule = `${GREY}${'─'.repeat(width)}${RESET}`;
+  const pad = Math.max(...rows.map((r) => r.name.length)) + 2;
+  const lines = [` ${BOLD}panels run${RESET}  ${title}`, rule];
+  for (const r of rows) {
+    const head = `${String(r.rank).padStart(3)}  ${r.name.padEnd(pad)}`;
+    if (r.state === 'running') lines.push(`${ORANGE}${head}…${RESET}`);
+    else if (r.state === 'queued') lines.push(`${head}${GREY}queued${RESET}`);
+    else if (r.state === 'error') lines.push(`${head}error`);
+    else lines.push(`${head}${r.result.score} ${GREY}b${r.result.band}${RESET}`);
+  }
+  lines.push(rule);
+  if (final) {
+    const at4 = rows.filter((r) => r.state === 'done' && r.result.score === 4).length;
+    lines.push(` ${BOLD}${final.status.toUpperCase()}${RESET}  ${at4} of ${rows.length} at 4`, ` report ${final.path}`);
+  }
+  lines.push(confirm ? ' judges still run. y quits, another key stays' : `${GREY} 1-${Math.min(rows.length, 9)} open judge   q quit${RESET}`);
+  return lines;
+}
+
+function main(runDir) {
+  const read = (file) => JSON.parse(readFileSync(join(runDir, file), 'utf8'));
+  const panel = read('panel.json');
+  const { session, harness, cwd, target } = read('run.json');
+  const criteria = panel.criteria.map((c, i) => ({ rank: i + 1, ...c }));
+  const started = [];
+  let rows = [];
+  let final = null;
+  let confirm = false;
+
+  const judgeDir = (rank) => join(runDir, 'judges', String(rank));
+
+  function state(c, i) {
+    const file = join(judgeDir(c.rank), 'answer.json');
+    if (existsSync(file)) {
+      const answer = parseAnswer(readFileSync(file, 'utf8'));
+      const errors = check(answer, [c]);
+      return errors.length ? { state: 'error', error: `invalid answer: ${errors.join(' ')}` } : { state: 'done', result: answer.results[0] };
+    }
+    if (started[i] === undefined) return { state: 'queued' };
+    if (Date.now() - started[i] > MINUTES_PER_CRITERION * 60_000) return { state: 'error', error: `no answer.json after ${MINUTES_PER_CRITERION} minutes` };
+    return { state: 'running' };
+  }
+
+  function start(rank) {
+    started[rank - 1] = Date.now();
+    execFileSync('tmux', ['new-window', '-d', '-t', `${session}:${rank}`, '-n', `judge-${rank}`, '-c', cwd, 'sh', join(judgeDir(rank), 'start.sh')]);
+  }
+
+  function finish() {
+    const outcomes = rows.map((r) => (r.state === 'done' ? { results: [r.result] } : { error: r.error }));
+    const report = buildReport({ panel, target, cwd, harness, judges: rows.map((_, i) => [i]), outcomes });
+    const reportPath = join(runDir, 'report.json');
+    writeFileSync(join(runDir, 'report.md'), renderMarkdown(report));
+    writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`);
+    final = { status: report.status, path: relative(cwd, reportPath) };
+  }
+
+  function draw() {
+    const lines = render({ title: `${panel.owner}/${panel.name} v${panel.version}`, rows, final, confirm, width: Math.min(process.stdout.columns, 100) });
+    process.stdout.write(`\x1b[H${lines.map((l) => `${l}\x1b[K`).join('\n')}\x1b[J`);
+  }
+
+  function tick() {
+    if (!final) {
+      rows = criteria.map((c, i) => ({ ...c, ...state(c, i) }));
+      const live = rows.filter((r) => r.state === 'running').length;
+      for (const r of rows.filter((x) => x.state === 'queued').slice(0, LIVE_JUDGES - live)) {
+        start(r.rank);
+        r.state = 'running';
+      }
+      if (rows.every((r) => r.state === 'done' || r.state === 'error')) finish();
+    }
+    draw();
+  }
+
+  const quit = () => execFileSync('tmux', ['kill-session', '-t', session]);
+
+  process.stdout.write('\x1b[?1049h\x1b[?25l');
+  process.stdin.setRawMode(true);
+  process.stdin.on('data', (data) => {
+    const key = data.toString();
+    if (confirm) {
+      confirm = false;
+      if (key === 'y') quit();
+    } else if (key === 'q') {
+      if (rows.some((r) => r.state === 'running')) confirm = true;
+      else quit();
+    } else if (/^[1-9]$/.test(key) && Number(key) <= rows.length) {
+      // A queued judge has no window yet, so tmux fails, and its message goes to the ignored pipe.
+      spawnSync('tmux', ['select-window', '-t', `${session}:${key}`]);
+    }
+    draw();
+  });
+  process.stdout.on('resize', draw);
+  tick();
+  setInterval(tick, 1000);
+}
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) main(process.argv[2]);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xsft/panels",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "private": false,
   "description": "Run a panel of AI judges from cross.software/panels against a target.",
   "type": "module",

package/lib/split.js

@@ -1,14 +0,0 @@
-/** Returns, for each judge, the indexes of the criteria it scores. n is the number of criteria. */
-export function split(n, size) {
-  if (size <= n) {
-    const judges = [];
-    let start = 0;
-    for (let j = 0; j < size; j++) {
-      const length = Math.floor(n / size) + (j < n % size ? 1 : 0);
-      judges.push(Array.from({ length }, (_, x) => start + x));
-      start += length;
-    }
-    return judges;
-  }
-  return Array.from({ length: n }, (_, i) => Array(Math.floor(size / n) + (i < size % n ? 1 : 0)).fill([i])).flat();
-}